From a81e5f498f92d82d0658aab2eb33acceb4f7183e Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 10:32:38 +0200 Subject: [PATCH 01/26] Make the rsync host publishable Signed-off-by: Polina Simonenko --- README.md | 19 ++- adapters/host/rsync/rsync_test.go | 22 +++ cmd/snailmail/main.go | 4 +- engine/preview_capability_test.go | 222 ++++++++++++++++++++++++++++++ engine/workspace.go | 50 ++++++- host/host.go | 32 ++++- 6 files changed, 335 insertions(+), 14 deletions(-) create mode 100644 engine/preview_capability_test.go diff --git a/README.md b/README.md index 816822d..834753f 100644 --- a/README.md +++ b/README.md @@ -13,8 +13,9 @@ MIT licensed. See [LICENSE](LICENSE) and [NOTICE.md](NOTICE.md). (`pip`), Helm, and `raw` for artifacts that carry no ecosystem metadata. Debian, RPM, Alpine and Helm are signed with the scheme their own clients verify. -**Three hosts today.** A local directory, GitHub Pages, and S3-compatible object -storage — though not every format on every host; see the table below. +**Four hosts today.** A local directory, a directory reached over ssh, GitHub +Pages, and S3-compatible object storage — though not every format on every host; +see the table below. **Planned, not built:** npm, OCI, Cargo, Go, Maven and Nix repositories, and publishing into registries you do not own (AUR, Homebrew, nixpkgs, npmjs, PyPI, @@ -56,6 +57,7 @@ Every command that takes `--workspace` also accepts `--root`. | Host | pypi | deb | rpm | apk | helm | raw | |---|---|---|---|---|---|---| | local directory | yes | yes | yes | yes | yes | yes | +| ssh directory (`rsync`) | yes | yes | yes | yes | yes | yes | | GitHub Pages | yes | yes | yes | yes | yes | yes | | S3 / R2 / GCS | yes | — | unsigned only | — | yes | yes | @@ -63,8 +65,12 @@ An object store makes a revision live by writing one object, so it can serve a format only where a single path switches. Debian needs a `Release` and its detached signature to become live together; Alpine has one index per architecture; and a *signed* yum repository has to switch `repomd.xml` with the `repomd.xml.asc` that -signs it. Those are structural limits, not missing work — a local directory or -GitHub Pages commits a whole tree at once and serves all six. +signs it. Those are structural limits, not missing work — a local directory, an +ssh directory or GitHub Pages commits a whole tree at once and serves all six. + +The `rsync` host serves every format but publishes no preview site, so it works +under the `auto` gate and is refused under `pr` and `approval`, which exist to +review one. [Publishing over ssh](#publishing-over-ssh) has the detail. One caveat for object storage: the browsable `index.html` is generated fresh for every revision, so it is kept with the release rather than at the repository root. @@ -445,6 +451,11 @@ Requirements and limits, stated because they are not checked from this side: establishing that the release is still intact is not, because nothing on the far side verifies it. The adapter declines rather than claiming a rollback it cannot check. +- **No preview site**, so the `pr` and `approval` gates are refused. Nothing is + copied to the far side before the commit, so there is no URL a reviewer could + install from. Under the `auto` gate a real client still installs from the exact + staged bytes before they are published; what goes unchecked is that the far side + serves them correctly, which is what a preview buys. ## Browsing a bucket-hosted repository diff --git a/adapters/host/rsync/rsync_test.go b/adapters/host/rsync/rsync_test.go index a5f2cf0..cb91183 100644 --- a/adapters/host/rsync/rsync_test.go +++ b/adapters/host/rsync/rsync_test.go @@ -371,6 +371,28 @@ func TestRestoreIsDeclined(t *testing.T) { } } +// The declared capabilities, pinned. +// +// FaithfulPreview is the one that matters: Stage copies nothing to the far side, +// so there is no URL a client could install from before a revision is live, and +// claiming otherwise would have the engine verify against an endpoint that does +// not exist. Reporting it false once made this host unpublishable, because the +// planner required a preview of every repository rather than only of the gates +// that review one — so the honest answer is asserted here, and the engine's +// handling of it is covered by TestPlanAndApplyPreviewlessHostUnderAutoGate. +func TestTheDeclaredCapabilities(t *testing.T) { + adapter := New(&localRunner{}) + repository, _ := publishedRepository(t) + capabilities, err := adapter.Capabilities(context.Background(), repository) + if err != nil { + t.Fatal(err) + } + want := host.Capabilities{ConditionalCommit: true} + if capabilities != want { + t.Fatalf("capabilities = %#v, want %#v", capabilities, want) + } +} + func TestTheAdapterIsAHost(t *testing.T) { var _ host.Host = New(&localRunner{}) } diff --git a/cmd/snailmail/main.go b/cmd/snailmail/main.go index fee99cd..f2f6fc2 100644 --- a/cmd/snailmail/main.go +++ b/cmd/snailmail/main.go @@ -157,14 +157,14 @@ func runInit(args []string, stdout, stderr io.Writer) error { func runSetup(args []string, stdout, stderr io.Writer) error { if len(args) == 0 { - return errors.New("usage: snailmail setup --name NAME --host [host options]") + return errors.New("usage: snailmail setup --name NAME --host [host options]") } format := args[0] flags := newCommandFlags("setup "+format, stderr).withWorkspace().withJSON() name := flags.String("name", "", "repository name") output := flags.String("output", "", "published directory: workspace-relative for a local host, an absolute remote path for rsync") - hostType := flags.String("host", "local", "host type: local, s3, or github-pages") + hostType := flags.String("host", "local", "host type: local, s3, rsync, or github-pages") visibility := flags.String("visibility", "public", "repository visibility") gatePolicy := flags.String("gate", "auto", "publication gate: auto, pr, or approval") approvalKeys := flags.String("approval-keys", "", "comma-separated allowed Ed25519 public keys") diff --git a/engine/preview_capability_test.go b/engine/preview_capability_test.go new file mode 100644 index 0000000..46d360c --- /dev/null +++ b/engine/preview_capability_test.go @@ -0,0 +1,222 @@ +package engine + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/shellcell/snailmail/host" + "github.com/shellcell/snailmail/internal/state" +) + +// A host that publishes but serves no preview endpoint. +// +// This is the rsync host's shape: it copies nothing to the far side before +// Commit, so there is no URL a client could install from before a revision is +// live, and it reports FaithfulPreview false rather than claiming one. Every +// other host here serves a preview, so nothing in the suite covered a host +// without one — which is how the planner came to require the capability of +// every repository and made rsync unpublishable. +type previewlessHost struct { + stageCalls int + commitCalls int + revision host.PublishedRevision +} + +func (remote *previewlessHost) Capabilities(context.Context, host.Repository) (host.Capabilities, error) { + return host.Capabilities{ConditionalCommit: true}, nil +} + +func (remote *previewlessHost) Observe(context.Context, host.Repository) (host.PublishedRevision, error) { + return remote.revision, nil +} + +// No endpoint: a previewless host has nothing to hand a client before commit. +func (remote *previewlessHost) ReadAccess(context.Context, host.Repository, host.PublishedRevision) (host.ClientAccess, error) { + return host.ClientAccess{}, nil +} + +func (remote *previewlessHost) Stage(_ context.Context, _ host.Repository, request host.StageRequest) (host.StagedPublication, error) { + remote.stageCalls++ + return host.StagedPublication{ + ID: request.PlanID + ":" + request.ChangeID, PlanID: request.PlanID, ChangeID: request.ChangeID, + PreviousRevision: request.PreviousRevision, TreeSHA256: request.TreeSHA256, + Files: request.Files, CommitPaths: request.CommitPaths, + }, nil +} + +func (remote *previewlessHost) Commit(_ context.Context, _ host.Repository, staged host.StagedPublication, + _ host.ExpectedRevision) (host.CommitResult, error) { + remote.commitCalls++ + remote.revision = host.PublishedRevision{ + NativeRevision: staged.TreeSHA256, TreeSHA256: staged.TreeSHA256, + PlanID: staged.PlanID, ChangeID: staged.ChangeID, + } + return host.CommitResult{Revision: remote.revision, CanonicalEndpoint: "https://packages.example/tools"}, nil +} + +// Declining, like the real rsync adapter: it cannot verify the release it would +// roll back to, so it never claims ConditionalRestore. +func (remote *previewlessHost) Restore(context.Context, host.Repository, host.RestoreRef, host.ExpectedRevision) (host.PublishedRevision, error) { + return host.PublishedRevision{}, errors.New("restore is not offered") +} + +func (remote *previewlessHost) Abort(context.Context, host.Repository, host.StagedPublication) error { + return nil +} + +// previewlessWorkspace builds a raw repository on the rsync host with one +// artifact recorded and committed. +// +// raw is deliberate: its verification is pure Go, so this exercises the real +// client-verification branch rather than skipping it, without needing a +// container runtime. +func previewlessWorkspace(t *testing.T, gate string) string { + t.Helper() + root := t.TempDir() + command := exec.Command("git", "init", "-b", "main") + command.Dir = root + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, output) + } + initRequest := InitWorkspaceRequest{Root: root, Name: "previewless"} + if gate == "pr" { + initRequest.Forge, initRequest.ForgeRepository = "github", "shellcell/state" + } + if err := InitWorkspace(initRequest); err != nil { + t.Fatal(err) + } + setup := SetupRepositoryRequest{ + Root: root, Name: "tools", Format: "raw", HostType: "rsync", Visibility: "public", + Target: "deploy@packages.example", Output: "/srv/www/tools", + CanonicalEndpoint: "https://packages.example/tools", Gate: gate, + } + if gate == "approval" { + // An approval gate needs a key to check against; the plan is refused + // before any signature is read, so any valid Ed25519 public key does. + setup.ApprovalKeys = []string{"kBUq5vJt6gYFvJEJPD2yjBGYLzuYqNRTPUxUvJ6qHUM="} + } + if err := SetupRepository(setup); err != nil { + t.Fatal(err) + } + // Content unique to this test. Facts are memoised per process by content + // digest, so two tests sharing bytes under different filenames share a cache + // entry and the second reads the first's architecture. + artifact := filepath.Join(t.TempDir(), "ttysvg_0.1.2_linux_amd64.tar.gz") + if err := os.WriteFile(artifact, []byte("previewless host payload"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := AddArtifacts(AddArtifactsRequest{ + Context: context.Background(), Root: root, Repository: "tools", Artifacts: []string{artifact}, + }); err != nil { + t.Fatal(err) + } + commitWorkspace(t, root, "record raw artifact") + return root +} + +// A host that serves no preview must still publish under the auto gate. +// +// Two independent defects made this impossible and each hid the other: the +// planner required FaithfulPreview of every repository, and validateApplyPlan +// carried its own list of host types that had never gained "rsync". Fixing +// either alone leaves the other, so this covers plan and apply together. +func TestPlanAndApplyPreviewlessHostUnderAutoGate(t *testing.T) { + root := previewlessWorkspace(t, "auto") + remote := &previewlessHost{} + resolver := staticHostResolver{host: remote} + createdAt := time.Date(2026, time.August, 22, 1, 2, 3, 0, time.UTC) + planName := filepath.Join(root, "previewless.json") + + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, + ExpiresIn: time.Hour, Hosts: resolver, + }); err != nil { + t.Fatalf("planning a repository on a previewless host failed: %v", err) + } + + plan, err := state.LoadPlan(planName) + if err != nil { + t.Fatal(err) + } + if plan.Payload.Repositories[0].Host.Type != "rsync" { + t.Fatalf("plan did not record the rsync host: %#v", plan.Payload.Repositories[0]) + } + if plan.Payload.Repositories[0].FaithfulPreview { + t.Fatal("plan claimed a faithful preview the host does not serve") + } + + // Not StructuralOnly: the point is that the real client verification path + // runs against the staged tree when there is no endpoint to install from. + result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + Root: root, Plan: planName, now: createdAt.Add(time.Minute), Hosts: resolver, + }) + if err != nil { + t.Fatalf("applying to a previewless host failed: %v", err) + } + if result.Applied != 1 || remote.stageCalls != 1 || remote.commitCalls != 1 { + t.Fatalf("unexpected apply result %#v stage=%d commit=%d", result, remote.stageCalls, remote.commitCalls) + } +} + +// A gate that waits for a person to review a preview cannot be satisfied by a +// host that serves none, and the refusal has to say which of the two to change. +func TestPlanRefusesPreviewlessHostUnderHumanGate(t *testing.T) { + for _, gate := range []string{"pr", "approval"} { + t.Run(gate, func(t *testing.T) { + root := previewlessWorkspace(t, gate) + createdAt := time.Date(2026, time.August, 22, 1, 2, 3, 0, time.UTC) + _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + Root: root, Output: filepath.Join(root, "refused.json"), + createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, + Hosts: staticHostResolver{host: &previewlessHost{}}, + }) + if err == nil { + t.Fatalf("gate %q planned against a host serving no preview", gate) + } + if !strings.Contains(err.Error(), gate) || !strings.Contains(err.Error(), "preview") { + t.Fatalf("refusal names neither the gate nor the preview: %v", err) + } + }) + } +} + +// The capability is required of the gates that review a preview and of nothing +// else, so a host without one is not refused for a reason that does not apply. +func TestRequireHostCapabilitiesSeparatesEachReason(t *testing.T) { + previewless := host.Capabilities{ConditionalCommit: true} + for _, testcase := range []struct { + name string + repository state.Repository + reported host.Capabilities + wants string + }{ + {"auto gate needs no preview", state.Repository{Gate: "auto", Visibility: "public"}, previewless, ""}, + {"pr gate needs a preview", state.Repository{Gate: "pr", Visibility: "public"}, previewless, "preview"}, + {"conditional commit is always required", state.Repository{Gate: "auto", Visibility: "public"}, host.Capabilities{}, "conditionally"}, + { + "private needs scoped credentials", + state.Repository{Gate: "auto", Visibility: "private"}, + previewless, + "scoped read credentials", + }, + } { + t.Run(testcase.name, func(t *testing.T) { + err := requireHostCapabilities("tools", testcase.repository, testcase.reported) + if testcase.wants == "" { + if err != nil { + t.Fatalf("unexpected refusal: %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), testcase.wants) { + t.Fatalf("want a refusal mentioning %q, got %v", testcase.wants, err) + } + }) + } +} diff --git a/engine/workspace.go b/engine/workspace.go index 85617fd..89a3c60 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -599,8 +599,8 @@ func (preparation *planPreparation) planRepository(name string) (state.PlanRepos if err != nil { return state.PlanRepository{}, nil, err } - if !capabilities.FaithfulPreview || !capabilities.ConditionalCommit || (repository.Visibility == "private" && !capabilities.PrivateRead) { - return state.PlanRepository{}, nil, fmt.Errorf("repository %q host cannot provide verified conditional publication", name) + if err := requireHostCapabilities(name, repository, capabilities); err != nil { + return state.PlanRepository{}, nil, err } observed, err := selectedHost.Observe(ctx, hostRepository) if err != nil { @@ -667,6 +667,42 @@ func (preparation *planPreparation) planRepository(name string) (state.PlanRepos }, planned, nil } +// requireHostCapabilities refuses a repository whose host cannot carry out the +// publication its configuration asks for. +// +// A faithful preview is required only where a human gate waits on one. PLAN.md +// §"Phase 3" states the rule: a preview site exists so a reviewer can install +// from a revision before it becomes live, so under an `auto` gate its absence is +// a stated trade rather than a fault. stageOnHosts already implements that +// trade — with no preview endpoint it runs the real client against the staged +// tree, exactly as a local host is verified — so requiring the capability of +// every repository refused hosts the apply path was already built to serve. +// +// The rsync host is what that cost. It serves every format, publishes atomically +// by renaming a symlink, and copies nothing to the far side before Commit — so +// it has no preview URL and honestly reports FaithfulPreview false. An +// unconditional requirement here meant `plan` refused it before `apply` was +// ever reached, and every rsync repository was unpublishable. +// +// Each capability is reported separately: "cannot provide verified conditional +// publication" named none of them, and the operator's next move is different for +// each. +func requireHostCapabilities(name string, repository state.Repository, capabilities host.Capabilities) error { + if !capabilities.ConditionalCommit { + return fmt.Errorf("repository %q host cannot commit conditionally, so a publication could overwrite a revision it did not expect", name) + } + if repository.Visibility == "private" && !capabilities.PrivateRead { + return fmt.Errorf("repository %q is private, but its host cannot issue scoped read credentials", name) + } + // pr and approval both hold a publication open for a person to look at. That + // is the only thing a preview is for. + if (repository.Gate == "pr" || repository.Gate == "approval") && !capabilities.FaithfulPreview { + return fmt.Errorf("repository %q uses the %q gate, which reviews a preview, but its host serves none; use the auto gate or a host with a preview site", + name, repository.Gate) + } + return nil +} + func PlanWorkspace(ctx context.Context, request PlanWorkspaceRequest) (PlanWorkspaceResult, error) { root, err := workspaceRoot(request.Root) if err != nil { @@ -1825,8 +1861,14 @@ func validateApplyPlan(plan state.Plan) error { } } } - if repository.Host.Type != "local" && repository.Host.Type != "s3" && repository.Host.Type != "github-pages" { - return fmt.Errorf("plan repository %q has unsupported host type %q", repository.Name, repository.Host.Type) + // Read from the declared support matrix rather than a list repeated here. + // The list had drifted: it never gained "rsync", so a plan for a host that + // serves every format was refused at apply even once planning allowed it. + // Asking the matrix also validates the pair rather than only the host, so a + // format a host cannot serve is refused with the same check. + if !host.Supports(repository.Host.Type, repository.Format).Publish { + return fmt.Errorf("plan repository %q has unsupported host type %q for format %q", + repository.Name, repository.Host.Type, repository.Format) } if repository.CanonicalEndpoint == "" { return fmt.Errorf("plan repository %q has no canonical endpoint", repository.Name) diff --git a/host/host.go b/host/host.go index ecaf372..0220f47 100644 --- a/host/host.go +++ b/host/host.go @@ -81,11 +81,35 @@ type Repository struct { PreviewEndpoint string } +// Capabilities is what one host can offer a publication. Each is reported +// rather than inferred from the host's type, so the engine asks what a host can +// do instead of knowing which hosts exist. type Capabilities struct { - FaithfulPreview bool - ConditionalCommit bool - ConditionalRestore bool - PrivateRead bool + // FaithfulPreview reports whether the host serves a staged revision at an + // endpoint behaving like its canonical one — same relative paths, encoding, + // headers and authentication — so a client can install from a revision + // before it becomes live. ARCHITECTURE §"Host" describes the layout each + // host class uses for this. + // + // False is a legitimate answer, not a defect: the rsync host copies nothing + // to the far side before Commit, so there is no URL to preview. A repository + // on such a host is still verified by a real client against the staged tree; + // what goes unchecked is that the host serves that tree correctly. Only a + // gate that waits for a person to review a preview requires this. + FaithfulPreview bool + // ConditionalCommit reports whether the host refuses a publication when the + // live revision is not the one the plan expected. Required of every host: + // without it a concurrent publisher is silently overwritten. + ConditionalCommit bool + // ConditionalRestore reports whether the host can put the prior revision + // back, conditional on the failed one still being live. A host without it is + // never given an automatic-rollback promise. + ConditionalRestore bool + // PrivateRead reports whether the host can issue scoped read credentials for + // a private repository. + PrivateRead bool + // CredentialBrokerIdentity names the broker that issued those credentials, + // pinned in the plan so the identity cannot change between plan and apply. CredentialBrokerIdentity string } From 9c6cc15f139e8c41652dab36051cb1cfa304b839 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 10:49:52 +0200 Subject: [PATCH 02/26] Let two artifacts of one version hold identical bytes Signed-off-by: Polina Simonenko --- engine/shared_blob_test.go | 114 +++++++++++++++++++++++++++++++++++++ engine/status.go | 6 +- engine/workspace.go | 15 ++--- internal/state/cas.go | 35 +++++++++++- internal/state/ledger.go | 34 +++++++++-- internal/status/render.go | 10 ++-- 6 files changed, 186 insertions(+), 28 deletions(-) create mode 100644 engine/shared_blob_test.go diff --git a/engine/shared_blob_test.go b/engine/shared_blob_test.go new file mode 100644 index 0000000..c99853f --- /dev/null +++ b/engine/shared_blob_test.go @@ -0,0 +1,114 @@ +package engine + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/shellcell/snailmail/internal/state" +) + +// Two artifacts of one version can hold identical bytes. +// +// One architecture-independent installer shipped for amd64 and arm64 is the +// ordinary case, and it broke four ways at once. Facts were memoised by content +// digest alone, so the arm64 entry read the amd64 entry's architecture and the +// workspace was refused with "blob content is corrupt" — a valid lock, valid +// bytes, and an accusation true of neither. Behind that, three separate places +// built a version's blob binding by sorting without reducing to a set, while +// every validator of that binding required strictly increasing digests: apply +// refused the publication, and status reported the binding incomplete forever. +// +// raw is the only format this reaches, because it is the only one whose +// identity does not come from the bytes. +func TestOneVersionMayHoldTwoArtifactsWithIdenticalBytes(t *testing.T) { + root := t.TempDir() + command := exec.Command("git", "init", "-b", "main") + command.Dir = root + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, output) + } + if err := InitWorkspace(InitWorkspaceRequest{Root: root, Name: "shared-blob"}); err != nil { + t.Fatal(err) + } + if err := SetupRepository(SetupRepositoryRequest{ + Root: root, Name: "tools", Format: "raw", HostType: "local", + Output: "public/tools", Visibility: "public", + }); err != nil { + t.Fatal(err) + } + + // Identical bytes, and a filename convention that reads a different + // architecture out of each. + staging := t.TempDir() + installer := []byte("#!/bin/sh\necho install\n") + for _, architecture := range []string{"amd64", "arm64"} { + name := filepath.Join(staging, "installer_1.0.0_linux_"+architecture+".tar.gz") + if err := os.WriteFile(name, installer, 0o644); err != nil { + t.Fatal(err) + } + if _, err := AddArtifacts(AddArtifactsRequest{ + Context: context.Background(), Root: root, Repository: "tools", Artifacts: []string{name}, + }); err != nil { + t.Fatalf("adding the %s installer failed: %v", architecture, err) + } + } + commitWorkspace(t, root, "one installer, two architectures") + + planName := filepath.Join(root, "shared.snailmail-plan.json") + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName}); err != nil { + t.Fatalf("planning a version whose artifacts share bytes failed: %v", err) + } + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName}); err != nil { + t.Fatalf("applying a version whose artifacts share bytes failed: %v", err) + } + + // Both architectures are published, each at its own path, from one blob. + sums, err := os.ReadFile(filepath.Join(root, "public", "tools", "SHA256SUMS")) + if err != nil { + t.Fatal(err) + } + for _, architecture := range []string{"amd64", "arm64"} { + if !strings.Contains(string(sums), "installer_1.0.0_linux_"+architecture+".tar.gz") { + t.Fatalf("%s was not published: %s", architecture, sums) + } + } + + status, err := StatusWorkspace(context.Background(), StatusWorkspaceRequest{Root: root}) + if err != nil { + t.Fatal(err) + } + if state := status.Repositories[0].VisibleBindingState; state != "complete" { + t.Fatalf("binding state = %q, want complete", state) + } + + // The binding has to settle: while the recorded set and the derived list + // could not compare equal, every plan re-recorded it and never converged. + settled := filepath.Join(root, "settled.snailmail-plan.json") + result, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: settled}) + if err != nil { + t.Fatal(err) + } + if result.Changes != 0 { + t.Fatalf("replanning an applied workspace found %d changes, want none", result.Changes) + } +} + +func TestBlobDigestsIsASet(t *testing.T) { + version := state.PackageVersion{Blobs: []state.LockedBlob{ + {SHA256: "bb"}, {SHA256: "aa"}, {SHA256: "bb"}, {SHA256: "cc"}, {SHA256: "aa"}, + }} + got := state.BlobDigests(version) + want := []string{"aa", "bb", "cc"} + if len(got) != len(want) { + t.Fatalf("BlobDigests = %v, want %v", got, want) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("BlobDigests = %v, want %v", got, want) + } + } +} diff --git a/engine/status.go b/engine/status.go index d30bdf8..01cdf85 100644 --- a/engine/status.go +++ b/engine/status.go @@ -144,11 +144,7 @@ func StatusWorkspace(ctx context.Context, request StatusWorkspaceRequest) (Statu statusRepository.VisibleBindingState = "incomplete" } for _, packageVersion := range visible { - digests := make([]string, 0, len(packageVersion.Blobs)) - for _, locked := range packageVersion.Blobs { - digests = append(digests, locked.SHA256) - } - sort.Strings(digests) + digests := state.BlobDigests(packageVersion) binding := "complete" if missingBindings[packageVersion.Package+"\x00"+packageVersion.Version] { binding = "incomplete" diff --git a/engine/workspace.go b/engine/workspace.go index 89a3c60..213c787 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -1494,11 +1494,7 @@ func missingPublicationBindings(lock state.RepositoryLock, repository state.Repo } var missing []state.PlanPublicationBinding for _, version := range visiblePackageVersions(lock, repository) { - digests := make([]string, 0, len(version.Blobs)) - for _, artifact := range version.Blobs { - digests = append(digests, artifact.SHA256) - } - sort.Strings(digests) + digests := state.BlobDigests(version) if !reflect.DeepEqual(published[version.Package+"\x00"+version.Version], digests) { missing = append(missing, state.PlanPublicationBinding{Package: version.Package, Version: version.Version, BlobSHA256: digests}) } @@ -1509,12 +1505,9 @@ func missingPublicationBindings(lock state.RepositoryLock, repository state.Repo func publicationBindingsForVersions(versions []state.PackageVersion) []state.PlanPublicationBinding { bindings := make([]state.PlanPublicationBinding, 0, len(versions)) for _, version := range versions { - digests := make([]string, 0, len(version.Blobs)) - for _, artifact := range version.Blobs { - digests = append(digests, artifact.SHA256) - } - sort.Strings(digests) - bindings = append(bindings, state.PlanPublicationBinding{Package: version.Package, Version: version.Version, BlobSHA256: digests}) + bindings = append(bindings, state.PlanPublicationBinding{ + Package: version.Package, Version: version.Version, BlobSHA256: state.BlobDigests(version), + }) } return bindings } diff --git a/internal/state/cas.go b/internal/state/cas.go index e2b3a28..6828160 100644 --- a/internal/state/cas.go +++ b/internal/state/cas.go @@ -468,7 +468,24 @@ func validateLockedBlobOpenContext(ctx context.Context, file *os.File, pathInfo legacyDigestConflict(locked, LockedBlob{MD5: validated.MD5, SHA1: validated.SHA1}) { return domain.Blob{}, fmt.Errorf("%w: blob sha256:%s disagrees with its lock", blob.ErrCorrupt, locked.SHA256) } - facts, cached := factscache.Lookup(format, validated.SHA256) + // Memoised by content digest, which identifies the facts only where the + // facts come from the content. A raw artifact carries no metadata, so its + // name, version and architecture are read from the filename or supplied by + // the operator — two lock entries can hold identical bytes and legitimately + // differ. Publishing one architecture-independent installer for amd64 and + // arm64 is exactly that, and reusing the first entry's facts for the second + // failed the architecture check below: a valid workspace was refused for the + // rest of its life with "blob content is corrupt", accusing the operator's + // bytes of something that was true of neither them nor the lock. + // + // Nothing is lost by not caching these. The memo exists because inspecting a + // package can decompress hundreds of megabytes; inspecting a raw artifact + // parses its filename. + memoisable := formatDerivesIdentityFromBytes(format) + facts, cached := domain.PackageFacts{}, false + if memoisable { + facts, cached = factscache.Lookup(format, validated.SHA256) + } if !cached { if _, err := file.Seek(0, io.SeekStart); err != nil { return domain.Blob{}, fmt.Errorf("%w: seek blob sha256:%s: %w", blob.ErrUnavailable, locked.SHA256, err) @@ -480,7 +497,9 @@ func validateLockedBlobOpenContext(ctx context.Context, file *os.File, pathInfo } return domain.Blob{}, fmt.Errorf("%w: inspect blob sha256:%s: %v", blob.ErrCorrupt, locked.SHA256, err) } - factscache.Store(format, validated.SHA256, facts) + if memoisable { + factscache.Store(format, validated.SHA256, facts) + } } validated.Facts = facts if facts.Architecture != locked.Architecture { @@ -561,6 +580,18 @@ func inspect(format, filename string, reader io.ReaderAt, size int64, supplied f return selected.Inspect(filename, reader, size, supplied) } +// formatDerivesIdentityFromBytes reports whether a format's facts are a +// function of the content alone, which is what makes them safe to memoise by +// content digest. An unknown format is not memoisable: inspect refuses it +// anyway, and guessing yes here would be the unsafe direction. +func formatDerivesIdentityFromBytes(format string) bool { + selected, err := formats.For(format) + if err != nil { + return false + } + return selected.DerivesIdentityFromBytes() +} + func formatMaximum(format string) (int64, error) { selected, err := formats.For(format) if err != nil { diff --git a/internal/state/ledger.go b/internal/state/ledger.go index 1700da3..d2dacc3 100644 --- a/internal/state/ledger.go +++ b/internal/state/ledger.go @@ -195,12 +195,12 @@ func ValidatePublishedBindings(lock RepositoryLock, records []PublicationRecord) lockBindings := make(map[string][]string) for _, packageVersion := range lock.PackageVersion { key := packageVersion.Package + "\x00" + packageVersion.Version - lockBindings[key] = blobDigests(packageVersion) + lockBindings[key] = BlobDigests(packageVersion) published := bindings[key] if published == nil { continue } - current := blobDigests(packageVersion) + current := BlobDigests(packageVersion) if !equalStrings(published, current) { return fmt.Errorf("published package %s@%s cannot change bytes", packageVersion.Package, packageVersion.Version) } @@ -378,7 +378,7 @@ func updatedPublicationRecords(existing []PublicationRecord, repository, planID, SchemaVersion: LedgerSchema, PlanID: planID, ChangeID: changeID, Repository: repository, Package: packageVersion.Package, Version: packageVersion.Version, - BlobSHA256: blobDigests(packageVersion), TreeSHA256: treeSHA, RecordedAt: recordedAt, + BlobSHA256: BlobDigests(packageVersion), TreeSHA256: treeSHA, RecordedAt: recordedAt, } if previous, exists := seen[key]; exists { if !publicationRecordEqual(previous, candidate) { @@ -410,13 +410,37 @@ func publicationRecordEqual(left, right PublicationRecord) bool { left.TreeSHA256 == right.TreeSHA256 && left.RecordedAt == right.RecordedAt && equalStrings(left.BlobSHA256, right.BlobSHA256) } -func blobDigests(packageVersion PackageVersion) []string { +// BlobDigests is the set of blob digests a package version is bound to, sorted +// and without repetition. +// +// A set rather than a list, because two artifacts of one version can hold +// identical bytes and are then bound to one blob rather than two. A raw version +// publishing one architecture-independent installer for both amd64 and arm64 is +// exactly that. Listing the digest twice said nothing the first did not, and +// every validator of this list requires strictly increasing digests — which is +// the same statement — so the repetition was read as a corrupt binding and +// refused the publication. +func BlobDigests(packageVersion PackageVersion) []string { digests := make([]string, 0, len(packageVersion.Blobs)) for _, blob := range packageVersion.Blobs { digests = append(digests, blob.SHA256) } sort.Strings(digests) - return digests + return compactSorted(digests) +} + +// compactSorted removes adjacent repeats from a sorted slice in place. +func compactSorted(sorted []string) []string { + kept := sorted[:0] + previous := "" + for index, value := range sorted { + if index != 0 && value == previous { + continue + } + kept = append(kept, value) + previous = value + } + return kept } func equalStrings(left, right []string) bool { diff --git a/internal/status/render.go b/internal/status/render.go index 9bf29ab..be04494 100644 --- a/internal/status/render.go +++ b/internal/status/render.go @@ -118,14 +118,14 @@ func publicEndpoint(repository state.Repository) string { return repository.Host.CanonicalEndpoint } +// equalDigests compares a recorded binding with the lock's current one. Both +// are sets: a ledger record holds each distinct blob once, so the lock side is +// reduced the same way rather than reporting a version as unpublished because +// two of its artifacts hold identical bytes. func equalDigests(recorded []string, blobs []state.LockedBlob) bool { expected := append([]string(nil), recorded...) - actual := make([]string, 0, len(blobs)) - for _, blob := range blobs { - actual = append(actual, blob.SHA256) - } sort.Strings(expected) - sort.Strings(actual) + actual := state.BlobDigests(state.PackageVersion{Blobs: blobs}) return strings.Join(expected, "\x00") == strings.Join(actual, "\x00") } From b984f7c1a8203c6d4801f20e9d99ea270c0a2caa Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 11:34:38 +0200 Subject: [PATCH 03/26] Add a short test mode for the inner loop --- Makefile | 8 ++++++++ engine/engine_test.go | 30 +++++++++++++++++++++-------- engine/workspace_test.go | 2 +- internal/app/deb_endpoint_test.go | 5 +++++ internal/app/platform_image_test.go | 3 +++ internal/app/pypi_endpoint_test.go | 6 ++++++ internal/state/lockshard_test.go | 24 +++++++++++++++++++++++ 7 files changed, 69 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 604e8ee..0dab0e9 100644 --- a/Makefile +++ b/Makefile @@ -67,6 +67,14 @@ vet: ## Vet the default build and the one with S3 compiled out test: ## Run the suite go test -count=1 ./... +.PHONY: test-short +# The inner loop. Leaves out what needs a container, a pip install, or a lock +# sharded past its threshold — everything whose cost is the machine rather than +# the code. `check` and CI still run the full suite, so this skips nothing +# permanently, only between edits. +test-short: ## Run the suite without the slow machine-dependent tests + go test -short -count=1 ./... + .PHONY: test-race # The race detector is written in C, so it needs cgo — and the CGO_ENABLED=0 above # turns that into "go: -race requires cgo" rather than into a static binary. That diff --git a/engine/engine_test.go b/engine/engine_test.go index 1fedee0..b5a6205 100644 --- a/engine/engine_test.go +++ b/engine/engine_test.go @@ -113,10 +113,7 @@ func TestVerifyAcceptsPhaseTwoRepositoryManifest(t *testing.T) { } func TestVerifyPyPIInstallsWithPip(t *testing.T) { - python, err := exec.LookPath("python3") - if err != nil { - t.Skip("python3 is unavailable") - } + python := availablePython(t) if err := exec.Command(python, "-m", "pip", "--version").Run(); err != nil { t.Skip("pip is unavailable") } @@ -141,10 +138,7 @@ func TestVerifyPyPIInstallsWithPip(t *testing.T) { } func TestVerifyPyPIFailsWhenDependencyIsMissing(t *testing.T) { - python, err := exec.LookPath("python3") - if err != nil { - t.Skip("python3 is unavailable") - } + python := availablePython(t) input := t.TempDir() if _, err := testutil.WriteWheelWithDependencies(input, "Demo-Pkg", "1.2.3", "", []string{"Missing-Pkg == 9.9.9"}); err != nil { t.Fatal(err) @@ -429,6 +423,12 @@ func readTree(t *testing.T, root string) map[string]string { // a noexec tmpfs broke this path unnoticed. --pull=missing fetches the image. func availableContainerRunner(t *testing.T) string { t.Helper() + // -short is the developer's loop. Running a real apt, dnf, apk or helm inside + // a container is the slowest thing here and the most dependent on the machine, + // so it is what a fast run gives up. CI runs the full suite. + if testing.Short() { + t.Skip("-short skips client verification in a container") + } for _, candidate := range []string{"podman", "docker"} { if _, err := exec.LookPath(candidate); err == nil { return candidate @@ -438,6 +438,20 @@ func availableContainerRunner(t *testing.T) string { return "" } +// availablePython finds the interpreter a pip install is run with. Skipped by +// -short for the same reason as a container: the install is what costs. +func availablePython(t *testing.T) string { + t.Helper() + if testing.Short() { + t.Skip("-short skips client verification with pip") + } + python, err := exec.LookPath("python3") + if err != nil { + t.Skip("python3 is unavailable") + } + return python +} + // hostDebianArchitecture is the Debian name for the architecture these tests // run on. func hostDebianArchitecture(t *testing.T) string { diff --git a/engine/workspace_test.go b/engine/workspace_test.go index 25dd2c1..0a1797f 100644 --- a/engine/workspace_test.go +++ b/engine/workspace_test.go @@ -266,7 +266,7 @@ func TestSignedDebianPlanEmbedsResponsesAndApplyNeedsNoSigner(t *testing.T) { t.Fatalf("missing %s: %v", name, err) } } - if runner, err := exec.LookPath("podman"); err == nil && exec.Command(runner, "image", "exists", DefaultDebianVerificationImage).Run() == nil { + if runner, err := exec.LookPath("podman"); !testing.Short() && err == nil && exec.Command(runner, "image", "exists", DefaultDebianVerificationImage).Run() == nil { verified, err := VerifyDeb(context.Background(), VerifyDebRequest{Repository: release, Runner: runner, Image: DefaultDebianVerificationImage, MaxWorkspaceBytes: 4 << 30}) if err != nil || verified.InstalledCases == 0 { t.Fatalf("apt signed-by verification=%#v err=%v", verified, err) diff --git a/internal/app/deb_endpoint_test.go b/internal/app/deb_endpoint_test.go index 6ef5f9e..977bef0 100644 --- a/internal/app/deb_endpoint_test.go +++ b/internal/app/deb_endpoint_test.go @@ -174,6 +174,11 @@ func TestDebEndpointVerificationInstallsOverHTTP(t *testing.T) { // during the run, so a runner that has never seen it still exercises the path. func containerRunner(t *testing.T) string { t.Helper() + // -short is the developer's loop; running a real apt inside a container is + // the slowest and most machine-dependent thing here. CI runs the full suite. + if testing.Short() { + t.Skip("-short skips client verification in a container") + } for _, candidate := range []string{"podman", "docker"} { if _, err := exec.LookPath(candidate); err == nil { return candidate diff --git a/internal/app/platform_image_test.go b/internal/app/platform_image_test.go index effac39..8323c0f 100644 --- a/internal/app/platform_image_test.go +++ b/internal/app/platform_image_test.go @@ -12,6 +12,9 @@ import ( // The pinned reference is a multi-platform index; resolving the child digest is // what lets a workstation verify a repository built for another architecture. func TestPlatformImageResolvesForeignArchitecture(t *testing.T) { + if testing.Short() { + t.Skip("-short skips resolving an image index over the network") + } runner := "" for _, candidate := range []string{"docker", "podman"} { if _, err := exec.LookPath(candidate); err == nil { diff --git a/internal/app/pypi_endpoint_test.go b/internal/app/pypi_endpoint_test.go index 0fd69a4..32f1cef 100644 --- a/internal/app/pypi_endpoint_test.go +++ b/internal/app/pypi_endpoint_test.go @@ -17,6 +17,9 @@ import ( ) func TestVerifyPyPIClientEndpointInstallsFromSelectedHost(t *testing.T) { + if testing.Short() { + t.Skip("-short skips client verification with pip") + } if err := exec.Command("python3", "-m", "pip", "--version").Run(); err != nil { t.Skip("python3 with pip is unavailable") } @@ -57,6 +60,9 @@ func TestVerifyPyPIClientEndpointInstallsFromSelectedHost(t *testing.T) { } func TestVerifyPyPIClientEndpointUsesBasicCredentialWithoutLeakingIt(t *testing.T) { + if testing.Short() { + t.Skip("-short skips client verification with pip") + } if err := exec.Command("python3", "-m", "pip", "--version").Run(); err != nil { t.Skip("python3 with pip is unavailable") } diff --git a/internal/state/lockshard_test.go b/internal/state/lockshard_test.go index bff0504..59196e9 100644 --- a/internal/state/lockshard_test.go +++ b/internal/state/lockshard_test.go @@ -11,6 +11,24 @@ import ( func shardingRepository() Repository { return Repository{Lock: "repos/apt.lock.toml"} } +// skipInShortMode leaves out a test that is correct but slow enough to make the +// suite unusable as an inner loop. +// +// Sharding writes one file per package past LockShardThreshold, so each of these +// writes and reads a few thousand files. Six of them accounted for roughly 200 +// of the suite's 400 seconds, and a developer who cannot run the tests in the +// time it takes to read a diff stops running them — which is how two publishing +// bugs reached main behind paths only a full run touches. +// +// -short is the developer's loop, not CI's: `make check` and every CI job run +// the whole suite, so nothing here goes unverified, only less often. +func skipInShortMode(t *testing.T, what string) { + t.Helper() + if testing.Short() { + t.Skipf("-short skips %s", what) + } +} + func lockOf(versions int) RepositoryLock { lock := RepositoryLock{SchemaVersion: LockSchema, Repository: "apt"} for index := range versions { @@ -59,6 +77,7 @@ func TestASmallLockStaysOneFile(t *testing.T) { // Past the threshold the lock becomes a root plus one file per package, and reading // it back gives exactly what was written. func TestALargeLockShardsAndRoundTrips(t *testing.T) { + skipInShortMode(t, "lock sharding at scale") root := workspaceWithRepos(t) written := lockOf(LockShardThreshold + 500) if err := WriteLock(root, shardingRepository(), written); err != nil { @@ -103,6 +122,7 @@ func TestALargeLockShardsAndRoundTrips(t *testing.T) { // must rewrite one small file, not all of them — otherwise every publication is a // whole-repository diff and review does not scale. func TestAddingOneVersionRewritesOneFile(t *testing.T) { + skipInShortMode(t, "lock sharding at scale") root := workspaceWithRepos(t) lock := lockOf(LockShardThreshold + 500) if err := WriteLock(root, shardingRepository(), lock); err != nil { @@ -167,6 +187,7 @@ func shardModificationTimes(t *testing.T, directory string) map[string]time.Time // A lock is the record a publication is verified against, so a shard edited on its // own has to be an error rather than a quietly different repository. func TestAnEditedShardIsRefused(t *testing.T) { + skipInShortMode(t, "lock sharding at scale") root := workspaceWithRepos(t) if err := WriteLock(root, shardingRepository(), lockOf(LockShardThreshold+1)); err != nil { t.Fatal(err) @@ -200,6 +221,7 @@ func TestAnEditedShardIsRefused(t *testing.T) { // quietly serves less than it says, so the Merkle root covers which shards exist // and not only what each contains. func TestRemovingAShardFromTheIndexIsRefused(t *testing.T) { + skipInShortMode(t, "lock sharding at scale") root := workspaceWithRepos(t) if err := WriteLock(root, shardingRepository(), lockOf(LockShardThreshold+1)); err != nil { t.Fatal(err) @@ -231,6 +253,7 @@ func TestRemovingAShardFromTheIndexIsRefused(t *testing.T) { // A package dropped from the lock has its file removed, so the directory says the // same thing the root does. func TestARemovedPackageLosesItsShard(t *testing.T) { + skipInShortMode(t, "lock sharding at scale") root := workspaceWithRepos(t) lock := lockOf(LockShardThreshold + 500) if err := WriteLock(root, shardingRepository(), lock); err != nil { @@ -272,6 +295,7 @@ func TestPackagesDifferingOnlyInCaseGetTheirOwnShards(t *testing.T) { // A path in the index that leaves the lock directory would make loading a lock // read arbitrary files. func TestAShardPathCannotLeaveTheLockDirectory(t *testing.T) { + skipInShortMode(t, "lock sharding at scale") root := workspaceWithRepos(t) if err := WriteLock(root, shardingRepository(), lockOf(LockShardThreshold+1)); err != nil { t.Fatal(err) From 7d7bfb604ddc8d2857af38d2a4bd1baed8c6ee49 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 11:37:36 +0200 Subject: [PATCH 04/26] Read a bare 404 from a blob store as a missing blob --- adapters/blob/s3/aws.go | 8 ++++++++ adapters/blob/s3/aws_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/adapters/blob/s3/aws.go b/adapters/blob/s3/aws.go index 6fdefbc..020fc36 100644 --- a/adapters/blob/s3/aws.go +++ b/adapters/blob/s3/aws.go @@ -116,6 +116,14 @@ func normalizeAWSError(err error) error { var responseError *transport.ResponseError if errors.As(err, &responseError) { switch responseError.HTTPStatusCode() { + // A bare status with no error code in the body. HEAD has no body to carry + // one, and several S3-compatible stores answer a missing key that way, so + // without this a blob that is merely absent surfaced as an unrecognised + // failure — the caller could not tell "not uploaded yet" from "the store + // is broken", and an ordinary first upload read as an outage. The host + // adapter's copy of this function has always mapped it; the two drifted. + case 404: + return fmt.Errorf("%w: %v", blob.ErrNotFound, err) case 412: return fmt.Errorf("%w: %v", blob.ErrPrecondition, err) } diff --git a/adapters/blob/s3/aws_test.go b/adapters/blob/s3/aws_test.go index 492d88a..e6dbcdb 100644 --- a/adapters/blob/s3/aws_test.go +++ b/adapters/blob/s3/aws_test.go @@ -4,12 +4,22 @@ package s3blob import ( "errors" + "net/http" "testing" "github.com/aws/smithy-go" + transport "github.com/aws/smithy-go/transport/http" "github.com/shellcell/snailmail/blob" ) +// bareStatus is a response carrying a status and no error code, which is what a +// HEAD against an S3-compatible store returns: there is no body to put one in. +func bareStatus(code int) error { + return &transport.ResponseError{ + Response: &transport.Response{Response: &http.Response{StatusCode: code}}, + } +} + func TestNormalizeAWSErrorDoesNotTreatMissingBucketAsMissingBlob(t *testing.T) { err := normalizeAWSError(&smithy.GenericAPIError{Code: "NoSuchBucket", Message: "missing bucket"}) if errors.Is(err, blob.ErrNotFound) { @@ -17,6 +27,21 @@ func TestNormalizeAWSErrorDoesNotTreatMissingBucketAsMissingBlob(t *testing.T) { } } +// A HEAD has no body to carry an error code, so a missing key comes back as a +// bare 404. Several S3-compatible stores answer that way and the blob store has +// to read it as absence, or a first upload looks like an outage. +func TestNormalizeAWSErrorTreatsABareNotFoundStatusAsMissing(t *testing.T) { + if err := normalizeAWSError(bareStatus(http.StatusNotFound)); !errors.Is(err, blob.ErrNotFound) { + t.Fatalf("a bare 404 was not normalized as a missing blob: %v", err) + } +} + +func TestNormalizeAWSErrorTreatsABarePreconditionStatusAsPrecondition(t *testing.T) { + if err := normalizeAWSError(bareStatus(http.StatusPreconditionFailed)); !errors.Is(err, blob.ErrPrecondition) { + t.Fatalf("a bare 412 was not normalized as a failed precondition: %v", err) + } +} + func TestValidateAWSEndpointRequiresHTTPSOutsideLoopback(t *testing.T) { if err := validateAWSEndpoint("http://objects.example"); err == nil { t.Fatal("plaintext remote endpoint was accepted") From ced7b626ee1253c815ac40e21a612f2b065e6f46 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 11:51:30 +0200 Subject: [PATCH 05/26] s3 fixes Signed-off-by: Polina Simonenko --- adapters/host/githubpages/githubpages.go | 7 +++- adapters/host/s3/s3.go | 8 +++- cmd/snailmail/main.go | 6 ++- engine/preview_capability_test.go | 50 ++++++++++++++++++++++++ engine/workspace.go | 19 +++++++-- engine/workspace_test.go | 8 +++- host/host.go | 13 ++++++ internal/state/model.go | 9 +++-- 8 files changed, 109 insertions(+), 11 deletions(-) diff --git a/adapters/host/githubpages/githubpages.go b/adapters/host/githubpages/githubpages.go index 9e4f32e..19a0fee 100644 --- a/adapters/host/githubpages/githubpages.go +++ b/adapters/host/githubpages/githubpages.go @@ -58,7 +58,12 @@ func (adapter *Adapter) Capabilities(ctx context.Context, repository host.Reposi return host.Capabilities{}, err } } - return host.Capabilities{FaithfulPreview: true, ConditionalCommit: true, ConditionalRestore: true}, nil + return host.Capabilities{ + FaithfulPreview: true, ConditionalCommit: true, ConditionalRestore: true, + // The publication commit records the manifest digest beside the tree, so a + // revision changing only generated metadata is still distinguishable. + ReportsManifestDigest: true, + }, nil } func (adapter *Adapter) Observe(ctx context.Context, repository host.Repository) (host.PublishedRevision, error) { diff --git a/adapters/host/s3/s3.go b/adapters/host/s3/s3.go index 25f9407..10b9d5e 100644 --- a/adapters/host/s3/s3.go +++ b/adapters/host/s3/s3.go @@ -63,7 +63,13 @@ func (adapter *Adapter) Capabilities(_ context.Context, repository host.Reposito if _, err := singleRootPath(repository); err != nil { return host.Capabilities{}, err } - capabilities := host.Capabilities{FaithfulPreview: true, ConditionalCommit: true, ConditionalRestore: true, PrivateRead: adapter.broker != nil} + capabilities := host.Capabilities{ + FaithfulPreview: true, ConditionalCommit: true, ConditionalRestore: true, + // The root object carries tree, plan, change, release and manifest digests + // as object metadata, so Observe reads them back with the revision. + ReportsManifestDigest: true, + PrivateRead: adapter.broker != nil, + } if adapter.broker != nil { capabilities.CredentialBrokerIdentity = adapter.broker.Identity() } diff --git a/cmd/snailmail/main.go b/cmd/snailmail/main.go index f2f6fc2..3609c7f 100644 --- a/cmd/snailmail/main.go +++ b/cmd/snailmail/main.go @@ -214,8 +214,12 @@ func runSetup(args []string, stdout, stderr io.Writer) error { }); err != nil { return err } + // Report the URL people will install from where there is one, and the path + // otherwise. Keyed on having an endpoint rather than on a list of host names, + // which had left the rsync host — which has a --base-url like any other + // remote — reporting a far-side filesystem path nobody types into a browser. target := *output - if *hostType == "s3" || *hostType == "github-pages" { + if *canonicalEndpoint != "" { target = *canonicalEndpoint } if done, err := flags.emit(stdout, setupResult{Repository: *name, Format: format, Target: target}); done || err != nil { diff --git a/engine/preview_capability_test.go b/engine/preview_capability_test.go index 46d360c..795529e 100644 --- a/engine/preview_capability_test.go +++ b/engine/preview_capability_test.go @@ -164,6 +164,56 @@ func TestPlanAndApplyPreviewlessHostUnderAutoGate(t *testing.T) { } } +// A host that does not report a manifest digest must still settle. +// +// The engine used to decide this by asking whether the host was named "s3" or +// "github-pages". Comparing a desired manifest digest against the empty string +// such a host returns would plan an update on every run, so the question is +// real — but it is a question about what the host reports, and the answer now +// comes from the host rather than from a list of names the engine carries. +func TestAPreviewlessHostSettlesWithoutAManifestDigest(t *testing.T) { + root := previewlessWorkspace(t, "auto") + remote := &previewlessHost{} + resolver := staticHostResolver{host: remote} + createdAt := time.Date(2026, time.August, 22, 1, 2, 3, 0, time.UTC) + + planName := filepath.Join(root, "first.json") + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, + ExpiresIn: time.Hour, Hosts: resolver, + }); err != nil { + t.Fatal(err) + } + plan, err := state.LoadPlan(planName) + if err != nil { + t.Fatal(err) + } + if plan.Payload.Repositories[0].ReportsManifestDigest { + t.Fatal("plan claimed a manifest digest the host does not report") + } + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + Root: root, Plan: planName, now: createdAt.Add(time.Minute), Hosts: resolver, + }); err != nil { + t.Fatal(err) + } + + // GeneratedAt is held to what the first plan used, because the build-graph + // manifest embeds it: a later timestamp changes the manifest digest while the + // tree digest stays put, and the receipt would rightly report the generated + // metadata as changed. The CLI pins it to a fixed epoch for the same reason. + settled, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + Root: root, Output: filepath.Join(root, "second.json"), + createdAt: createdAt.Add(time.Hour), GeneratedAt: createdAt, + ExpiresIn: time.Hour, Hosts: resolver, + }) + if err != nil { + t.Fatal(err) + } + if settled.Changes != 0 { + t.Fatalf("replanning found %d changes; a host reporting no manifest digest never settles", settled.Changes) + } +} + // A gate that waits for a person to review a preview cannot be satisfied by a // host that serves none, and the refusal has to say which of the two to change. func TestPlanRefusesPreviewlessHostUnderHumanGate(t *testing.T) { diff --git a/engine/workspace.go b/engine/workspace.go index 213c787..bd77c96 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -634,7 +634,7 @@ func (preparation *planPreparation) planRepository(name string) (state.PlanRepos } publicationRecords := len(publicationBindings) != 0 action := "noop" - if observed.TreeSHA256 != desired.TreeSHA256 || ((hostRepository.Type == "s3" || hostRepository.Type == "github-pages") && observed.ManifestSHA256 != desired.ManifestSHA256) || publicationRecords || !deploymentMatchesDesired(deployment, observed, desired.TreeSHA256, desired.ManifestSHA256, desiredSigningState) { + if observed.TreeSHA256 != desired.TreeSHA256 || (capabilities.ReportsManifestDigest && observed.ManifestSHA256 != desired.ManifestSHA256) || publicationRecords || !deploymentMatchesDesired(deployment, observed, desired.TreeSHA256, desired.ManifestSHA256, desiredSigningState) { action = "update" if observed.NativeRevision == "" { action = "create" @@ -659,6 +659,7 @@ func (preparation *planPreparation) planRepository(name string) (state.PlanRepos Acquisitions: acquisitions, FaithfulPreview: capabilities.FaithfulPreview, ConditionalCommit: capabilities.ConditionalCommit, ConditionalRestore: capabilities.ConditionalRestore, + ReportsManifestDigest: capabilities.ReportsManifestDigest, PrivateRead: capabilities.PrivateRead, CredentialBrokerIdentity: capabilities.CredentialBrokerIdentity, InstallDocSHA256: installDocDigest, @@ -1866,10 +1867,16 @@ func validateApplyPlan(plan state.Plan) error { if repository.CanonicalEndpoint == "" { return fmt.Errorf("plan repository %q has no canonical endpoint", repository.Name) } - if (repository.Host.Type == "s3" || repository.Host.Type == "github-pages") && !hexdigest.ValidSHA256(repository.InstallDocSHA256) { + // The same predicate the digest is computed under, rather than a second + // list of host names beside it. These had drifted: rsync generates an + // install document and this did not check it. + if host.Supports(repository.Host.Type, repository.Format).InstallDocument && !hexdigest.ValidSHA256(repository.InstallDocSHA256) { return fmt.Errorf("plan repository %q has an invalid install document digest", repository.Name) } - if (repository.Host.Type == "s3" || repository.Host.Type == "github-pages") && !hexdigest.ValidSHA256(repository.DesiredManifestSHA256) { + // Required of a host that reports one. A forged plan claiming otherwise + // gains nothing: prepareRepository re-reads the live host's capabilities + // and refuses a plan whose claims have drifted from them. + if repository.ReportsManifestDigest && !hexdigest.ValidSHA256(repository.DesiredManifestSHA256) { return fmt.Errorf("plan repository %q has an invalid desired manifest digest", repository.Name) } if repository.Visibility == "private" && !repository.PrivateRead { @@ -2524,6 +2531,7 @@ func (preparation *applyPreparation) prepareRepository(planned state.PlanReposit return applyRepository{}, err } if capabilities.FaithfulPreview != planned.FaithfulPreview || capabilities.ConditionalCommit != planned.ConditionalCommit || capabilities.ConditionalRestore != planned.ConditionalRestore || + capabilities.ReportsManifestDigest != planned.ReportsManifestDigest || capabilities.PrivateRead != planned.PrivateRead || capabilities.CredentialBrokerIdentity != planned.CredentialBrokerIdentity { return applyRepository{}, fmt.Errorf("stale plan: repository %q host capabilities changed", planned.Name) } @@ -2576,7 +2584,10 @@ func (preparation *applyPreparation) prepareRepository(planned state.PlanReposit return applyRepository{}, fmt.Errorf("repository %q: %w", planned.Name, err) } matchesObserved := revisionMatchesPlanObservation(observed, planned) - managedRemote := repository.Host.Type == "s3" || repository.Host.Type == "github-pages" + // Declared by the host rather than derived from its name. The plan carries + // what the host reported when it was made, and the drift check above has + // already established that the live host still reports the same. + managedRemote := planned.ReportsManifestDigest matchesApplied := planned.Action != "noop" && observed.TreeSHA256 == planned.DesiredTreeSHA256 && (!managedRemote || (observed.PlanID == preparation.plan.PlanID && observed.ChangeID == planned.ChangeID && observed.ManifestSHA256 == planned.DesiredManifestSHA256)) deploymentApplied := deployment.PlanID == preparation.plan.PlanID && deployment.ChangeID == planned.ChangeID && deployment.TreeSHA256 == planned.DesiredTreeSHA256 && deployment.ManifestSHA256 == planned.DesiredManifestSHA256 && deployment.NativeRevision == observed.NativeRevision && deploymentSigningMatches(deployment, desiredSigningState) diff --git a/engine/workspace_test.go b/engine/workspace_test.go index 0a1797f..a045dcd 100644 --- a/engine/workspace_test.go +++ b/engine/workspace_test.go @@ -1938,8 +1938,14 @@ type recordingHost struct { commitCalls int } +// Stands in for a managed remote — S3 or Pages — so it reports what those +// report, including the manifest digest that lets a revision changing only +// generated metadata be told apart from one that changed nothing. func (remote *recordingHost) Capabilities(context.Context, host.Repository) (host.Capabilities, error) { - return host.Capabilities{FaithfulPreview: true, ConditionalCommit: true, ConditionalRestore: true}, nil + return host.Capabilities{ + FaithfulPreview: true, ConditionalCommit: true, ConditionalRestore: true, + ReportsManifestDigest: true, + }, nil } func (remote *recordingHost) Observe(context.Context, host.Repository) (host.PublishedRevision, error) { diff --git a/host/host.go b/host/host.go index 0220f47..6761214 100644 --- a/host/host.go +++ b/host/host.go @@ -105,6 +105,19 @@ type Capabilities struct { // back, conditional on the failed one still being live. A host without it is // never given an automatic-rollback promise. ConditionalRestore bool + // ReportsManifestDigest reports whether Observe returns the build-graph + // manifest digest of the live revision alongside its tree digest. + // + // Where it does, two revisions with the same tree but a different manifest + // are distinguishable, so a plan can tell that a repository needs + // republishing when only generated metadata changed. Where it does not, + // the tree digest is the whole of what the host can be asked, and comparing + // a manifest digest against the empty string it returns would plan an + // update on every run. + // + // A host that stores per-object metadata answers this; one that publishes a + // bare directory tree does not. + ReportsManifestDigest bool // PrivateRead reports whether the host can issue scoped read credentials for // a private repository. PrivateRead bool diff --git a/internal/state/model.go b/internal/state/model.go index 1bbe627..f8b0a1b 100644 --- a/internal/state/model.go +++ b/internal/state/model.go @@ -3,9 +3,11 @@ package state import "time" const ( - ManifestSchema = 7 - LockSchema = 2 - PlanSchema = 10 + ManifestSchema = 7 + LockSchema = 2 + // 11 added PlanRepository.ReportsManifestDigest, which the engine had been + // deriving by comparing the host type against a list of names. + PlanSchema = 11 LedgerSchema = 1 DeploymentSchema = 2 ) @@ -310,6 +312,7 @@ type PlanRepository struct { FaithfulPreview bool `json:"faithful_preview"` ConditionalCommit bool `json:"conditional_commit"` ConditionalRestore bool `json:"conditional_restore"` + ReportsManifestDigest bool `json:"reports_manifest_digest"` PrivateRead bool `json:"private_read"` CredentialBrokerIdentity string `json:"credential_broker_identity,omitempty"` Signing []PlanSigning `json:"signing,omitempty"` From 7a8dabc82d495bd43732bb0af0d7fbf258c231ab Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 12:50:36 +0200 Subject: [PATCH 06/26] Show setup the flags this repository actually uses --- cmd/snailmail/main.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/cmd/snailmail/main.go b/cmd/snailmail/main.go index 3609c7f..0a86aca 100644 --- a/cmd/snailmail/main.go +++ b/cmd/snailmail/main.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "flag" "fmt" "io" "net" @@ -32,7 +33,14 @@ const defaultGeneratedAt = "1970-01-01T00:00:00Z" func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - if err := run(ctx, os.Args[1:], os.Stdout, os.Stderr); err != nil { + err := run(ctx, os.Args[1:], os.Stdout, os.Stderr) + // Asking for help is not a failure. The flag package reports it as one, and + // every `--help` ended with "snailmail: flag: help requested" printed under + // the usage the reader had just asked for. + if errors.Is(err, flag.ErrHelp) { + return + } + if err != nil { fmt.Fprintf(os.Stderr, "snailmail: %v\n", err) os.Exit(1) } @@ -169,7 +177,11 @@ func runSetup(args []string, stdout, stderr io.Writer) error { gatePolicy := flags.String("gate", "auto", "publication gate: auto, pr, or approval") approvalKeys := flags.String("approval-keys", "", "comma-separated allowed Ed25519 public keys") signingKey := flags.String("signing-key", "", "repository signing key name") - allowUnsigned := flags.Bool("allow-unsigned", false, "explicitly allow a new unsigned Debian repository") + // Named for the format being configured. It said "Debian" while applying to + // every format snailmail signs, so an rpm repository was refused for want of + // a flag whose help said it was for something else. + allowUnsigned := flags.Bool("allow-unsigned", false, + fmt.Sprintf("explicitly allow a new unsigned %s repository", format)) track := flags.String("track", "stable", "rendered placement track") keep := flags.Int("keep", 0, "publications to retain when collecting; 0 uses the default and can be changed later in snailmail.toml") @@ -196,9 +208,17 @@ func runSetup(args []string, stdout, stderr io.Writer) error { defaultArchitectures = "x86_64" } architectures := flags.String("architectures", defaultArchitectures, "comma-separated architectures the repository serves") + // The host is read before parsing only to decide which flags to describe; + // *hostType below is what actually configures anything. + flags.set.Usage = func() { + writeSetupUsage(stderr, flags.set, format, hostFromArguments(args[1:])) + } if err := flags.parse(args[1:]); err != nil { return err } + if err := rejectInapplicableSetupFlags(flags.set, format, *hostType); err != nil { + return err + } resolvedApprovalKeys := splitList(*approvalKeys) sort.Strings(resolvedApprovalKeys) if err := engine.SetupRepository(engine.SetupRepositoryRequest{ From aef63511369609caa8e69a74b0eaca22b6df24d0 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 12:51:45 +0200 Subject: [PATCH 07/26] Stop .gitignore hiding the cmd/snailmail directory Signed-off-by: Polina Simonenko --- .gitignore | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index f2c19d0..a420188 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ -dist -snailmail +# Anchored to the repository root. Without the leading slash, "snailmail" +# matched any path component of that name — which includes the cmd/snailmail +# directory, so every new file added to the CLI was silently invisible to git. +# Already-tracked files kept working, which is what hid it. +/snailmail /build/ /dist/ *.test - From 87b99e39eeb0756f41fb0e753bfbcf31bf89035d Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 12:51:45 +0200 Subject: [PATCH 08/26] Add the setup flag scope table that 1c15786 was missing Signed-off-by: Polina Simonenko --- cmd/snailmail/setupflags.go | 187 +++++++++++++++++++++++++++++++ cmd/snailmail/setupflags_test.go | 151 +++++++++++++++++++++++++ 2 files changed, 338 insertions(+) create mode 100644 cmd/snailmail/setupflags.go create mode 100644 cmd/snailmail/setupflags_test.go diff --git a/cmd/snailmail/setupflags.go b/cmd/snailmail/setupflags.go new file mode 100644 index 0000000..1210671 --- /dev/null +++ b/cmd/snailmail/setupflags.go @@ -0,0 +1,187 @@ +package main + +import ( + "flag" + "fmt" + "io" + "sort" + "strings" + + "github.com/shellcell/snailmail/formats" +) + +// setup declares thirty flags because it configures six formats across four +// hosts. Any one repository uses a handful. +// +// Go's flag package prints all of them, alphabetically, so `setup pypi --help` +// opened with an Alpine architecture list and a GitHub Pages preview branch and +// buried --name in the middle. Every flag stays registered — parsing has to +// recognise --bucket to say anything useful about it — but the help describes +// only what the chosen format and host use, and a flag belonging to some other +// combination is refused by name instead of being silently ignored or rejected +// much later by a manifest validator talking about "S3-only configuration". +type flagGroup struct { + title string + flags []string +} + +// setupFlagGroups is the single source of truth for which flags apply: the help +// renders it and the post-parse check reads it, so the two cannot disagree +// about what belongs to a pypi repository on an rsync host. +func setupFlagGroups(format, hostType string) []flagGroup { + groups := []flagGroup{{ + title: "Repository", + flags: []string{"name", "output", "track", "keep", "visibility"}, + }} + + if selected, err := formats.For(format); err == nil && selected.ImplementsSigning() { + groups = append(groups, flagGroup{ + title: "Signing", + flags: []string{"signing-key", "allow-unsigned"}, + }) + } + + switch format { + case "deb": + groups = append(groups, flagGroup{ + title: "Debian layout", + flags: []string{"suite", "component", "architectures"}, + }) + case "apk": + groups = append(groups, flagGroup{ + title: "Alpine layout", + flags: []string{"architectures"}, + }) + } + + hostFlags := []string{"host", "base-url"} + switch hostType { + case "s3": + hostFlags = append(hostFlags, "bucket", "prefix", "region", "endpoint", + "use-path-style", "read-auth", "credential-broker") + case "rsync": + hostFlags = append(hostFlags, "target") + case "github-pages": + hostFlags = append(hostFlags, "github-repo", "branch", + "github-preview-repo", "preview-branch", "preview-url") + } + groups = append(groups, + flagGroup{title: "Host (" + hostType + ")", flags: hostFlags}, + flagGroup{title: "Review gate", flags: []string{"gate", "approval-keys"}}, + flagGroup{title: "Output", flags: []string{"json", "workspace"}}, + ) + return groups +} + +// setupFlagOwner says what a flag belongs to, for the message a person gets +// when they reach for one that does not apply here. Naming the owner is the +// whole point: "--bucket is an s3 host flag" answers the question, where +// "flag provided but not defined" invites them to think it does not exist. +var setupFlagOwner = map[string]string{ + "suite": "a Debian repository", + "component": "a Debian repository", + "architectures": "a Debian or Alpine repository", + "signing-key": "a format whose repositories snailmail signs", + "allow-unsigned": "a format whose repositories snailmail signs", + "target": "the rsync host", + "bucket": "the s3 host", + "prefix": "the s3 host", + "region": "the s3 host", + "endpoint": "the s3 host", + "use-path-style": "the s3 host", + "read-auth": "the s3 host", + "credential-broker": "the s3 host", + "github-repo": "the github-pages host", + "branch": "the github-pages host", + "github-preview-repo": "the github-pages host", + "preview-branch": "the github-pages host", + "preview-url": "the github-pages host", +} + +// rejectInapplicableSetupFlags refuses a flag that was given but belongs to +// another format or host. +// +// The manifest validator already refused most of these, but only after the +// whole command had run and in terms of the stored configuration — "local host +// has S3-only configuration" rather than which flag to drop. A few it did not +// refuse at all: --suite on a pypi repository was accepted and discarded. +func rejectInapplicableSetupFlags(set *flag.FlagSet, format, hostType string) error { + applicable := make(map[string]bool) + for _, group := range setupFlagGroups(format, hostType) { + for _, name := range group.flags { + applicable[name] = true + } + } + // --root is the documented alias of --workspace and shares its destination. + applicable["root"] = true + var refused []string + set.Visit(func(given *flag.Flag) { + if !applicable[given.Name] { + refused = append(refused, given.Name) + } + }) + if len(refused) == 0 { + return nil + } + sort.Strings(refused) + described := make([]string, 0, len(refused)) + for _, name := range refused { + owner, known := setupFlagOwner[name] + if !known { + owner = "another configuration" + } + described = append(described, fmt.Sprintf("--%s belongs to %s", name, owner)) + } + return fmt.Errorf("%s repository on the %s host: %s", + format, hostType, strings.Join(described, "; ")) +} + +// writeSetupUsage prints the flags this format and host actually use, grouped in +// the order someone fills them in rather than alphabetically. +func writeSetupUsage(output io.Writer, set *flag.FlagSet, format, hostType string) { + fmt.Fprintf(output, "Usage: snailmail setup %s --name NAME [flags]\n", format) + for _, group := range setupFlagGroups(format, hostType) { + shown := make([]*flag.Flag, 0, len(group.flags)) + for _, name := range group.flags { + if declared := set.Lookup(name); declared != nil { + shown = append(shown, declared) + } + } + if len(shown) == 0 { + continue + } + fmt.Fprintf(output, "\n%s:\n", group.title) + for _, declared := range shown { + fmt.Fprintf(output, " --%-21s %s", declared.Name, declared.Usage) + // A false or zero default says nothing the usage text has not; both + // read as clutter beside the flags whose default is a real choice. + if declared.DefValue != "" && declared.DefValue != "false" && declared.DefValue != "0" { + fmt.Fprintf(output, " (default %s)", declared.DefValue) + } + fmt.Fprintln(output) + } + } + fmt.Fprintf(output, "\nOther hosts take other flags; --host TYPE then --help shows them.\n") +} + +// hostFromArguments reads --host before the flag set is parsed, so the usage +// text can describe the host being configured. Parsing proper still decides the +// value; this only chooses which flags to write about. +func hostFromArguments(arguments []string) string { + for index, argument := range arguments { + if argument == "--" { + break + } + if !strings.HasPrefix(argument, "-") { + continue + } + name := strings.TrimLeft(argument, "-") + if value, found := strings.CutPrefix(name, "host="); found { + return value + } + if name == "host" && index+1 < len(arguments) { + return arguments[index+1] + } + } + return "local" +} diff --git a/cmd/snailmail/setupflags_test.go b/cmd/snailmail/setupflags_test.go new file mode 100644 index 0000000..e999998 --- /dev/null +++ b/cmd/snailmail/setupflags_test.go @@ -0,0 +1,151 @@ +package main + +import ( + "bytes" + "context" + "sort" + "strings" + "testing" +) + +// The flags each format and host actually uses, pinned. +// +// setup declares thirty and any repository wants a handful, so the help used to +// open with an Alpine architecture list whatever you were configuring. Pinning +// the sets is what stops the next host widening every format's help again: a +// flag added to the s3 group has to be added here to appear anywhere. +func TestSetupFlagsAreScopedToFormatAndHost(t *testing.T) { + common := []string{ + "name", "output", "track", "keep", "visibility", + "host", "base-url", "gate", "approval-keys", "json", "workspace", + } + signing := []string{"signing-key", "allow-unsigned"} + + for _, testcase := range []struct { + format string + host string + extra []string + }{ + {"pypi", "local", nil}, + {"raw", "local", nil}, + {"helm", "local", signing}, + {"deb", "local", append(append([]string{}, signing...), "suite", "component", "architectures")}, + {"apk", "local", append(append([]string{}, signing...), "architectures")}, + {"rpm", "rsync", append(append([]string{}, signing...), "target")}, + {"pypi", "s3", []string{ + "bucket", "prefix", "region", "endpoint", + "use-path-style", "read-auth", "credential-broker", + }}, + {"deb", "github-pages", append(append([]string{}, signing...), + "suite", "component", "architectures", + "github-repo", "branch", "github-preview-repo", "preview-branch", "preview-url")}, + } { + t.Run(testcase.format+"/"+testcase.host, func(t *testing.T) { + var shown []string + for _, group := range setupFlagGroups(testcase.format, testcase.host) { + shown = append(shown, group.flags...) + } + want := append(append([]string{}, common...), testcase.extra...) + sort.Strings(shown) + sort.Strings(want) + if strings.Join(shown, ",") != strings.Join(want, ",") { + t.Fatalf("flags for %s on %s:\n got %v\nwant %v", testcase.format, testcase.host, shown, want) + } + }) + } +} + +// Every flag a group names has to exist, or the help silently omits it and the +// rejection silently allows it. +func TestEverySetupFlagInAGroupIsDeclared(t *testing.T) { + for _, format := range []string{"pypi", "deb", "helm", "raw", "rpm", "apk"} { + declared := declaredSetupFlags(t, format) + for _, hostType := range []string{"local", "s3", "rsync", "github-pages"} { + for _, group := range setupFlagGroups(format, hostType) { + for _, name := range group.flags { + if declared[name] { + continue + } + t.Errorf("%s/%s names --%s, which setup does not declare", format, hostType, name) + } + } + } + } +} + +// declaredSetupFlags reports the flags setup declares, by running it with +// --help for every host and reading back what it wrote. +func declaredSetupFlags(t *testing.T, format string) map[string]bool { + t.Helper() + var stdout, stderr bytes.Buffer + // Every host in turn, because the usage shows one host's flags at a time. + declared := make(map[string]bool) + for _, hostType := range []string{"local", "s3", "rsync", "github-pages"} { + stdout.Reset() + stderr.Reset() + _ = run(context.Background(), []string{"setup", format, "--host", hostType, "--help"}, &stdout, &stderr) + for _, line := range strings.Split(stderr.String(), "\n") { + trimmed := strings.TrimSpace(line) + if !strings.HasPrefix(trimmed, "--") { + continue + } + name := strings.TrimPrefix(strings.Fields(trimmed)[0], "--") + declared[name] = true + } + } + return declared +} + +// A flag from another format or host is refused by name. It used to be either +// discarded without a word — --suite on a PyPI repository — or refused much +// later by the manifest validator, in terms of the configuration it had built +// rather than the flag to drop. +func TestSetupRefusesAFlagFromAnotherFormatOrHost(t *testing.T) { + for _, testcase := range []struct { + name string + arguments []string + wants []string + }{ + {"debian layout on pypi", []string{"setup", "pypi", "--name", "p", "--output", "public/p", "--suite", "trixie"}, + []string{"--suite", "Debian"}}, + {"s3 bucket on a local host", []string{"setup", "pypi", "--name", "p", "--output", "public/p", "--bucket", "mine"}, + []string{"--bucket", "s3"}}, + {"pages branch on an rsync host", []string{"setup", "raw", "--name", "r", "--host", "rsync", "--target", "h", "--output", "/srv/r", "--branch", "gh-pages"}, + []string{"--branch", "github-pages"}}, + {"signing on a format that is not signed", []string{"setup", "pypi", "--name", "p", "--output", "public/p", "--allow-unsigned"}, + []string{"--allow-unsigned"}}, + } { + t.Run(testcase.name, func(t *testing.T) { + var stdout, stderr bytes.Buffer + err := run(context.Background(), testcase.arguments, &stdout, &stderr) + if err == nil { + t.Fatal("the flag was accepted") + } + for _, want := range testcase.wants { + if !strings.Contains(err.Error(), want) { + t.Fatalf("error %q does not mention %q", err, want) + } + } + }) + } +} + +func TestHostFromArgumentsReadsTheHostBeforeParsing(t *testing.T) { + for _, testcase := range []struct { + arguments []string + want string + }{ + {[]string{"--name", "p"}, "local"}, + {[]string{"--host", "s3"}, "s3"}, + {[]string{"--host=rsync"}, "rsync"}, + {[]string{"-host", "github-pages"}, "github-pages"}, + {[]string{"--name", "p", "--host", "s3", "--bucket", "b"}, "s3"}, + // After the terminator nothing is a flag, so a file called "--host" does + // not choose one. + {[]string{"--", "--host", "s3"}, "local"}, + } { + if got := hostFromArguments(testcase.arguments); got != testcase.want { + t.Errorf("hostFromArguments(%v) = %q, want %q", testcase.arguments, got, testcase.want) + } + } +} From 26ec65998564eaec162180fcc7a3e3484b08e67d Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 12:57:12 +0200 Subject: [PATCH 09/26] Exit with what to do about the failure Signed-off-by: Polina Simonenko --- README.md | 21 ++++++++++ cmd/snailmail/exitcode_test.go | 63 ++++++++++++++++++++++++++++++ cmd/snailmail/main.go | 71 +++++++++++++++++++++++++++++++++- 3 files changed, 154 insertions(+), 1 deletion(-) create mode 100644 cmd/snailmail/exitcode_test.go diff --git a/README.md b/README.md index 834753f..5836d93 100644 --- a/README.md +++ b/README.md @@ -841,6 +841,27 @@ That drops the stripped binary from about 20.6 MB to 12.8 MB. S3 hosts and S3 blob stores then report that the build has no S3 support; every other format, host, and command is unaffected. The default build keeps S3. +## Exit codes + +Every command exits `0` on success. A failure exits with what to do about it, +so a CI job can retry a flaky network without also retrying a malformed bucket +name: + +| Code | Meaning | What to do | +|---|---|---| +| `1` | Failed | Read the message. | +| `2` | The configuration is wrong | Fix the workspace or the host configuration; retrying fails again. | +| `3` | The host or network failed | Retry. | +| `4` | The plan no longer matches the world | Run `plan` again, review the diff, then `apply`. | +| `5` | A publication may or may not have taken effect | **Do not retry blindly.** Check the host, then `snailmail status`. | + +Codes `3`, `4` and `5` also print a line saying the same thing, for whoever is +reading the log rather than branching on the number. + +Only failures reported by a host carry this detail; anything else exits `1`. +Codes are stable — a job that branches on them should not break under an +upgrade. + ## Container image and CI examples `Dockerfile` builds the runtime image. `examples/github-actions.yml` is a pinned diff --git a/cmd/snailmail/exitcode_test.go b/cmd/snailmail/exitcode_test.go new file mode 100644 index 0000000..ff98ec0 --- /dev/null +++ b/cmd/snailmail/exitcode_test.go @@ -0,0 +1,63 @@ +package main + +import ( + "errors" + "fmt" + "testing" + + "github.com/shellcell/snailmail/host" +) + +// The exit code is an interface. A CI job branches on it to decide whether to +// retry, so the mapping is pinned rather than left to whatever the switch +// happens to say. +func TestExitCodeForReportsWhatToDoAboutTheFailure(t *testing.T) { + for _, testcase := range []struct { + name string + err error + want int + }{ + {"a plain error says nothing more than that it failed", + errors.New("something went wrong"), exitFailure}, + {"nil is not an error", nil, exitFailure}, + {"a bad bucket name is the operator's to fix", + &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host"}, exitConfiguration}, + {"a timeout is worth retrying", + &host.Error{Kind: host.ErrorInfrastructure, Operation: "observe S3 root", Retryable: true}, exitInfrastructure}, + {"a plan overtaken by events needs planning again", + &host.Error{Kind: host.ErrorStale, Operation: "commit rsync repository"}, exitStale}, + {"an unknown outcome must not be retried blindly", + &host.Error{Kind: host.ErrorIndeterminate, Operation: "commit S3 root metadata", EffectMayHaveOccurred: true}, exitIndeterminate}, + // The effect outranks the kind: whatever it was reported as, the next + // move is to look at the host. + {"an effect that may have happened outranks its kind", + &host.Error{Kind: host.ErrorInfrastructure, Operation: "send rsync tree", EffectMayHaveOccurred: true}, exitIndeterminate}, + {"a kind this build does not know is just a failure", + &host.Error{Kind: host.ErrorKind("something-new"), Operation: "do a new thing"}, exitFailure}, + // Errors are wrapped on the way up through the engine, so the kind has to + // survive being buried. + {"a wrapped host error keeps its kind", + fmt.Errorf("repository %q: %w", "python", + &host.Error{Kind: host.ErrorStale, Operation: "commit S3 root"}), exitStale}, + } { + t.Run(testcase.name, func(t *testing.T) { + if got := exitCodeFor(testcase.err); got != testcase.want { + t.Fatalf("exitCodeFor(%v) = %d, want %d", testcase.err, got, testcase.want) + } + }) + } +} + +// Every code that is not a plain failure tells the reader what to do about it, +// because an undocumented non-zero exit is indistinguishable from a crash. +func TestEveryDistinctExitCodeExplainsItself(t *testing.T) { + for _, code := range []int{exitInfrastructure, exitStale, exitIndeterminate} { + if exitNote(code) == "" { + t.Errorf("exit %d has no note", code) + } + } + // A configuration error explains itself: the message names what is wrong. + if exitNote(exitConfiguration) != "" || exitNote(exitFailure) != "" { + t.Error("a code whose error already says enough should not add a note") + } +} diff --git a/cmd/snailmail/main.go b/cmd/snailmail/main.go index 0a86aca..b94c555 100644 --- a/cmd/snailmail/main.go +++ b/cmd/snailmail/main.go @@ -21,6 +21,7 @@ import ( httpsource "github.com/shellcell/snailmail/adapters/source/http" "github.com/shellcell/snailmail/engine" "github.com/shellcell/snailmail/gate" + "github.com/shellcell/snailmail/host" "github.com/shellcell/snailmail/internal/state" "github.com/shellcell/snailmail/internal/version" "github.com/shellcell/snailmail/internal/wire" @@ -42,10 +43,78 @@ func main() { } if err != nil { fmt.Fprintf(os.Stderr, "snailmail: %v\n", err) - os.Exit(1) + code := exitCodeFor(err) + if note := exitNote(code); note != "" { + fmt.Fprintf(os.Stderr, "snailmail: %s (exit %d)\n", note, code) + } + os.Exit(code) } } +// Exit codes. +// +// A host adapter already says whether a failure was the operator's +// configuration, a flaky network, a plan overtaken by events, or a publication +// whose outcome is unknown — 129 sites populate host.Error.Kind. Nothing read +// it, and every failure exited 1, so a CI job could not retry a timeout without +// also retrying a malformed bucket name. +// +// Codes are stable: they are an interface, and a job that branches on them +// breaks if they move. +const ( + // exitFailure is anything without a more specific answer. + exitFailure = 1 + // exitConfiguration means the workspace or its host configuration is wrong. + // Retrying runs it again and fails again. + exitConfiguration = 2 + // exitInfrastructure means a host or the network failed. Retrying is + // reasonable and is the point of telling this apart from the rest. + exitInfrastructure = 3 + // exitStale means the plan no longer describes the world. Plan again. + exitStale = 4 + // exitIndeterminate means a publication may or may not have taken effect. + // This is the one never to retry blindly. + exitIndeterminate = 5 +) + +func exitCodeFor(err error) int { + var hostError *host.Error + if !errors.As(err, &hostError) { + return exitFailure + } + // An effect that may have happened outranks the kind it was reported under. + // Whatever went wrong on the way there, the operator's next move is to look + // at the host before doing anything else. + if hostError.EffectMayHaveOccurred { + return exitIndeterminate + } + switch hostError.Kind { + case host.ErrorInvalidConfiguration: + return exitConfiguration + case host.ErrorInfrastructure: + return exitInfrastructure + case host.ErrorStale: + return exitStale + case host.ErrorIndeterminate: + return exitIndeterminate + } + return exitFailure +} + +// exitNote says what to do about a code, for the reader who is not going to +// look it up. Only where the answer is not obvious from the error itself. +func exitNote(code int) string { + switch code { + case exitInfrastructure: + return "the host or network failed rather than the request; retrying is reasonable" + case exitStale: + return "the plan no longer matches the world; run snailmail plan again" + case exitIndeterminate: + return "a publication may have taken effect; check the host before retrying" + } + return "" +} + func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { if len(args) == 0 { printUsage(stdout) From d28b4b3e34ab86c09bef2c01239dbbf83009e26b Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 13:25:41 +0200 Subject: [PATCH 10/26] Decide what an S3 error means in one place --- adapters/blob/s3/aws.go | 59 +++++------------- adapters/host/s3/aws.go | 53 +++++----------- adapters/host/s3/aws_test.go | 25 ++++++++ internal/awss3/awss3.go | 113 +++++++++++++++++++++++++++++++++++ internal/awss3/awss3_test.go | 96 +++++++++++++++++++++++++++++ 5 files changed, 266 insertions(+), 80 deletions(-) create mode 100644 internal/awss3/awss3.go create mode 100644 internal/awss3/awss3_test.go diff --git a/adapters/blob/s3/aws.go b/adapters/blob/s3/aws.go index 020fc36..31f951d 100644 --- a/adapters/blob/s3/aws.go +++ b/adapters/blob/s3/aws.go @@ -12,13 +12,11 @@ import ( "net/url" "github.com/aws/aws-sdk-go-v2/aws" - awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/smithy-go" - transport "github.com/aws/smithy-go/transport/http" "github.com/shellcell/snailmail/blob" + "github.com/shellcell/snailmail/internal/awss3" "github.com/shellcell/snailmail/internal/hexdigest" ) @@ -34,20 +32,13 @@ func NewAWS(ctx context.Context, configuration blob.Configuration) (*Store, erro if err := validateAWSEndpoint(configuration.Endpoint); err != nil { return nil, err } - options := []func(*awsconfig.LoadOptions) error{} - if configuration.Region != "" { - options = append(options, awsconfig.WithRegion(configuration.Region)) - } - loaded, err := awsconfig.LoadDefaultConfig(ctx, options...) + client, err := awss3.NewClient(ctx, awss3.Config{ + Bucket: configuration.Bucket, Region: configuration.Region, + Endpoint: configuration.Endpoint, UsePathStyle: configuration.UsePathStyle, + }) if err != nil { - return nil, fmt.Errorf("load AWS configuration: %w", err) + return nil, err } - client := s3.NewFromConfig(loaded, func(options *s3.Options) { - options.UsePathStyle = configuration.UsePathStyle - if configuration.Endpoint != "" { - options.BaseEndpoint = aws.String(configuration.Endpoint) - } - }) return New(&AWSClient{client: client, bucket: configuration.Bucket}, configuration) } @@ -98,35 +89,17 @@ func (client *AWSClient) Get(ctx context.Context, key string) (io.ReadCloser, Ob return result.Body, ObjectInfo{Size: aws.ToInt64(result.ContentLength), SHA256: hexdigest.FromBase64(aws.ToString(result.ChecksumSHA256)), Metadata: result.Metadata}, nil } +// normalizeAWSError translates a store's answer into this package's vocabulary. +// What the answer means is decided once, in awss3; what it is called is decided +// here, because blob.ErrNotFound is about an artifact rather than about an +// object and the host adapter's identically-named sentinel is not the same +// thing. func normalizeAWSError(err error) error { - if err == nil { - return nil - } - var apiError smithy.APIError - if errors.As(err, &apiError) { - switch apiError.ErrorCode() { - case "NoSuchKey", "NotFound": - return fmt.Errorf("%w: %v", blob.ErrNotFound, err) - case "NoSuchBucket": - return err - case "PreconditionFailed": - return fmt.Errorf("%w: %v", blob.ErrPrecondition, err) - } - } - var responseError *transport.ResponseError - if errors.As(err, &responseError) { - switch responseError.HTTPStatusCode() { - // A bare status with no error code in the body. HEAD has no body to carry - // one, and several S3-compatible stores answer a missing key that way, so - // without this a blob that is merely absent surfaced as an unrecognised - // failure — the caller could not tell "not uploaded yet" from "the store - // is broken", and an ordinary first upload read as an outage. The host - // adapter's copy of this function has always mapped it; the two drifted. - case 404: - return fmt.Errorf("%w: %v", blob.ErrNotFound, err) - case 412: - return fmt.Errorf("%w: %v", blob.ErrPrecondition, err) - } + switch awss3.Classify(err) { + case awss3.NotFound: + return fmt.Errorf("%w: %v", blob.ErrNotFound, err) + case awss3.Precondition: + return fmt.Errorf("%w: %v", blob.ErrPrecondition, err) } return err } diff --git a/adapters/host/s3/aws.go b/adapters/host/s3/aws.go index bab8c3f..cb6f50c 100644 --- a/adapters/host/s3/aws.go +++ b/adapters/host/s3/aws.go @@ -14,13 +14,11 @@ import ( "strings" "github.com/aws/aws-sdk-go-v2/aws" - awsconfig "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/smithy-go" - transport "github.com/aws/smithy-go/transport/http" "github.com/shellcell/snailmail/host" + "github.com/shellcell/snailmail/internal/awss3" "github.com/shellcell/snailmail/internal/hexdigest" ) @@ -33,20 +31,13 @@ func NewAWS(ctx context.Context, repository host.Repository, brokers ...host.Cre if err := validateRepository(repository); err != nil { return nil, err } - options := []func(*awsconfig.LoadOptions) error{} - if repository.Region != "" { - options = append(options, awsconfig.WithRegion(repository.Region)) - } - configuration, err := awsconfig.LoadDefaultConfig(ctx, options...) + client, err := awss3.NewClient(ctx, awss3.Config{ + Bucket: repository.Bucket, Region: repository.Region, + Endpoint: repository.Endpoint, UsePathStyle: repository.UsePathStyle, + }) if err != nil { - return nil, fmt.Errorf("load AWS configuration: %w", err) + return nil, err } - client := s3.NewFromConfig(configuration, func(options *s3.Options) { - options.UsePathStyle = repository.UsePathStyle - if repository.Endpoint != "" { - options.BaseEndpoint = aws.String(repository.Endpoint) - } - }) return New(&AWSClient{client: client, bucket: repository.Bucket}, brokers...), nil } @@ -199,29 +190,17 @@ func (client *AWSClient) Delete(ctx context.Context, key string, conditions Cond return normalizeAWSError(err) } +// normalizeAWSError translates a store's answer into this package's vocabulary. +// What the answer means is decided once, in awss3; what it is called is decided +// here, because this ErrNotFound is about a published revision rather than about +// an object and the blob store's identically-named sentinel is not the same +// thing. func normalizeAWSError(err error) error { - if err == nil { - return nil - } - var apiError smithy.APIError - if errors.As(err, &apiError) { - switch apiError.ErrorCode() { - case "NoSuchKey", "NotFound": - return fmt.Errorf("%w: %v", ErrNotFound, err) - case "NoSuchBucket": - return err - case "PreconditionFailed": - return fmt.Errorf("%w: %v", ErrPrecondition, err) - } - } - var responseError *transport.ResponseError - if errors.As(err, &responseError) { - switch responseError.HTTPStatusCode() { - case 404: - return fmt.Errorf("%w: %v", ErrNotFound, err) - case 412: - return fmt.Errorf("%w: %v", ErrPrecondition, err) - } + switch awss3.Classify(err) { + case awss3.NotFound: + return fmt.Errorf("%w: %v", ErrNotFound, err) + case awss3.Precondition: + return fmt.Errorf("%w: %v", ErrPrecondition, err) } return err } diff --git a/adapters/host/s3/aws_test.go b/adapters/host/s3/aws_test.go index b57e986..be5a604 100644 --- a/adapters/host/s3/aws_test.go +++ b/adapters/host/s3/aws_test.go @@ -22,3 +22,28 @@ func TestNormalizeAWSErrorDoesNotTreatConditionalConflictAsPrecondition(t *testi t.Fatal("conditional request conflict was normalized as a failed precondition") } } + +// The positive direction. Classification lives in internal/awss3 and is tested +// there; what these cover is the translation into this package's vocabulary, +// which is the part that is deliberately not shared — ErrNotFound here is about +// a published revision, and the blob store's identically-named sentinel is +// about an artifact. +func TestNormalizeAWSErrorTranslatesAMissingObject(t *testing.T) { + err := normalizeAWSError(&smithy.GenericAPIError{Code: "NoSuchKey", Message: "no such key"}) + if !errors.Is(err, ErrNotFound) { + t.Fatalf("a missing key was not normalized as ErrNotFound: %v", err) + } +} + +func TestNormalizeAWSErrorTranslatesARefusedPrecondition(t *testing.T) { + err := normalizeAWSError(&smithy.GenericAPIError{Code: "PreconditionFailed", Message: "precondition failed"}) + if !errors.Is(err, ErrPrecondition) { + t.Fatalf("a refused precondition was not normalized as ErrPrecondition: %v", err) + } +} + +func TestNormalizeAWSErrorPassesNilThrough(t *testing.T) { + if err := normalizeAWSError(nil); err != nil { + t.Fatalf("nil became %v", err) + } +} diff --git a/internal/awss3/awss3.go b/internal/awss3/awss3.go new file mode 100644 index 0000000..78d5b7f --- /dev/null +++ b/internal/awss3/awss3.go @@ -0,0 +1,113 @@ +// Package awss3 is the part of talking to an S3-compatible object store that +// does not depend on what is being stored. +// +// Two adapters use it. The blob store keeps content-addressed artifact bytes +// and streams them; the repository host publishes a tree, lists it, copies +// within it and switches a root object conditionally. Those are different +// clients and this does not try to be both — it is the part that was written +// twice and had already drifted: building the SDK client, and deciding what an +// error from it means. +// +// Classification returns a neutral kind rather than either adapter's sentinel. +// The two vocabularies are deliberately distinct — blob.ErrNotFound is about an +// artifact, the host's ErrNotFound about a published revision — so each adapter +// translates, and neither has to import the other's errors to reuse this. +package awss3 + +import ( + "context" + "errors" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + transport "github.com/aws/smithy-go/transport/http" +) + +// Config is what both adapters need to reach a bucket. Everything else about a +// repository or a blob store is the adapter's business. +type Config struct { + Bucket string + Region string + Endpoint string + UsePathStyle bool +} + +// NewClient builds an S3 client for one bucket. +// +// Region, endpoint and path style are the three settings an S3-compatible store +// that is not AWS invariably needs. Credentials are left to the SDK's own chain +// so an operator configures them where they already do — environment, profile, +// instance role — rather than in a workspace manifest that is committed. +func NewClient(ctx context.Context, configuration Config) (*s3.Client, error) { + options := []func(*awsconfig.LoadOptions) error{} + if configuration.Region != "" { + options = append(options, awsconfig.WithRegion(configuration.Region)) + } + loaded, err := awsconfig.LoadDefaultConfig(ctx, options...) + if err != nil { + return nil, fmt.Errorf("load AWS configuration: %w", err) + } + return s3.NewFromConfig(loaded, func(options *s3.Options) { + options.UsePathStyle = configuration.UsePathStyle + if configuration.Endpoint != "" { + options.BaseEndpoint = aws.String(configuration.Endpoint) + } + }), nil +} + +// Kind is what an error from an object store means to a caller. +type Kind int + +const ( + // Other is anything a caller has no special handling for, including a + // missing bucket: a bucket that does not exist is a configuration mistake, + // not an object that has yet to be written, and reading it as absence would + // have a first upload silently succeed against nothing. + Other Kind = iota + // NotFound means the object is not there. + NotFound + // Precondition means a conditional request was refused because the + // precondition did not hold, which is how both adapters detect a concurrent + // writer rather than an outage. + Precondition +) + +// Classify says what an error from the SDK means. +// +// A recognised error code decides on its own and the HTTP status is not +// consulted, which is the whole reason this is not a status switch: NoSuchBucket +// arrives as a 404, and reading the status alone would report a missing bucket +// as a missing object. +// +// An unrecognised code falls through to the status, because a bare status is +// all some stores give. A HEAD has no body to carry an error code, and MinIO, R2 +// and Ceph answer a missing key with the status and nothing else. +func Classify(err error) Kind { + if err == nil { + return Other + } + var apiError smithy.APIError + if errors.As(err, &apiError) { + switch apiError.ErrorCode() { + case "NoSuchKey", "NotFound": + return NotFound + case "NoSuchBucket": + return Other + case "PreconditionFailed": + return Precondition + } + } + var responseError *transport.ResponseError + if errors.As(err, &responseError) { + switch responseError.HTTPStatusCode() { + case 404: + return NotFound + case 412: + return Precondition + } + } + return Other +} diff --git a/internal/awss3/awss3_test.go b/internal/awss3/awss3_test.go new file mode 100644 index 0000000..d9c48d6 --- /dev/null +++ b/internal/awss3/awss3_test.go @@ -0,0 +1,96 @@ +package awss3 + +import ( + "net/http" + "testing" + + "github.com/aws/smithy-go" + transport "github.com/aws/smithy-go/transport/http" +) + +// coded is an error carrying an S3 error code, as a store returns when there is +// a body to put one in. +func coded(code string, status int) error { + return &transport.ResponseError{ + Response: &transport.Response{Response: &http.Response{StatusCode: status}}, + Err: &smithy.GenericAPIError{Code: code, Message: code}, + } +} + +// bare is a response carrying a status and no code, which is what a HEAD +// returns: there is no body to put one in. +func bare(status int) error { + return &transport.ResponseError{ + Response: &transport.Response{Response: &http.Response{StatusCode: status}}, + } +} + +func TestClassify(t *testing.T) { + for _, testcase := range []struct { + name string + err error + want Kind + }{ + {"no error", nil, Other}, + {"a missing key", coded("NoSuchKey", http.StatusNotFound), NotFound}, + {"a missing object on HEAD", coded("NotFound", http.StatusNotFound), NotFound}, + {"a refused precondition", coded("PreconditionFailed", http.StatusPreconditionFailed), Precondition}, + + // The reason this is not a switch on the status. A missing bucket is a + // configuration mistake and arrives as a 404 like any other; reading the + // status alone would report it as an object that has yet to be written, + // and a first upload would appear to succeed against nothing. + {"a missing bucket is not a missing object", coded("NoSuchBucket", http.StatusNotFound), Other}, + + // No body, so no code. MinIO, R2 and Ceph answer this way. + {"a bare not-found status", bare(http.StatusNotFound), NotFound}, + {"a bare precondition status", bare(http.StatusPreconditionFailed), Precondition}, + + // An unrecognised code still falls through to the status, because the + // store may be one whose codes this does not know. + {"an unknown code falls through to its status", coded("SomethingNew", http.StatusNotFound), NotFound}, + {"a conditional conflict is not a failed precondition", + coded("ConditionalRequestConflict", http.StatusConflict), Other}, + {"a server error is nothing in particular", bare(http.StatusInternalServerError), Other}, + } { + t.Run(testcase.name, func(t *testing.T) { + if got := Classify(testcase.err); got != testcase.want { + t.Fatalf("Classify(%v) = %v, want %v", testcase.err, got, testcase.want) + } + }) + } +} + +// The endpoint is the setting an S3-compatible store that is not AWS always +// needs, and getting it onto the client is the reason this package exists. +func TestNewClientAppliesTheStoreSettings(t *testing.T) { + client, err := NewClient(t.Context(), Config{ + Bucket: "packages", Region: "us-east-1", + Endpoint: "https://objects.example", UsePathStyle: true, + }) + if err != nil { + t.Fatal(err) + } + options := client.Options() + if got := *options.BaseEndpoint; got != "https://objects.example" { + t.Errorf("BaseEndpoint = %q, want the configured endpoint", got) + } + if !options.UsePathStyle { + t.Error("UsePathStyle was not applied") + } + if options.Region != "us-east-1" { + t.Errorf("Region = %q, want us-east-1", options.Region) + } +} + +// An unset endpoint leaves the SDK to resolve AWS's own, rather than pointing +// the client at an empty string. +func TestNewClientLeavesAnUnsetEndpointAlone(t *testing.T) { + client, err := NewClient(t.Context(), Config{Bucket: "packages", Region: "us-east-1"}) + if err != nil { + t.Fatal(err) + } + if client.Options().BaseEndpoint != nil { + t.Errorf("BaseEndpoint = %q, want unset", *client.Options().BaseEndpoint) + } +} From 025086ca21e4f085a760871681cace2fff8c0770 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 13:43:46 +0200 Subject: [PATCH 11/26] Let the caller say how to reach hosts --- engine/adopt_test.go | 2 +- engine/apply_concurrent_test.go | 14 +-- engine/collect.go | 6 +- engine/collect_test.go | 6 +- engine/endpoint_support_test.go | 2 +- engine/forge_identity_test.go | 2 +- engine/gitattributes_test.go | 2 +- engine/localhosts_test.go | 60 ++++++++++++ engine/origin_restore_test.go | 12 +-- engine/placements_test.go | 22 ++--- engine/plan_bench_test.go | 2 +- engine/preview_capability_test.go | 20 ++-- engine/progress_test.go | 16 +-- engine/prune_test.go | 14 +-- engine/raw_workspace_test.go | 4 +- engine/rollback.go | 6 +- engine/rollback_test.go | 4 +- engine/shared_blob_test.go | 6 +- engine/staging_test.go | 2 +- engine/two_runners_test.go | 6 +- engine/workspace.go | 30 +++--- engine/workspace_test.go | 156 +++++++++++++----------------- 22 files changed, 213 insertions(+), 181 deletions(-) create mode 100644 engine/localhosts_test.go diff --git a/engine/adopt_test.go b/engine/adopt_test.go index ab576e3..42768de 100644 --- a/engine/adopt_test.go +++ b/engine/adopt_test.go @@ -77,7 +77,7 @@ func TestAdoptArtifactPinsOriginAndSupportsDryRun(t *testing.T) { } commitWorkspace(t, root, "record adopted artifact") planName := filepath.Join(root, "adopt-plan.json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName}) + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName}) if err != nil { t.Fatal(err) } diff --git a/engine/apply_concurrent_test.go b/engine/apply_concurrent_test.go index b2967b4..7f27c03 100644 --- a/engine/apply_concurrent_test.go +++ b/engine/apply_concurrent_test.go @@ -53,7 +53,7 @@ func TestApplyPreparesRepositoriesConcurrently(t *testing.T) { root := multiRepositoryWorkspace(t, formats...) planName := filepath.Join(root, "plan.json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, ExpiresIn: time.Hour, VerificationMode: "structural", }) if err != nil { @@ -62,7 +62,7 @@ func TestApplyPreparesRepositoriesConcurrently(t *testing.T) { if planned.Changes != len(formats) { t.Fatalf("planned %d changes, want %d", planned.Changes, len(formats)) } - applied, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + applied, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, StructuralOnly: true, }) if err != nil { @@ -87,7 +87,7 @@ func TestApplyPreparesRepositoriesConcurrently(t *testing.T) { func TestApplyReportsTheFirstFailureInPlanOrder(t *testing.T) { root := multiRepositoryWorkspace(t, "pypi", "deb", "helm") planName := filepath.Join(root, "plan.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, ExpiresIn: time.Hour, VerificationMode: "structural", }); err != nil { t.Fatal(err) @@ -104,7 +104,7 @@ func TestApplyReportsTheFirstFailureInPlanOrder(t *testing.T) { } first := "" for range 5 { - _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, StructuralOnly: true, }) if err == nil { @@ -129,7 +129,7 @@ func TestApplyReportsTheFirstFailureInPlanOrder(t *testing.T) { func TestFailedApplyReleasesTheWorkspaceLock(t *testing.T) { root := multiRepositoryWorkspace(t, "pypi", "deb") planName := filepath.Join(root, "plan.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, ExpiresIn: time.Hour, VerificationMode: "structural", }); err != nil { t.Fatal(err) @@ -143,7 +143,7 @@ func TestFailedApplyReleasesTheWorkspaceLock(t *testing.T) { if err := os.WriteFile(lock, append(content, []byte("\n# changed after planning\n")...), 0o644); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, StructuralOnly: true, }); err == nil { t.Fatal("a stale plan was accepted") @@ -167,7 +167,7 @@ func TestConcurrentBlobLoadingIsDeterministic(t *testing.T) { digests := map[string]bool{} for range 4 { os.RemoveAll(filepath.Join(root, "public")) - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, ExpiresIn: time.Hour, VerificationMode: "structural", }); err != nil { t.Fatal(err) diff --git a/engine/collect.go b/engine/collect.go index bc2881e..d148b1a 100644 --- a/engine/collect.go +++ b/engine/collect.go @@ -91,6 +91,9 @@ func retentionFor(repository state.Repository, requested *int) int { } func CollectWorkspace(ctx context.Context, request CollectWorkspaceRequest) (CollectWorkspaceResult, error) { + if request.Hosts == nil { + return CollectWorkspaceResult{}, errNoHostResolver + } if request.Keep != nil && *request.Keep < 0 { return CollectWorkspaceResult{}, fmt.Errorf("keep %d is not a number of publications", *request.Keep) } @@ -112,9 +115,6 @@ func CollectWorkspace(ctx context.Context, request CollectWorkspaceRequest) (Col return CollectWorkspaceResult{}, err } hosts := request.Hosts - if hosts == nil { - hosts = localHostResolver{} - } result := CollectWorkspaceResult{ SchemaVersion: collectSchemaVersion, Workspace: manifest.Workspace.Name, GitRevision: revision, Applied: request.Apply, Repositories: []CollectRepository{}, diff --git a/engine/collect_test.go b/engine/collect_test.go index 6d74e04..6ba01d5 100644 --- a/engine/collect_test.go +++ b/engine/collect_test.go @@ -98,7 +98,7 @@ func keepOf(value int) *int { return &value } // strength of a number nobody checked. func TestCollectRefusesANegativeKeep(t *testing.T) { root := multiRepositoryWorkspace(t, "pypi") - if _, err := CollectWorkspace(context.Background(), CollectWorkspaceRequest{Root: root, Keep: keepOf(-1)}); err == nil { + if _, err := CollectWorkspace(context.Background(), CollectWorkspaceRequest{Hosts: localHosts(), Root: root, Keep: keepOf(-1)}); err == nil { t.Error("a negative keep was accepted") } } @@ -107,7 +107,7 @@ func TestCollectRefusesANegativeKeep(t *testing.T) { // cannot tell "nothing to do" from "not looked at". func TestAHostThatKeepsNothingSaysSo(t *testing.T) { root := multiRepositoryWorkspace(t, "pypi", "deb") - result, err := CollectWorkspace(context.Background(), CollectWorkspaceRequest{Root: root, Keep: keepOf(5)}) + result, err := CollectWorkspace(context.Background(), CollectWorkspaceRequest{Hosts: localHosts(), Root: root, Keep: keepOf(5)}) if err != nil { t.Fatal(err) } @@ -131,7 +131,7 @@ func TestAHostThatKeepsNothingSaysSo(t *testing.T) { // than an empty result that looks like success. func TestCollectRefusesAnUnknownRepository(t *testing.T) { root := multiRepositoryWorkspace(t, "pypi") - _, err := CollectWorkspace(context.Background(), CollectWorkspaceRequest{Root: root, Repository: "absent"}) + _, err := CollectWorkspace(context.Background(), CollectWorkspaceRequest{Hosts: localHosts(), Root: root, Repository: "absent"}) if err == nil || !strings.Contains(err.Error(), "absent") { t.Errorf("error = %v, want the unknown repository named", err) } diff --git a/engine/endpoint_support_test.go b/engine/endpoint_support_test.go index bba9227..ebe7c1e 100644 --- a/engine/endpoint_support_test.go +++ b/engine/endpoint_support_test.go @@ -31,7 +31,7 @@ func TestNoDeclaredFormatIsRefusedOutright(t *testing.T) { err := verifyEndpointClient(context.Background(), state.Repository{Format: format, Host: state.HostConfig{Type: hostType}}, t.TempDir(), host.ClientAccess{Endpoint: "https://example.test/repo"}, - ApplyWorkspaceRequest{StructuralOnly: true}) + ApplyWorkspaceRequest{Hosts: localHosts(), StructuralOnly: true}) if err != nil && strings.Contains(err.Error(), "client verification is not implemented") { t.Errorf("host %q declares %q verifiable, but the engine has no probe: %v", hostType, format, err) } diff --git a/engine/forge_identity_test.go b/engine/forge_identity_test.go index cae9898..9ee51d2 100644 --- a/engine/forge_identity_test.go +++ b/engine/forge_identity_test.go @@ -49,7 +49,7 @@ func TestThePlanCarriesTheForgeItWasPlannedAgainst(t *testing.T) { if output, err := commit.CombinedOutput(); err != nil { t.Fatalf("git commit: %v: %s", err, output) } - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root}) + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root}) if err != nil { t.Fatal(err) } diff --git a/engine/gitattributes_test.go b/engine/gitattributes_test.go index a80e0fc..498dad4 100644 --- a/engine/gitattributes_test.go +++ b/engine/gitattributes_test.go @@ -60,7 +60,7 @@ func TestPlanAcceptsWorkspaceWithLineEndingConversion(t *testing.T) { if !bytes.Contains(manifest, []byte("\r\n")) { t.Fatal("worktree manifest was not converted to CRLF, so this case proves nothing") } - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root}); err != nil { t.Fatalf("planning a line-ending-converting workspace failed: %v", err) } } diff --git a/engine/localhosts_test.go b/engine/localhosts_test.go new file mode 100644 index 0000000..20db8f9 --- /dev/null +++ b/engine/localhosts_test.go @@ -0,0 +1,60 @@ +package engine + +import ( + "context" + "errors" + "fmt" + "testing" + + localhost "github.com/shellcell/snailmail/adapters/host/local" + "github.com/shellcell/snailmail/host" +) + +// localHosts resolves the local directory host and nothing else. +// +// The engine used to fall back to exactly this when a caller supplied no +// resolver, which is how it came to import a driver. Selecting drivers is the +// composition root's job, and for a test the test is the composition root — so +// it lives here, where a test that means "publish to a directory" says so. +func localHosts() host.Resolver { return localResolver{} } + +type localResolver struct{} + +func (localResolver) Resolve(_ context.Context, repository host.Repository) (host.Host, error) { + if repository.Type != "local" { + return nil, fmt.Errorf("this test resolves only the local host, not %q", repository.Type) + } + return localhost.New(), nil +} + +// A caller that does not say how to reach hosts is refused rather than given +// one. The engine used to build a local-only resolver in that case, which both +// selected a driver and hid that it had — a workspace configured for S3 failed +// with a message about resolvers instead of publishing. +func TestEveryEntryPointRequiresAHostResolver(t *testing.T) { + root := t.TempDir() + for name, call := range map[string]func() error{ + "plan": func() error { + _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root}) + return err + }, + "apply": func() error { + _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root}) + return err + }, + "collect": func() error { + _, err := CollectWorkspace(context.Background(), CollectWorkspaceRequest{Root: root}) + return err + }, + "rollback": func() error { + _, err := RollbackRepository(context.Background(), RollbackRepositoryRequest{Root: root, Repository: "tools"}) + return err + }, + } { + t.Run(name, func(t *testing.T) { + if err := call(); !errors.Is(err, errNoHostResolver) { + t.Fatalf("%s without a resolver returned %v, want errNoHostResolver", name, err) + } + }) + } +} diff --git a/engine/origin_restore_test.go b/engine/origin_restore_test.go index c62d337..9172f4d 100644 --- a/engine/origin_restore_test.go +++ b/engine/origin_restore_test.go @@ -68,13 +68,13 @@ func TestPlanRestoresAbsentBlobsFromTheirRecordedOrigin(t *testing.T) { discardCAS(t, root) // Without the origin this is the failure a fresh clone hits today. - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "no-origin.snailmail-plan.json"), }); err == nil { t.Fatal("planning without any blob authority was expected to fail") } - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "restored.snailmail-plan.json"), Sources: fetcher, }); err != nil { t.Fatalf("planning with a recorded origin failed: %v", err) @@ -89,7 +89,7 @@ func TestRestoreRejectsAnOriginServingDifferentBytes(t *testing.T) { tampered := append(append([]byte(nil), content...), " extra"...) fetcher := &adoptMemoryFetcher{responses: map[string]source.Response{origin: {StatusCode: 200, Body: tampered}}} - _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "tampered.snailmail-plan.json"), Sources: fetcher, }) if err == nil { @@ -107,7 +107,7 @@ func TestRestoreReportsAnUnreachableOrigin(t *testing.T) { discardCAS(t, root) fetcher := &adoptMemoryFetcher{responses: map[string]source.Response{origin: {StatusCode: 404}}} - _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "missing.snailmail-plan.json"), Sources: fetcher, }) if err == nil { @@ -132,7 +132,7 @@ func TestRestoreLeavesACorruptLocalBlobAlone(t *testing.T) { if err := os.WriteFile(object, []byte("corrupted"), 0o444); err != nil { t.Fatal(err) } - _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "corrupt.snailmail-plan.json"), Sources: fetcher, }) if err == nil { @@ -172,7 +172,7 @@ func TestRestoreRefusesANonPublicOrigin(t *testing.T) { fetcher := &adoptMemoryFetcher{responses: map[string]source.Response{ "http://127.0.0.1:9/" + "demo-1.2.3-py3-none-any.whl": {StatusCode: 200, Body: content}, }} - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "private.snailmail-plan.json"), Sources: fetcher, }); err == nil { t.Fatal("a loopback origin was fetched") diff --git a/engine/placements_test.go b/engine/placements_test.go index b44d70a..53eea41 100644 --- a/engine/placements_test.go +++ b/engine/placements_test.go @@ -35,7 +35,7 @@ func TestPromoteAndYankPlacementLifecycle(t *testing.T) { planAndApply := func(label string, at time.Time, wantChanges int) { t.Helper() planName := filepath.Join(root, label+".json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: at, GeneratedAt: baseTime, ExpiresIn: time.Hour, VerificationMode: "structural", }) if err != nil { @@ -44,7 +44,7 @@ func TestPromoteAndYankPlacementLifecycle(t *testing.T) { if planned.Changes != wantChanges { t.Fatalf("%s changes=%d want=%d", label, planned.Changes, wantChanges) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: at.Add(time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: at.Add(time.Minute), StructuralOnly: true}); err != nil { t.Fatalf("%s apply: %v", label, err) } } @@ -57,7 +57,7 @@ func TestPromoteAndYankPlacementLifecycle(t *testing.T) { if repeated, err := Promote(PlacementMutationRequest{Root: root, Repository: "python", Package: "snail-demo", Version: "1.2.3", Track: "testing"}); err != nil || repeated.Changed != 0 { t.Fatalf("duplicate promote=%#v err=%v", repeated, err) } - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: filepath.Join(root, "dirty.json"), createdAt: baseTime.Add(time.Hour)}); err == nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "dirty.json"), createdAt: baseTime.Add(time.Hour)}); err == nil { t.Fatal("planning accepted uncommitted promotion") } commitWorkspace(t, root, "promote package to testing") @@ -69,10 +69,10 @@ func TestPromoteAndYankPlacementLifecycle(t *testing.T) { } commitWorkspace(t, root, "yank stable placement") planAndApply("one-placement", baseTime.Add(4*time.Hour), 1) - if result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: filepath.Join(root, "one-placement.json"), now: baseTime.Add(4*time.Hour + 2*time.Minute), StructuralOnly: true}); err != nil || result.Current != 1 { + if result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: filepath.Join(root, "one-placement.json"), now: baseTime.Add(4*time.Hour + 2*time.Minute), StructuralOnly: true}); err != nil || result.Current != 1 { t.Fatalf("removal-only apply retry=%#v err=%v", result, err) } - converged, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + converged, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "one-placement-converged.json"), createdAt: baseTime.Add(5 * time.Hour), GeneratedAt: baseTime, ExpiresIn: time.Hour, VerificationMode: "structural", }) if err != nil || converged.Changes != 0 { @@ -96,10 +96,10 @@ func TestPromoteAndYankPlacementLifecycle(t *testing.T) { } commitWorkspace(t, root, "yank final placement") planAndApply("empty", baseTime.Add(6*time.Hour), 0) - if result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: filepath.Join(root, "empty.json"), now: baseTime.Add(6*time.Hour + 2*time.Minute), StructuralOnly: true}); err != nil || result.Current != 1 { + if result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: filepath.Join(root, "empty.json"), now: baseTime.Add(6*time.Hour + 2*time.Minute), StructuralOnly: true}); err != nil || result.Current != 1 { t.Fatalf("empty apply retry=%#v err=%v", result, err) } - converged, err = PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + converged, err = PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "empty-converged.json"), createdAt: baseTime.Add(7 * time.Hour), GeneratedAt: baseTime, ExpiresIn: time.Hour, VerificationMode: "structural", }) if err != nil || converged.Changes != 0 { @@ -119,10 +119,10 @@ func TestPromoteAndYankPlacementLifecycle(t *testing.T) { } commitWorkspace(t, root, "restore stable placement") planAndApply("restored", baseTime.Add(8*time.Hour), 1) - if result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: filepath.Join(root, "restored.json"), now: baseTime.Add(8*time.Hour + 2*time.Minute), StructuralOnly: true}); err != nil || result.Current != 1 { + if result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: filepath.Join(root, "restored.json"), now: baseTime.Add(8*time.Hour + 2*time.Minute), StructuralOnly: true}); err != nil || result.Current != 1 { t.Fatalf("restored apply retry=%#v err=%v", result, err) } - restoredPlan, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + restoredPlan, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "restored-converged.json"), createdAt: baseTime.Add(9 * time.Hour), GeneratedAt: baseTime, ExpiresIn: time.Hour, VerificationMode: "structural", }) if err != nil || restoredPlan.Changes != 0 { @@ -177,10 +177,10 @@ func TestFinalYankBuildsEveryEmptyRepositoryFormat(t *testing.T) { commitWorkspace(t, root, "configure empty "+format+" repository") at := time.Date(2026, time.July, 26, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "empty.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName, createdAt: at, GeneratedAt: at, ExpiresIn: time.Hour, VerificationMode: "structural"}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: at, GeneratedAt: at, ExpiresIn: time.Hour, VerificationMode: "structural"}); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: at.Add(time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: at.Add(time.Minute), StructuralOnly: true}); err != nil { t.Fatal(err) } manifest, err := app.VerifyRepository(filepath.Join(root, "public", "packages")) diff --git a/engine/plan_bench_test.go b/engine/plan_bench_test.go index 46a5909..1ed8a5a 100644 --- a/engine/plan_bench_test.go +++ b/engine/plan_bench_test.go @@ -55,7 +55,7 @@ func BenchmarkPlanWorkspace(b *testing.B) { func benchPlan(b *testing.B, versions int) { root := benchWorkspace(b, versions) for b.Loop() { - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(b.TempDir(), "plan.json"), ExpiresIn: time.Hour, VerificationMode: "structural", }); err != nil { diff --git a/engine/preview_capability_test.go b/engine/preview_capability_test.go index 795529e..7f51c4e 100644 --- a/engine/preview_capability_test.go +++ b/engine/preview_capability_test.go @@ -133,8 +133,7 @@ func TestPlanAndApplyPreviewlessHostUnderAutoGate(t *testing.T) { createdAt := time.Date(2026, time.August, 22, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "previewless.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ - Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, Hosts: resolver, }); err != nil { t.Fatalf("planning a repository on a previewless host failed: %v", err) @@ -153,9 +152,7 @@ func TestPlanAndApplyPreviewlessHostUnderAutoGate(t *testing.T) { // Not StructuralOnly: the point is that the real client verification path // runs against the staged tree when there is no endpoint to install from. - result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ - Root: root, Plan: planName, now: createdAt.Add(time.Minute), Hosts: resolver, - }) + result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: createdAt.Add(time.Minute), Hosts: resolver}) if err != nil { t.Fatalf("applying to a previewless host failed: %v", err) } @@ -178,8 +175,7 @@ func TestAPreviewlessHostSettlesWithoutAManifestDigest(t *testing.T) { createdAt := time.Date(2026, time.August, 22, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "first.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ - Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, Hosts: resolver, }); err != nil { t.Fatal(err) @@ -191,9 +187,7 @@ func TestAPreviewlessHostSettlesWithoutAManifestDigest(t *testing.T) { if plan.Payload.Repositories[0].ReportsManifestDigest { t.Fatal("plan claimed a manifest digest the host does not report") } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ - Root: root, Plan: planName, now: createdAt.Add(time.Minute), Hosts: resolver, - }); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: createdAt.Add(time.Minute), Hosts: resolver}); err != nil { t.Fatal(err) } @@ -201,8 +195,7 @@ func TestAPreviewlessHostSettlesWithoutAManifestDigest(t *testing.T) { // manifest embeds it: a later timestamp changes the manifest digest while the // tree digest stays put, and the receipt would rightly report the generated // metadata as changed. The CLI pins it to a fixed epoch for the same reason. - settled, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ - Root: root, Output: filepath.Join(root, "second.json"), + settled, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: filepath.Join(root, "second.json"), createdAt: createdAt.Add(time.Hour), GeneratedAt: createdAt, ExpiresIn: time.Hour, Hosts: resolver, }) @@ -221,8 +214,7 @@ func TestPlanRefusesPreviewlessHostUnderHumanGate(t *testing.T) { t.Run(gate, func(t *testing.T) { root := previewlessWorkspace(t, gate) createdAt := time.Date(2026, time.August, 22, 1, 2, 3, 0, time.UTC) - _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ - Root: root, Output: filepath.Join(root, "refused.json"), + _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: filepath.Join(root, "refused.json"), createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, Hosts: staticHostResolver{host: &previewlessHost{}}, }) diff --git a/engine/progress_test.go b/engine/progress_test.go index de48467..c3ebf74 100644 --- a/engine/progress_test.go +++ b/engine/progress_test.go @@ -40,13 +40,13 @@ func appliedWithProgress(t *testing.T, formats ...string) *progressLog { t.Helper() root := multiRepositoryWorkspace(t, formats...) planName := filepath.Join(root, "plan.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, ExpiresIn: time.Hour, VerificationMode: "structural", }); err != nil { t.Fatal(err) } log := &progressLog{} - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, StructuralOnly: true, Progress: log.record, }); err != nil { t.Fatal(err) @@ -124,12 +124,12 @@ func TestPrepareReportsOncePerRepository(t *testing.T) { func TestApplyWithoutProgressStillWorks(t *testing.T) { root := multiRepositoryWorkspace(t, "pypi") planName := filepath.Join(root, "plan.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, ExpiresIn: time.Hour, VerificationMode: "structural", }); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, StructuralOnly: true, }); err != nil { t.Fatal(err) @@ -166,13 +166,13 @@ func TestPhasesAreReportedInOrder(t *testing.T) { func TestDryRunWritesNothingToTheHost(t *testing.T) { root := multiRepositoryWorkspace(t, "pypi") planName := filepath.Join(root, "plan.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, ExpiresIn: time.Hour, VerificationMode: "structural", }); err != nil { t.Fatal(err) } log := &progressLog{} - result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, StructuralOnly: true, DryRun: true, Progress: log.record, }) if err != nil { @@ -196,12 +196,12 @@ func TestDryRunWritesNothingToTheHost(t *testing.T) { } // And a real apply afterwards still publishes, so a dry run leaves nothing // behind that blocks the thing it was previewing. - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, ExpiresIn: time.Hour, VerificationMode: "structural", }); err != nil { t.Fatal(err) } - applied, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + applied, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, StructuralOnly: true, }) if err != nil { diff --git a/engine/prune_test.go b/engine/prune_test.go index cda672b..0bd2da5 100644 --- a/engine/prune_test.go +++ b/engine/prune_test.go @@ -34,10 +34,10 @@ func TestPruneRetainsHistoryAndSupportsRepromotion(t *testing.T) { commitWorkspace(t, root, "configure prune fixture") baseTime := time.Date(2026, time.July, 26, 2, 3, 4, 0, time.UTC) initialPlan := filepath.Join(root, "initial-prune.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: initialPlan, createdAt: baseTime, GeneratedAt: baseTime, ExpiresIn: time.Hour, VerificationMode: "structural"}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: initialPlan, createdAt: baseTime, GeneratedAt: baseTime, ExpiresIn: time.Hour, VerificationMode: "structural"}); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: initialPlan, now: baseTime.Add(time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: initialPlan, now: baseTime.Add(time.Minute), StructuralOnly: true}); err != nil { t.Fatal(err) } result, err := Prune(PruneRequest{Root: root, Repository: "python", Keep: 1}) @@ -66,12 +66,12 @@ func TestPruneRetainsHistoryAndSupportsRepromotion(t *testing.T) { } } } - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: filepath.Join(root, "dirty-prune.json"), createdAt: baseTime.Add(time.Hour)}); err == nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "dirty-prune.json"), createdAt: baseTime.Add(time.Hour)}); err == nil { t.Fatal("planning accepted uncommitted prune") } commitWorkspace(t, root, "prune old stable placement") prunePlanName := filepath.Join(root, "prune.json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: prunePlanName, createdAt: baseTime.Add(2 * time.Hour), GeneratedAt: baseTime, ExpiresIn: time.Hour, VerificationMode: "structural", }) if err != nil { @@ -84,10 +84,10 @@ func TestPruneRetainsHistoryAndSupportsRepromotion(t *testing.T) { if planned.Changes != 1 || plan.Payload.Repositories[0].PublicationRecords || len(plan.Payload.Repositories[0].PublicationBindings) != 0 { t.Fatalf("prune plan effects %#v", plan.Payload.Repositories[0]) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: prunePlanName, now: baseTime.Add(2*time.Hour + time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: prunePlanName, now: baseTime.Add(2*time.Hour + time.Minute), StructuralOnly: true}); err != nil { t.Fatal(err) } - if retry, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: prunePlanName, now: baseTime.Add(2*time.Hour + 2*time.Minute), StructuralOnly: true}); err != nil || retry.Current != 1 { + if retry, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: prunePlanName, now: baseTime.Add(2*time.Hour + 2*time.Minute), StructuralOnly: true}); err != nil || retry.Current != 1 { t.Fatalf("prune retry=%#v err=%v", retry, err) } published, err := app.VerifyRepository(filepath.Join(root, "public", "python")) @@ -102,7 +102,7 @@ func TestPruneRetainsHistoryAndSupportsRepromotion(t *testing.T) { } commitWorkspace(t, root, "re-promote retained package version") restorePlanName := filepath.Join(root, "prune-restore.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: restorePlanName, createdAt: baseTime.Add(4 * time.Hour), GeneratedAt: baseTime, ExpiresIn: time.Hour, VerificationMode: "structural"}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: restorePlanName, createdAt: baseTime.Add(4 * time.Hour), GeneratedAt: baseTime, ExpiresIn: time.Hour, VerificationMode: "structural"}); err != nil { t.Fatal(err) } restorePlan, err := state.LoadPlan(restorePlanName) diff --git a/engine/raw_workspace_test.go b/engine/raw_workspace_test.go index a4d3ead..26bbea5 100644 --- a/engine/raw_workspace_test.go +++ b/engine/raw_workspace_test.go @@ -55,10 +55,10 @@ func TestRawWorkspacePublishesSuppliedIdentity(t *testing.T) { commitWorkspace(t, root, "record raw artifacts") plan := filepath.Join(root, "raw.snailmail-plan.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: plan}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: plan}); err != nil { t.Fatalf("planning a raw repository failed: %v", err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: plan}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: plan}); err != nil { t.Fatalf("applying a raw repository failed: %v", err) } diff --git a/engine/rollback.go b/engine/rollback.go index 55e55f0..6b164ad 100644 --- a/engine/rollback.go +++ b/engine/rollback.go @@ -47,6 +47,9 @@ type RollbackRepositoryResult struct { } func RollbackRepository(ctx context.Context, request RollbackRepositoryRequest) (RollbackRepositoryResult, error) { + if request.Hosts == nil { + return RollbackRepositoryResult{}, errNoHostResolver + } if request.Repository == "" { return RollbackRepositoryResult{}, errors.New("rollback requires the repository to roll back") } @@ -68,9 +71,6 @@ func RollbackRepository(ctx context.Context, request RollbackRepositoryRequest) return RollbackRepositoryResult{}, fmt.Errorf("repository %q is not configured", request.Repository) } hosts := request.Hosts - if hosts == nil { - hosts = localHostResolver{} - } hostIdentity, err := repositoryHostIdentity(repository) if err != nil { return RollbackRepositoryResult{}, err diff --git a/engine/rollback_test.go b/engine/rollback_test.go index 6032c34..4979f1f 100644 --- a/engine/rollback_test.go +++ b/engine/rollback_test.go @@ -66,9 +66,7 @@ func livingHost(restorable bool) *rollbackHost { } func rollbackRequest(root string, remote host.Host) RollbackRepositoryRequest { - return RollbackRepositoryRequest{ - Root: root, Repository: "python", Hosts: staticHostResolver{host: remote}, - } + return RollbackRepositoryRequest{Root: root, Repository: "python", Hosts: staticHostResolver{host: remote}} } // The gap this closes: a publication that succeeded and turned out to be wrong had diff --git a/engine/shared_blob_test.go b/engine/shared_blob_test.go index c99853f..bd35dc8 100644 --- a/engine/shared_blob_test.go +++ b/engine/shared_blob_test.go @@ -59,10 +59,10 @@ func TestOneVersionMayHoldTwoArtifactsWithIdenticalBytes(t *testing.T) { commitWorkspace(t, root, "one installer, two architectures") planName := filepath.Join(root, "shared.snailmail-plan.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName}); err != nil { t.Fatalf("planning a version whose artifacts share bytes failed: %v", err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName}); err != nil { t.Fatalf("applying a version whose artifacts share bytes failed: %v", err) } @@ -88,7 +88,7 @@ func TestOneVersionMayHoldTwoArtifactsWithIdenticalBytes(t *testing.T) { // The binding has to settle: while the recorded set and the derived list // could not compare equal, every plan re-recorded it and never converged. settled := filepath.Join(root, "settled.snailmail-plan.json") - result, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: settled}) + result, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: settled}) if err != nil { t.Fatal(err) } diff --git a/engine/staging_test.go b/engine/staging_test.go index 372e839..d87acf7 100644 --- a/engine/staging_test.go +++ b/engine/staging_test.go @@ -23,7 +23,7 @@ func TestPlanStagesInsideWorkspaceRatherThanTempDir(t *testing.T) { t.Fatal(err) } commitWorkspace(t, root, "initialize workspace") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root}); err != nil { t.Fatal(err) } diff --git a/engine/two_runners_test.go b/engine/two_runners_test.go index 5b2be9c..1effd0c 100644 --- a/engine/two_runners_test.go +++ b/engine/two_runners_test.go @@ -73,7 +73,7 @@ func commitAll(t *testing.T, root, message string) { func planFor(t *testing.T, root string) string { t.Helper() name := filepath.Join(root, "plan.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: name, ExpiresIn: time.Hour, VerificationMode: "structural", }); err != nil { t.Fatal(err) @@ -89,7 +89,7 @@ func TestApplyingOnePlanTwiceIsNotTwoPublications(t *testing.T) { commitAll(t, root, "configure") plan := planFor(t, root) - firstResult, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + firstResult, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: plan, StructuralOnly: true, }) if err != nil { @@ -100,7 +100,7 @@ func TestApplyingOnePlanTwiceIsNotTwoPublications(t *testing.T) { } // The same plan again. Either it is refused or it is recognised as already // applied; what it must not be is a second publication. - secondResult, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + secondResult, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: plan, StructuralOnly: true, }) if err != nil { diff --git a/engine/workspace.go b/engine/workspace.go index bd77c96..f789a1c 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -15,7 +15,6 @@ import ( "sync" "time" - localhost "github.com/shellcell/snailmail/adapters/host/local" "github.com/shellcell/snailmail/blob" "github.com/shellcell/snailmail/formats" "github.com/shellcell/snailmail/gate" @@ -705,6 +704,9 @@ func requireHostCapabilities(name string, repository state.Repository, capabilit } func PlanWorkspace(ctx context.Context, request PlanWorkspaceRequest) (PlanWorkspaceResult, error) { + if request.Hosts == nil { + return PlanWorkspaceResult{}, errNoHostResolver + } root, err := workspaceRoot(request.Root) if err != nil { return PlanWorkspaceResult{}, err @@ -780,9 +782,6 @@ func PlanWorkspace(ctx context.Context, request PlanWorkspaceRequest) (PlanWorks changes := 0 var plannedAcquisitions []PlannedAcquisition hosts := request.Hosts - if hosts == nil { - hosts = localHostResolver{} - } preparation := &planPreparation{ ctx: ctx, root: root, request: request, manifest: manifest, hosts: hosts, blobStore: blobStore, createdAt: createdAt, generatedAt: generatedAt, expiresIn: expiresIn, @@ -952,9 +951,6 @@ func openApply(ctx context.Context, request ApplyWorkspaceRequest) (*applyPrepar return nil, nil, err } hosts := request.Hosts - if hosts == nil { - hosts = localHostResolver{} - } seenRepositories := make(map[string]bool) preparation := &applyPreparation{ ctx: ctx, root: root, request: request, plan: plan, manifest: manifest, hosts: hosts, @@ -973,6 +969,9 @@ func openApply(ctx context.Context, request ApplyWorkspaceRequest) (*applyPrepar } func ApplyWorkspace(ctx context.Context, request ApplyWorkspaceRequest) (ApplyWorkspaceResult, error) { + if request.Hosts == nil { + return ApplyWorkspaceResult{}, errNoHostResolver + } preparation, unlock, err := openApply(ctx, request) if err != nil { return ApplyWorkspaceResult{}, err @@ -1625,14 +1624,15 @@ func optionalSigningKeys(name string) []string { return []string{name} } -type localHostResolver struct{} - -func (localHostResolver) Resolve(_ context.Context, repository host.Repository) (host.Host, error) { - if repository.Type != "local" { - return nil, fmt.Errorf("host type %q requires a configured host resolver", repository.Type) - } - return localhost.New(), nil -} +// errNoHostResolver is returned when a caller does not say how to reach hosts. +// +// The engine used to fall back to a resolver it built itself out of the local +// adapter — the one driver the application layer knew by name, imported +// directly. ARCHITECTURE §2 puts driver selection at the composition root, and a +// default here meant the engine both selected a driver and concealed that it +// had: a caller who forgot to wire one got local-only publishing rather than an +// answer. +var errNoHostResolver = errors.New("no host resolver was provided; hosts are selected by the caller") func toHostRepository(root, workspaceID, hostIdentity, name string, repository state.Repository) host.Repository { var commitPaths []string diff --git a/engine/workspace_test.go b/engine/workspace_test.go index a045dcd..6191aa9 100644 --- a/engine/workspace_test.go +++ b/engine/workspace_test.go @@ -45,7 +45,7 @@ func TestWorkspacePlanApplyAllFormats(t *testing.T) { commitWorkspace(t, root, "add artifact") createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "reviewed.json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, }) if err != nil { @@ -54,7 +54,7 @@ func TestWorkspacePlanApplyAllFormats(t *testing.T) { if planned.Changes != 1 { t.Fatalf("planned %d changes, want 1", planned.Changes) } - applied, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + applied, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true, }) if err != nil { @@ -81,7 +81,7 @@ func TestWorkspacePlanApplyAllFormats(t *testing.T) { if len(records) != 1 || records[0].PlanID != planned.PlanID { t.Fatalf("unexpected publication records: %#v", records) } - retried, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + retried, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: createdAt.Add(2 * time.Minute), StructuralOnly: true, }) if err != nil { @@ -166,7 +166,7 @@ func TestSignedDebianPlanEmbedsResponsesAndApplyNeedsNoSigner(t *testing.T) { } commitWorkspace(t, root, "configure signed Debian repository") planName := filepath.Join(root, "signed.json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt.Add(time.Hour), GeneratedAt: createdAt.Add(time.Hour), ExpiresIn: time.Hour, VerificationMode: "structural", Signers: store, }) @@ -205,7 +205,7 @@ func TestSignedDebianPlanEmbedsResponsesAndApplyNeedsNoSigner(t *testing.T) { if err := state.WritePlan(extendedName, extended); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: extendedName, now: createdAt.Add(90 * time.Minute), StructuralOnly: true}); err == nil || !strings.Contains(err.Error(), "expires after its signing key") { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: extendedName, now: createdAt.Add(90 * time.Minute), StructuralOnly: true}); err == nil || !strings.Contains(err.Error(), "expires after its signing key") { t.Fatalf("apply accepted plan extending beyond signing key: %v", err) } tamperedPayload := plan.Payload @@ -226,10 +226,10 @@ func TestSignedDebianPlanEmbedsResponsesAndApplyNeedsNoSigner(t *testing.T) { if err := state.WritePlan(tamperedName, tampered); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: tamperedName, now: createdAt.Add(90 * time.Minute), StructuralOnly: true}); err == nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: tamperedName, now: createdAt.Add(90 * time.Minute), StructuralOnly: true}); err == nil { t.Fatal("apply accepted rehashed plan with invalid signature bytes") } - result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: createdAt.Add(90 * time.Minute), StructuralOnly: true, }) if err != nil || result.Applied != 1 || result.PlanID != planned.PlanID { @@ -250,7 +250,7 @@ func TestSignedDebianPlanEmbedsResponsesAndApplyNeedsNoSigner(t *testing.T) { if err := state.WritePlan(malformedName, malformed); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: malformedName, now: createdAt.Add(90 * time.Minute), StructuralOnly: true}); err == nil || !strings.Contains(err.Error(), "dependencies") { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: malformedName, now: createdAt.Add(90 * time.Minute), StructuralOnly: true}); err == nil || !strings.Contains(err.Error(), "dependencies") { t.Fatalf("current signed target accepted malformed recipe: %v", err) } release := filepath.Join(root, "public", "debian") @@ -273,10 +273,10 @@ func TestSignedDebianPlanEmbedsResponsesAndApplyNeedsNoSigner(t *testing.T) { } } defaultPlan := filepath.Join(root, "default-time.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: defaultPlan, ExpiresIn: time.Hour, Signers: store}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: defaultPlan, ExpiresIn: time.Hour, Signers: store}); err != nil { t.Fatalf("default wall-clock signed plan: %v", err) } - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: filepath.Join(root, "past-key-expiry.json"), ExpiresIn: 2 * 365 * 24 * time.Hour, Signers: store}); err == nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "past-key-expiry.json"), ExpiresIn: 2 * 365 * 24 * time.Hour, Signers: store}); err == nil { t.Fatal("plan lifetime exceeded signing key validity") } } @@ -337,13 +337,13 @@ func TestDebianSigningKeyRotationLifecycle(t *testing.T) { applyAt := func(label string, at time.Time) buildgraph.RepositoryManifest { t.Helper() planName := filepath.Join(root, label+".json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: at, GeneratedAt: at, ExpiresIn: time.Hour, VerificationMode: "structural", Signers: store, }); err != nil { t.Fatalf("%s plan: %v", label, err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: at.Add(time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: at.Add(time.Minute), StructuralOnly: true}); err != nil { t.Fatalf("%s apply: %v", label, err) } manifest, err := app.VerifyRepository(filepath.Join(root, "public", "debian")) @@ -390,19 +390,19 @@ func TestDebianSigningKeyRotationLifecycle(t *testing.T) { } introducedAt := rotationTime.Add(time.Hour) introducedPlan := filepath.Join(root, "introduced.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: introducedPlan, createdAt: introducedAt, GeneratedAt: introducedAt, ExpiresIn: time.Hour, VerificationMode: "structural", Signers: store, }); err != nil { t.Fatalf("introduced plan: %v", err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: introducedPlan, now: introducedAt.Add(time.Minute), StructuralOnly: true, beforeDeploymentCommit: func() error { return errors.New("simulated process interruption") }, }); err == nil { t.Fatal("introduction unexpectedly recorded receipt after simulated interruption") } - if result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: introducedPlan, now: introducedAt.Add(2 * time.Minute), StructuralOnly: true}); err != nil || result.Current != 1 { + if result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: introducedPlan, now: introducedAt.Add(2 * time.Minute), StructuralOnly: true}); err != nil || result.Current != 1 { t.Fatalf("introduction receipt recovery=%#v err=%v", result, err) } introduced, err := app.VerifyRepository(filepath.Join(root, "public", "debian")) @@ -481,7 +481,7 @@ func TestDebianSigningKeyRotationLifecycle(t *testing.T) { t.Fatal(err) } commitWorkspace(t, root, "attempt early direct activation") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "direct-activation.json"), createdAt: trustSince.Add(time.Hour), GeneratedAt: trustSince.Add(time.Hour), ExpiresIn: time.Hour, VerificationMode: "structural", Signers: store, }); err == nil { @@ -505,7 +505,7 @@ func TestDebianSigningKeyRotationLifecycle(t *testing.T) { t.Fatal(err) } commitWorkspace(t, root, "attempt to skip activated overlap") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "direct-retirement.json"), createdAt: trustSince.Add(minimumRefresh + time.Hour), GeneratedAt: trustSince.Add(minimumRefresh + time.Hour), ExpiresIn: time.Hour, VerificationMode: "structural", Signers: store, }); err == nil { @@ -552,7 +552,7 @@ func TestDebianSigningKeyRotationLifecycle(t *testing.T) { t.Fatal(err) } commitWorkspace(t, root, "attempt early direct retirement") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: filepath.Join(root, "early-retirement.json"), createdAt: activationTrustSince.Add(time.Hour), GeneratedAt: activationTrustSince.Add(time.Hour), ExpiresIn: time.Hour, VerificationMode: "structural", Signers: store, }); err == nil { @@ -640,8 +640,7 @@ func TestWorkspacePlanApplyRemoteHost(t *testing.T) { resolver := staticHostResolver{host: remote} createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "remote.json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ - Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, Hosts: resolver, }) if err != nil { @@ -669,18 +668,14 @@ func TestWorkspacePlanApplyRemoteHost(t *testing.T) { if !strings.Contains(string(document), "python -m pip install --index-url 'https://packages.example/python/simple' ") { t.Fatalf("unexpected install document %q", document) } - result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ - Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true, Hosts: resolver, - }) + result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true, Hosts: resolver}) if err != nil { t.Fatal(err) } if result.Applied != 1 || remote.stageCalls != 1 || remote.commitCalls != 1 || remote.revision.TreeSHA256 != plan.Payload.Repositories[0].DesiredTreeSHA256 { t.Fatalf("unexpected remote apply result %#v stage=%d commit=%d revision=%#v", result, remote.stageCalls, remote.commitCalls, remote.revision) } - retried, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ - Root: root, Plan: planName, now: createdAt.Add(2 * time.Minute), StructuralOnly: true, Hosts: resolver, - }) + retried, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: createdAt.Add(2 * time.Minute), StructuralOnly: true, Hosts: resolver}) if err != nil { t.Fatal(err) } @@ -688,9 +683,7 @@ func TestWorkspacePlanApplyRemoteHost(t *testing.T) { t.Fatalf("remote retry was not idempotent: %#v", retried) } remote.revision.ManifestSHA256 = strings.Repeat("f", 64) - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ - Root: root, Plan: planName, now: createdAt.Add(3 * time.Minute), StructuralOnly: true, Hosts: resolver, - }); err == nil || !strings.Contains(err.Error(), "desired tree was published by another change") { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: createdAt.Add(3 * time.Minute), StructuralOnly: true, Hosts: resolver}); err == nil || !strings.Contains(err.Error(), "desired tree was published by another change") { t.Fatalf("apply accepted a different desired manifest: %v", err) } records, err := state.LoadLedger(root, "python") @@ -702,8 +695,7 @@ func TestWorkspacePlanApplyRemoteHost(t *testing.T) { } remote.revision.ManifestSHA256 = plan.Payload.Repositories[0].DesiredManifestSHA256 secondPlanName := filepath.Join(root, "remote-second.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ - Root: root, Output: secondPlanName, createdAt: createdAt.Add(10 * time.Minute), GeneratedAt: createdAt.Add(10 * time.Minute), + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: secondPlanName, createdAt: createdAt.Add(10 * time.Minute), GeneratedAt: createdAt.Add(10 * time.Minute), ExpiresIn: time.Hour, Hosts: resolver, }); err != nil { t.Fatal(err) @@ -716,9 +708,7 @@ func TestWorkspacePlanApplyRemoteHost(t *testing.T) { secondPlan.Payload.Repositories[0].ObservedManifestSHA256 == secondPlan.Payload.Repositories[0].DesiredManifestSHA256 { t.Fatalf("same-tree manifest change was not planned as an update: %#v", secondPlan.Payload.Repositories[0]) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ - Root: root, Plan: secondPlanName, now: createdAt.Add(11 * time.Minute), StructuralOnly: true, Hosts: resolver, - }); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: secondPlanName, now: createdAt.Add(11 * time.Minute), StructuralOnly: true, Hosts: resolver}); err != nil { t.Fatalf("apply same-tree manifest update: %v", err) } } @@ -749,22 +739,16 @@ func TestWorkspacePlanApplyGitHubPagesHost(t *testing.T) { resolver := staticHostResolver{host: remote} createdAt := time.Date(2026, time.July, 25, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "pages.json") - planResult, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ - Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, Hosts: resolver, - }) + planResult, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, Hosts: resolver}) if err != nil { t.Fatal(err) } - result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ - Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true, Hosts: resolver, - }) + result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true, Hosts: resolver}) if err != nil || result.Applied != 1 || remote.stageCalls != 1 || remote.commitCalls != 1 || remote.staged.PreviousRevision != "" { t.Fatalf("Pages apply result=%#v plan=%#v stage=%#v err=%v", result, planResult, remote.staged, err) } secondPlanName := filepath.Join(root, "pages-second.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ - Root: root, Output: secondPlanName, createdAt: createdAt.Add(10 * time.Minute), GeneratedAt: createdAt.Add(10 * time.Minute), ExpiresIn: time.Hour, Hosts: resolver, - }); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: secondPlanName, createdAt: createdAt.Add(10 * time.Minute), GeneratedAt: createdAt.Add(10 * time.Minute), ExpiresIn: time.Hour, Hosts: resolver}); err != nil { t.Fatal(err) } secondPlan, err := state.LoadPlan(secondPlanName) @@ -774,9 +758,7 @@ func TestWorkspacePlanApplyGitHubPagesHost(t *testing.T) { if secondPlan.Payload.Repositories[0].Action != "update" || secondPlan.Payload.Repositories[0].ObservedTreeSHA256 != secondPlan.Payload.Repositories[0].DesiredTreeSHA256 || secondPlan.Payload.Repositories[0].ObservedManifestSHA256 == secondPlan.Payload.Repositories[0].DesiredManifestSHA256 { t.Fatalf("Pages same-tree manifest update was not planned: %#v", secondPlan.Payload.Repositories[0]) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ - Root: root, Plan: secondPlanName, now: createdAt.Add(11 * time.Minute), StructuralOnly: true, Hosts: resolver, - }); err != nil || remote.staged.PreviousRevision != "revision-1" { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: secondPlanName, now: createdAt.Add(11 * time.Minute), StructuralOnly: true, Hosts: resolver}); err != nil || remote.staged.PreviousRevision != "revision-1" { t.Fatalf("apply Pages same-tree manifest update previous=%q err=%v", remote.staged.PreviousRevision, err) } } @@ -791,7 +773,7 @@ func TestAutoGateRechecksExpiryBeforePublicationEffects(t *testing.T) { commitWorkspace(t, root, "add expiry fixture") createdAt := time.Date(2026, time.July, 25, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "expiring.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Minute, }); err != nil { t.Fatal(err) @@ -804,7 +786,7 @@ func TestAutoGateRechecksExpiryBeforePublicationEffects(t *testing.T) { } return value } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, StructuralOnly: true, clock: clock}); err == nil || !strings.Contains(err.Error(), "expired before publication effect") { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, StructuralOnly: true, clock: clock}); err == nil || !strings.Contains(err.Error(), "expired before publication effect") { t.Fatalf("delayed auto apply error = %v", err) } if _, err := os.Lstat(filepath.Join(root, "public", "pypi")); !os.IsNotExist(err) { @@ -837,10 +819,10 @@ func TestApprovalGateBlocksBeforeStageAndAcceptsBoundEvidence(t *testing.T) { commitWorkspace(t, root, "request approval") now := time.Now().UTC().Truncate(time.Second) planName := filepath.Join(root, "approval.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName, createdAt: now, GeneratedAt: now, ExpiresIn: time.Hour}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: now, GeneratedAt: now, ExpiresIn: time.Hour}); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: now.Add(time.Minute), StructuralOnly: true}); err == nil || !strings.Contains(err.Error(), "requires approval gate evidence") { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: now.Add(time.Minute), StructuralOnly: true}); err == nil || !strings.Contains(err.Error(), "requires approval gate evidence") { t.Fatalf("approval gate did not block apply: %v", err) } if _, err := os.Lstat(filepath.Join(root, "public", "python")); !errors.Is(err, os.ErrNotExist) { @@ -854,14 +836,14 @@ func TestApprovalGateBlocksBeforeStageAndAcceptsBoundEvidence(t *testing.T) { if err != nil || approved.PlanID == "" { t.Fatalf("approve plan result=%#v err=%v", approved, err) } - result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: now.Add(2 * time.Minute), StructuralOnly: true, Gates: gate.NewDefaultEvaluator(approvalName, nil), }) if err != nil || result.Applied != 1 { t.Fatalf("approved apply result=%#v err=%v", result, err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, StructuralOnly: true, }); err == nil || !strings.Contains(err.Error(), "requires approval gate evidence") { t.Fatalf("current gated publication bypassed approval: %v", err) @@ -878,10 +860,10 @@ func TestRenderStatusWritesDeterministicManagedSite(t *testing.T) { commitWorkspace(t, root, "publish status fixture") now := time.Date(2026, time.July, 25, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "render-plan.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName, createdAt: now, GeneratedAt: now, ExpiresIn: time.Hour}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: now, GeneratedAt: now, ExpiresIn: time.Hour}); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: now.Add(time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: now.Add(time.Minute), StructuralOnly: true}); err != nil { t.Fatal(err) } output := filepath.Join(t.TempDir(), "site") @@ -914,10 +896,10 @@ func TestMissingDeploymentReceiptForcesRecoverableReconciliation(t *testing.T) { commitWorkspace(t, root, "publish recovery fixture") now := time.Date(2026, time.July, 25, 1, 2, 3, 0, time.UTC) firstPlan := filepath.Join(root, "first-recovery.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: firstPlan, createdAt: now, GeneratedAt: now, ExpiresIn: time.Hour}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: firstPlan, createdAt: now, GeneratedAt: now, ExpiresIn: time.Hour}); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: firstPlan, now: now.Add(time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: firstPlan, now: now.Add(time.Minute), StructuralOnly: true}); err != nil { t.Fatal(err) } deployment := filepath.Join(root, "deployments", "pypi.json") @@ -932,7 +914,7 @@ func TestMissingDeploymentReceiptForcesRecoverableReconciliation(t *testing.T) { t.Fatalf("commit missing receipt: %v: %s", err, output) } secondPlan := filepath.Join(root, "second-recovery.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: secondPlan, createdAt: now.Add(2 * time.Hour), GeneratedAt: now.Add(2 * time.Hour), ExpiresIn: time.Hour}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: secondPlan, createdAt: now.Add(2 * time.Hour), GeneratedAt: now.Add(2 * time.Hour), ExpiresIn: time.Hour}); err != nil { t.Fatal(err) } planned, err := state.LoadPlan(secondPlan) @@ -942,7 +924,7 @@ func TestMissingDeploymentReceiptForcesRecoverableReconciliation(t *testing.T) { if planned.Payload.Repositories[0].Action != "update" { t.Fatalf("missing receipt produced action %q", planned.Payload.Repositories[0].Action) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: secondPlan, now: now.Add(2*time.Hour + time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: secondPlan, now: now.Add(2*time.Hour + time.Minute), StructuralOnly: true}); err != nil { t.Fatal(err) } if _, err := state.LoadDeployment(root, "pypi"); err != nil { @@ -1011,7 +993,7 @@ func TestWorkspaceFetchesMissingBlobFromSharedStore(t *testing.T) { commitWorkspace(t, root, "configure shared blobs") createdAt := time.Date(2026, time.July, 25, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "shared.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, Blobs: resolver, }); err != nil { t.Fatal(err) @@ -1065,14 +1047,14 @@ func TestApplyRejectsLockChangedAfterPlan(t *testing.T) { commitWorkspace(t, root, "add first chart") createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "reviewed.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}); err != nil { t.Fatal(err) } second := workspaceArtifact(t, root, "helm", "2.0.0") if _, err := AddArtifacts(AddArtifactsRequest{Root: root, Repository: "helm", Artifacts: []string{second}}); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true}); err == nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true}); err == nil { t.Fatal("expected changed lock to make plan stale") } } @@ -1086,7 +1068,7 @@ func TestLoadPlanRejectsTamperedPayload(t *testing.T) { } commitWorkspace(t, root, "add wheel") planName := filepath.Join(root, "reviewed.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName}); err != nil { t.Fatal(err) } content, err := os.ReadFile(planName) @@ -1120,7 +1102,7 @@ func TestApplyRejectsStructurallyInvalidRehashedPlan(t *testing.T) { } commitWorkspace(t, root, "add wheel") planName := filepath.Join(root, "reviewed.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName}); err != nil { t.Fatal(err) } plan, err := state.LoadPlan(planName) @@ -1136,7 +1118,7 @@ func TestApplyRejectsStructurallyInvalidRehashedPlan(t *testing.T) { if err := state.WritePlan(planName, plan); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, StructuralOnly: true}); err == nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, StructuralOnly: true}); err == nil { t.Fatal("expected structurally invalid rehashed plan to be rejected") } } @@ -1151,10 +1133,10 @@ func TestPublishedChartCannotChangeBytes(t *testing.T) { commitWorkspace(t, root, "add chart") createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "reviewed.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true}); err != nil { t.Fatal(err) } metadata := "apiVersion: v2\nname: snail-demo\nversion: 1.2.3\ndescription: changed bytes\n" @@ -1181,7 +1163,7 @@ func TestPlanRequiresCommittedAuthoritativeState(t *testing.T) { if _, err := AddArtifacts(AddArtifactsRequest{Root: root, Repository: "pypi", Artifacts: []string{artifact}}); err != nil { t.Fatal(err) } - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root}); err == nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root}); err == nil { t.Fatal("expected uncommitted manifest and lock to block planning") } } @@ -1214,7 +1196,7 @@ func TestPlanRejectsUntrackedCustomLock(t *testing.T) { t.Fatal(err) } commitGitPaths(t, root, "configure custom lock", ".gitignore", "snailmail.toml") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root}); err == nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root}); err == nil { t.Fatal("expected untracked custom lock to block planning") } } @@ -1241,12 +1223,12 @@ func TestWorkspaceSupportsNestedGitDirectory(t *testing.T) { commitWorkspace(t, root, "add nested wheel") createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "reviewed.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, }); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true, }); err != nil { t.Fatal(err) @@ -1283,7 +1265,7 @@ func TestPlanRejectsAssumeUnchangedAuthoritativeFile(t *testing.T) { if err := file.Close(); err != nil { t.Fatal(err) } - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root}); err == nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root}); err == nil { t.Fatal("expected hidden authoritative change to block planning") } } @@ -1307,12 +1289,12 @@ func TestWorkspaceUsesConfiguredGitIndex(t *testing.T) { t.Setenv("GIT_INDEX_FILE", customIndex) createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "reviewed.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, }); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true, }); err != nil { t.Fatal(err) @@ -1332,10 +1314,10 @@ func TestNoopPlanDoesNotWritePublicationLedger(t *testing.T) { commitWorkspace(t, root, "add wheel") createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) firstPlan := filepath.Join(root, "first.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: firstPlan, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: firstPlan, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: firstPlan, now: createdAt.Add(time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: firstPlan, now: createdAt.Add(time.Minute), StructuralOnly: true}); err != nil { t.Fatal(err) } headBefore, err := exec.Command("git", "-C", root, "rev-parse", "HEAD").Output() @@ -1349,14 +1331,14 @@ func TestNoopPlanDoesNotWritePublicationLedger(t *testing.T) { } secondPlan := filepath.Join(root, "second.json") secondCreatedAt := createdAt.Add(5 * time.Minute) - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: secondPlan, createdAt: secondCreatedAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}) + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: secondPlan, createdAt: secondCreatedAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}) if err != nil { t.Fatal(err) } if planned.Changes != 0 { t.Fatalf("planned %d changes, want no-op", planned.Changes) } - result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: secondPlan, now: secondCreatedAt.Add(time.Minute), StructuralOnly: true}) + result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: secondPlan, now: secondCreatedAt.Add(time.Minute), StructuralOnly: true}) if err != nil { t.Fatal(err) } @@ -1406,7 +1388,7 @@ func TestApplyRejectsForgedLedgerRetryCommit(t *testing.T) { commitWorkspace(t, root, "add chart") createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "reviewed.json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, }) if err != nil { @@ -1449,7 +1431,7 @@ func TestApplyRejectsForgedLedgerRetryCommit(t *testing.T) { t.Fatal(err) } commitGitPaths(t, root, "forged publication\n\nSnailmail-Plan: "+planned.PlanID, "publications/helm.jsonl") - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true, }); err == nil { t.Fatal("expected forged publication commit to be rejected") @@ -1476,7 +1458,7 @@ func TestLedgerCommitRejectsChangedIndex(t *testing.T) { commitWorkspace(t, root, "add wheel") createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "reviewed.json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, }) if err != nil { @@ -1535,7 +1517,7 @@ func TestApplyResumesAfterLedgerCommitBeforePublication(t *testing.T) { commitWorkspace(t, root, "add chart") createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "reviewed.json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour, }) if err != nil { @@ -1566,7 +1548,7 @@ func TestApplyResumesAfterLedgerCommitBeforePublication(t *testing.T) { if _, err := os.Lstat(filepath.Join(root, "public", "helm")); !os.IsNotExist(err) { t.Fatal("ledger-only transaction unexpectedly published the target") } - result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{ + result, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: planName, now: createdAt.Add(time.Minute), StructuralOnly: true, }) if err != nil { @@ -1645,10 +1627,10 @@ func TestLedgerCommitStagesAssumeUnchangedLedger(t *testing.T) { commitWorkspace(t, root, "add first wheel") createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) firstPlan := filepath.Join(root, "first.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: firstPlan, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: firstPlan, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}); err != nil { t.Fatal(err) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: firstPlan, now: createdAt.Add(time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: firstPlan, now: createdAt.Add(time.Minute), StructuralOnly: true}); err != nil { t.Fatal(err) } second := workspaceArtifact(t, root, "pypi", "2.0.0") @@ -1658,13 +1640,13 @@ func TestLedgerCommitStagesAssumeUnchangedLedger(t *testing.T) { commitWorkspace(t, root, "add second wheel") secondCreatedAt := createdAt.Add(5 * time.Minute) secondPlan := filepath.Join(root, "second.json") - if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: secondPlan, createdAt: secondCreatedAt, GeneratedAt: secondCreatedAt, ExpiresIn: time.Hour}); err != nil { + if _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: secondPlan, createdAt: secondCreatedAt, GeneratedAt: secondCreatedAt, ExpiresIn: time.Hour}); err != nil { t.Fatal(err) } if output, err := exec.Command("git", "-C", root, "update-index", "--assume-unchanged", "publications/pypi.jsonl").CombinedOutput(); err != nil { t.Fatalf("git update-index: %v: %s", err, output) } - if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Root: root, Plan: secondPlan, now: secondCreatedAt.Add(time.Minute), StructuralOnly: true}); err != nil { + if _, err := ApplyWorkspace(context.Background(), ApplyWorkspaceRequest{Hosts: localHosts(), Root: root, Plan: secondPlan, now: secondCreatedAt.Add(time.Minute), StructuralOnly: true}); err != nil { t.Fatal(err) } records, err := state.LoadLedger(root, "pypi") @@ -1686,7 +1668,7 @@ func TestLedgerCommitRestoresIndexWhenRefTransactionFails(t *testing.T) { commitWorkspace(t, root, "add chart") createdAt := time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC) planName := filepath.Join(root, "reviewed.json") - planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}) + planned, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{Hosts: localHosts(), Root: root, Output: planName, createdAt: createdAt, GeneratedAt: createdAt, ExpiresIn: time.Hour}) if err != nil { t.Fatal(err) } From ae88d7a7c403a871198d920c1a7a197ecb73a4e3 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 13:52:23 +0200 Subject: [PATCH 12/26] Put install instructions and the endpoint probe on the format --- engine/workspace.go | 16 +-- formats/conformance_test.go | 87 +++++++++++++ formats/format.go | 23 +++- formats/instructions.go | 244 ++++++++++++++++++++++-------------- internal/state/store.go | 6 +- 5 files changed, 262 insertions(+), 114 deletions(-) diff --git a/engine/workspace.go b/engine/workspace.go index f789a1c..c6b1321 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -1319,7 +1319,7 @@ func verifyEndpointClient(ctx context.Context, repository state.Repository, stag // installs from the exact bytes, but nothing checks that the host serves // them correctly until they are live. Refusing outright was worse — it made // configuring a preview site turn a working repository into a failing one. - if !endpointProbeExists(repository.Format) { + if selected, err := formats.For(repository.Format); err != nil || !selected.HasEndpointProbe() { _, err := verifyStaged(ctx, repository.Format, staged, request) return err } @@ -2654,17 +2654,3 @@ func (preparation *applyPreparation) prepareRepository(planned state.PlanReposit item.stageRoot, item.stage, item.stagedManifest = stage, stageOutput, stagedManifest return item, nil } - -// endpointProbeExists reports whether a format can be verified through a served -// endpoint rather than only against the tree that was staged. -// -// Kept beside the switch it describes, so the two are edited together. The -// support matrix declares which pairs a host publishes; this is the narrower -// question of whether a preview site can be exercised for one. -func endpointProbeExists(format string) bool { - switch format { - case "pypi", "deb", "raw": - return true - } - return false -} diff --git a/formats/conformance_test.go b/formats/conformance_test.go index 3cff815..e816415 100644 --- a/formats/conformance_test.go +++ b/formats/conformance_test.go @@ -281,3 +281,90 @@ func TestSigningFormatsNameTheirAlgorithm(t *testing.T) { } } } + +// A repository that has not said where it is served from gets no instructions. +// Every format has to refuse rather than write a URL it invented: a pasted +// command that points at nothing is worse than a page that says nothing, and it +// is the one mistake every one of these could make independently. +func TestNoFormatInventsAnEndpoint(t *testing.T) { + for _, format := range All() { + if steps := format.InstallSteps(Repository{Name: "packages"}); len(steps) != 0 { + t.Errorf("format %q wrote install steps for a repository with no endpoint: %q", + format.Name(), steps) + } + } +} + +// Where there is an endpoint, every format writes instructions that contain it. +// A format that returned nothing would leave a published listing with an empty +// install section and no explanation. +func TestEveryFormatWritesInstructionsAgainstItsEndpoint(t *testing.T) { + const endpoint = "https://packages.example/repo" + for _, format := range All() { + steps := format.InstallSteps(Repository{Name: "packages", Endpoint: endpoint}) + if len(steps) == 0 { + t.Errorf("format %q writes no install steps", format.Name()) + continue + } + if !strings.Contains(strings.Join(steps, "\n"), endpoint) { + t.Errorf("format %q instructions do not mention the endpoint: %q", format.Name(), steps) + } + } +} + +// An unsigned repository says so. The listing states signing plainly rather than +// leaving it to be inferred from an absence, and the instructions are the place +// a reader is about to act on it — apt needs [trusted=yes], apk needs +// --allow-untrusted, and both deserve a sentence saying why. +func TestASignableFormatSaysWhenARepositoryIsUnsigned(t *testing.T) { + const endpoint = "https://packages.example/repo" + for _, format := range All() { + if !format.ImplementsSigning() { + continue + } + steps := strings.Join(format.InstallSteps(Repository{Name: "packages", Endpoint: endpoint}), "\n") + if !strings.Contains(steps, "unsigned") { + t.Errorf("format %q does not say that an unsigned repository is unsigned: %q", format.Name(), steps) + } + } +} + +// A signed repository installs the key before it installs anything else. +func TestASignedRepositoryInstallsItsKeyFirst(t *testing.T) { + const endpoint = "https://packages.example/repo" + for _, format := range All() { + if !format.ImplementsSigning() { + continue + } + steps := strings.Join(format.InstallSteps(Repository{ + Name: "packages", Endpoint: endpoint, Signed: true, + Signing: &RepositorySigning{Fingerprint: "AABB", KeyPath: "keys/packages.gpg"}, + }), "\n") + if !strings.Contains(steps, "keys/packages.gpg") { + t.Errorf("format %q does not install its signing key: %q", format.Name(), steps) + } + if strings.Contains(steps, "unsigned") { + t.Errorf("format %q called a signed repository unsigned: %q", format.Name(), steps) + } + } +} + +// The endpoint probe is a fact about the ecosystem, and the engine skips the +// served-URL check for a format that reports false. Pinned so that a format +// claiming one is a deliberate change rather than a default. +func TestEndpointProbeSupport(t *testing.T) { + want := map[string]bool{ + "pypi": true, "deb": true, "raw": true, + "helm": false, "rpm": false, "apk": false, + } + for _, format := range All() { + expected, known := want[format.Name()] + if !known { + t.Errorf("format %q is new here; say whether a client can be run against a served URL", format.Name()) + continue + } + if format.HasEndpointProbe() != expected { + t.Errorf("format %q HasEndpointProbe = %v, want %v", format.Name(), format.HasEndpointProbe(), expected) + } + } +} diff --git a/formats/format.go b/formats/format.go index 4d81024..bd440ed 100644 --- a/formats/format.go +++ b/formats/format.go @@ -162,6 +162,17 @@ type Format interface { // CommitPaths are the files whose switch makes a new revision live, which // a host must publish last and together. CommitPaths(repository Repository) []string + // InstallSteps are the commands a person runs to consume a repository of + // this format, written against the URL it is served from. Empty where there + // is no URL: a guessed one would be worse than none. + InstallSteps(repository Repository) []string + // HasEndpointProbe reports whether a client of this format can be run + // against a served endpoint, as opposed to only against the built tree. + // + // A fact about the ecosystem rather than about a host: the support matrix + // declares which format-and-host pairs are published, and this is the + // narrower question of whether a served URL can be exercised at all. + HasEndpointProbe() bool // Build renders a deterministic file tree from the given artifacts. Build(blobs []domain.Blob, options BuildOptions) (domain.RepositoryArtifact, error) @@ -247,7 +258,7 @@ func (pypiFormat) Build(blobs []domain.Blob, options BuildOptions) (domain.Repos if err != nil { return domain.RepositoryArtifact{}, err } - return AppendListing(artifact, "pypi", options, blobs) + return AppendListing(artifact, pypiFormat{}, options, blobs) } type debFormat struct{} @@ -309,7 +320,7 @@ func (debFormat) Build(blobs []domain.Blob, options BuildOptions) (domain.Reposi if err != nil { return domain.RepositoryArtifact{}, err } - return AppendListing(artifact, "deb", options, blobs) + return AppendListing(artifact, debFormat{}, options, blobs) } type helmFormat struct{} @@ -350,7 +361,7 @@ func (helmFormat) Build(blobs []domain.Blob, options BuildOptions) (domain.Repos if err != nil { return domain.RepositoryArtifact{}, err } - return AppendListing(artifact, "helm", options, blobs) + return AppendListing(artifact, helmFormat{}, options, blobs) } // Compile-time proof that every registered value satisfies the interface. @@ -414,7 +425,7 @@ func (rawFormat) Build(blobs []domain.Blob, options BuildOptions) (domain.Reposi if err != nil { return domain.RepositoryArtifact{}, err } - return AppendListing(artifact, "raw", options, blobs) + return AppendListing(artifact, rawFormat{}, options, blobs) } // rpmFormat serves RPM packages through a yum/dnf repository. @@ -472,7 +483,7 @@ func (rpmFormat) Build(blobs []domain.Blob, options BuildOptions) (domain.Reposi if err != nil { return domain.RepositoryArtifact{}, err } - return AppendListing(artifact, "rpm", options, blobs) + return AppendListing(artifact, rpmFormat{}, options, blobs) } // apkFormat serves Alpine packages through an APKINDEX repository. @@ -527,5 +538,5 @@ func (apkFormat) Build(blobs []domain.Blob, options BuildOptions) (domain.Reposi if err != nil { return domain.RepositoryArtifact{}, err } - return AppendListing(artifact, "apk", options, blobs) + return AppendListing(artifact, apkFormat{}, options, blobs) } diff --git a/formats/instructions.go b/formats/instructions.go index 417802c..fdc5fb4 100644 --- a/formats/instructions.go +++ b/formats/instructions.go @@ -10,113 +10,146 @@ import ( "github.com/shellcell/snailmail/internal/listing" ) -// InstallSteps are the commands a person runs to consume a repository, written -// against the URL it is actually served from. +// Install instructions, one method per ecosystem. // -// They live here rather than in each format because they are the one thing on a -// listing that is about the reader rather than about the artifacts, and because -// a signed repository and an unsigned one need genuinely different instructions: -// the signed form installs a key first, and the unsigned form has to tell the -// client not to check — which is worth seeing written out before it is pasted. -func InstallSteps(format string, repository Repository) []string { - endpoint := strings.TrimRight(repository.Endpoint, "/") +// They live in this file rather than beside each format's Build because they are +// the one thing on a listing that is about the reader rather than about the +// artifacts, and because a signed repository and an unsigned one need genuinely +// different instructions: the signed form installs a key first, and the unsigned +// form has to tell the client not to check — which is worth seeing written out +// before it is pasted. +// +// Every one begins by refusing to guess. A repository published to a directory +// has no URL to install from, and an invented one would be worse than silence. + +func (pypiFormat) InstallSteps(repository Repository) []string { + endpoint := installEndpoint(repository) if endpoint == "" { - // A repository published to a directory has no URL to install from, and - // a guessed one would be worse than none. return nil } - switch format { - case "deb": - suite := defaulted(repository.Suite, "stable") - component := defaulted(repository.Component, "main") - if repository.Signing == nil { - return []string{ - "# This repository is unsigned; apt will not verify what it installs.", - "echo 'deb [trusted=yes] " + endpoint + " " + suite + " " + component + "' \\", - " | sudo tee /etc/apt/sources.list.d/" + listName(repository) + ".list", - "sudo apt-get update && sudo apt-get install ", - } - } - keyring := "/usr/share/keyrings/" + listName(repository) + ".gpg" + // `python -m pip` rather than `pip`, which resolves to whichever + // interpreter's pip happens to be first on PATH; and the URL is quoted + // because it is pasted into a shell. + return []string{"python -m pip install --index-url '" + endpoint + "/simple' "} +} + +func (debFormat) InstallSteps(repository Repository) []string { + endpoint := installEndpoint(repository) + if endpoint == "" { + return nil + } + suite := defaulted(repository.Suite, "stable") + component := defaulted(repository.Component, "main") + if repository.Signing == nil { return []string{ - "curl -fsSL " + endpoint + "/" + repository.Signing.KeyPath + " \\", - " | sudo tee " + keyring + " > /dev/null", - "echo 'deb [signed-by=" + keyring + "] " + endpoint + " " + suite + " " + component + "' \\", + "# This repository is unsigned; apt will not verify what it installs.", + "echo 'deb [trusted=yes] " + endpoint + " " + suite + " " + component + "' \\", " | sudo tee /etc/apt/sources.list.d/" + listName(repository) + ".list", "sudo apt-get update && sudo apt-get install ", } - case "rpm": - lines := []string{ - "sudo tee /etc/yum.repos.d/" + listName(repository) + ".repo > /dev/null <<'REPO'", - "[" + listName(repository) + "]", - "name=" + listName(repository), - "baseurl=" + endpoint, - "enabled=1", - // gpgcheck covers signatures inside each package, which are made by - // whoever built it rather than by this repository. - "gpgcheck=0", - } - if repository.Signing == nil { - lines = append(lines, "repo_gpgcheck=0", "REPO", "sudo dnf install ") - return append([]string{"# This repository is unsigned; nothing verifies its metadata."}, lines...) - } - lines = append(lines, - "repo_gpgcheck=1", - "gpgkey="+endpoint+"/"+repository.Signing.KeyPath, - "REPO", - "sudo dnf install ") - return lines - case "apk": - if repository.Signing == nil { - return []string{ - "# This repository is unsigned; --allow-untrusted disables the check.", - "echo " + endpoint + " | sudo tee -a /etc/apk/repositories", - "sudo apk add --allow-untrusted ", - } - } - // apk finds the key by filename alone, so it must land under exactly the - // name the index names. + } + keyring := "/usr/share/keyrings/" + listName(repository) + ".gpg" + return []string{ + "curl -fsSL " + endpoint + "/" + repository.Signing.KeyPath + " \\", + " | sudo tee " + keyring + " > /dev/null", + "echo 'deb [signed-by=" + keyring + "] " + endpoint + " " + suite + " " + component + "' \\", + " | sudo tee /etc/apt/sources.list.d/" + listName(repository) + ".list", + "sudo apt-get update && sudo apt-get install ", + } +} + +func (rpmFormat) InstallSteps(repository Repository) []string { + endpoint := installEndpoint(repository) + if endpoint == "" { + return nil + } + lines := []string{ + "sudo tee /etc/yum.repos.d/" + listName(repository) + ".repo > /dev/null <<'REPO'", + "[" + listName(repository) + "]", + "name=" + listName(repository), + "baseurl=" + endpoint, + "enabled=1", + // gpgcheck covers signatures inside each package, which are made by + // whoever built it rather than by this repository. + "gpgcheck=0", + } + if repository.Signing == nil { + lines = append(lines, "repo_gpgcheck=0", "REPO", "sudo dnf install ") + return append([]string{"# This repository is unsigned; nothing verifies its metadata."}, lines...) + } + lines = append(lines, + "repo_gpgcheck=1", + "gpgkey="+endpoint+"/"+repository.Signing.KeyPath, + "REPO", + "sudo dnf install ") + return lines +} + +func (apkFormat) InstallSteps(repository Repository) []string { + endpoint := installEndpoint(repository) + if endpoint == "" { + return nil + } + if repository.Signing == nil { return []string{ - "sudo curl -fsSL -o /etc/apk/keys/" + path.Base(repository.Signing.KeyPath) + " \\", - " " + endpoint + "/" + repository.Signing.KeyPath, + "# This repository is unsigned; --allow-untrusted disables the check.", "echo " + endpoint + " | sudo tee -a /etc/apk/repositories", - "sudo apk add ", + "sudo apk add --allow-untrusted ", } - case "helm": - name := listName(repository) - if repository.Signing == nil { - return []string{ - "# This repository is unsigned; nothing verifies the charts you install.", - "helm repo add " + name + " " + endpoint, - "helm repo update", - "helm install " + name + "/", - } - } - // helm reads a binary OpenPGP keyring from a file it is pointed at, - // rather than a system trust store, so the key is downloaded and named - // on the command that uses it. - keyring := "~/.snailmail/" + name + ".gpg" + } + // apk finds the key by filename alone, so it must land under exactly the + // name the index names. + return []string{ + "sudo curl -fsSL -o /etc/apk/keys/" + path.Base(repository.Signing.KeyPath) + " \\", + " " + endpoint + "/" + repository.Signing.KeyPath, + "echo " + endpoint + " | sudo tee -a /etc/apk/repositories", + "sudo apk add ", + } +} + +func (helmFormat) InstallSteps(repository Repository) []string { + endpoint := installEndpoint(repository) + if endpoint == "" { + return nil + } + name := listName(repository) + if repository.Signing == nil { return []string{ - "mkdir -p ~/.snailmail", - "curl -fsSL " + endpoint + "/" + repository.Signing.KeyPath + " -o " + keyring, + "# This repository is unsigned; nothing verifies the charts you install.", "helm repo add " + name + " " + endpoint, "helm repo update", - "helm install " + name + "/ --verify --keyring " + keyring, - } - case "pypi": - // `python -m pip` rather than `pip`, which resolves to whichever - // interpreter's pip happens to be first on PATH; and the URL is quoted - // because it is pasted into a shell. - return []string{"python -m pip install --index-url '" + endpoint + "/simple' "} - case "raw": - return []string{ - "curl -LO " + endpoint + "///", - "curl -LO " + endpoint + "/SHA256SUMS", - "sha256sum -c --ignore-missing SHA256SUMS", + "helm install " + name + "/", } - default: + } + // helm reads a binary OpenPGP keyring from a file it is pointed at, + // rather than a system trust store, so the key is downloaded and named + // on the command that uses it. + keyring := "~/.snailmail/" + name + ".gpg" + return []string{ + "mkdir -p ~/.snailmail", + "curl -fsSL " + endpoint + "/" + repository.Signing.KeyPath + " -o " + keyring, + "helm repo add " + name + " " + endpoint, + "helm repo update", + "helm install " + name + "/ --verify --keyring " + keyring, + } +} + +func (rawFormat) InstallSteps(repository Repository) []string { + endpoint := installEndpoint(repository) + if endpoint == "" { return nil } + return []string{ + "curl -LO " + endpoint + "///", + "curl -LO " + endpoint + "/SHA256SUMS", + "sha256sum -c --ignore-missing SHA256SUMS", + } +} + +// installEndpoint is the URL instructions are written against, or empty where +// the repository has not said where it is served from. +func installEndpoint(repository Repository) string { + return strings.TrimRight(repository.Endpoint, "/") } // listName is the repository's own name where it has one, and a neutral default @@ -149,7 +182,8 @@ func defaulted(value, fallback string) string { // The artifacts it lists are the files carrying blob content — the packages — // found by matching each blob's digest to where the render placed it. Index // files are generated and are not what a visitor came for. -func AppendListing(artifact domain.RepositoryArtifact, format string, options BuildOptions, blobs []domain.Blob) (domain.RepositoryArtifact, error) { +func AppendListing(artifact domain.RepositoryArtifact, selected Format, options BuildOptions, blobs []domain.Blob) (domain.RepositoryArtifact, error) { + format := selected.Name() placed := make(map[string]string, len(artifact.Files)) for _, file := range artifact.Files { if file.BlobSHA256 != "" { @@ -174,7 +208,7 @@ func AppendListing(artifact domain.RepositoryArtifact, format string, options Bu } page := listing.Page{ Repository: listName(options.Repository), Format: format, - Endpoint: options.Repository.Endpoint, Install: InstallSteps(format, options.Repository), + Endpoint: options.Repository.Endpoint, Install: selected.InstallSteps(options.Repository), Artifacts: artifacts, } if signing := options.Repository.Signing; signing != nil { @@ -188,3 +222,29 @@ func AppendListing(artifact domain.RepositoryArtifact, format string, options Bu artifact.Files = files return artifact, nil } + +// Whether a real client of this format can be pointed at a served URL, as +// opposed to only at the built tree. +// +// This used to be a switch in the engine, kept beside itself with a comment +// saying the two had to be edited together — which is the shape of a fact that +// belongs on the thing it is a fact about. It is not the same question as the +// support matrix: that declares which format-and-host pairs are published, and +// this is whether a served URL can be exercised for one at all. + +// pip takes --index-url and installs over HTTP. +func (pypiFormat) HasEndpointProbe() bool { return true } + +// apt reads a sources.list entry pointing at a URL. +func (debFormat) HasEndpointProbe() bool { return true } + +// A raw consumer fetches a URL and checks it against SHA256SUMS, which is +// exactly what a served-byte probe does. +func (rawFormat) HasEndpointProbe() bool { return true } + +// The remaining three are verified against the staged tree instead. Their +// clients can read a URL, but no endpoint probe is implemented for them yet, and +// claiming one would have the engine skip the check it does have. +func (helmFormat) HasEndpointProbe() bool { return false } +func (rpmFormat) HasEndpointProbe() bool { return false } +func (apkFormat) HasEndpointProbe() bool { return false } diff --git a/internal/state/store.go b/internal/state/store.go index 64c0fe8..1533816 100644 --- a/internal/state/store.go +++ b/internal/state/store.go @@ -253,7 +253,11 @@ func installDocumentContent(name string, repository Repository, keys map[string] // write rather than commands to run, and it carries Signed-By. return debInstallDocument(name, repository) } - steps := formats.InstallSteps(repository.Format, installRepositoryView(name, repository, keys)) + selected, err := formats.For(repository.Format) + if err != nil { + return []byte("# Install from " + name + "\n\nThis repository publishes no install instructions.\n") + } + steps := selected.InstallSteps(installRepositoryView(name, repository, keys)) if len(steps) == 0 { return []byte("# Install from " + name + "\n\nThis repository publishes no install instructions.\n") } From 86b330083730884186a2706f6ea9b8c6e89cac0c Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 14:09:10 +0200 Subject: [PATCH 13/26] Verify a rendered tree through the format that produced it --- formats/conformance_test.go | 30 ++ formats/format.go | 24 ++ formats/verify.go | 438 +++++++++++++++++++++ internal/app/materialize_hardening_test.go | 7 +- internal/app/verify.go | 425 +------------------- 5 files changed, 509 insertions(+), 415 deletions(-) create mode 100644 formats/verify.go diff --git a/formats/conformance_test.go b/formats/conformance_test.go index e816415..b1f48ed 100644 --- a/formats/conformance_test.go +++ b/formats/conformance_test.go @@ -368,3 +368,33 @@ func TestEndpointProbeSupport(t *testing.T) { } } } + +// Every registered format answers the structure check, and a tree it has not +// seen is refused rather than accepted with nothing checked. +// +// The check used to be a chain of comparisons in internal/app that a format had +// to be added to. A tree whose identity matched none of them fell through with +// no structure verification and no error, which is the wrong answer twice over: +// a tree from a newer snailmail cannot be verified by this one, and one with a +// forged identity should not be able to opt out of the check by claiming to be +// something else. +func TestAnUnknownFormatIdentityIsRefused(t *testing.T) { + for _, identity := range []string{"", "test", "pypi", "pypi/v99", "deb/v1/extra"} { + if _, err := ForID(identity); err == nil { + t.Errorf("format identity %q was accepted", identity) + } + } +} + +func TestEveryRegisteredFormatIsReachableByItsIdentity(t *testing.T) { + for _, format := range All() { + found, err := ForID(format.ID()) + if err != nil { + t.Errorf("format %q is not reachable by its identity %q: %v", format.Name(), format.ID(), err) + continue + } + if found.Name() != format.Name() { + t.Errorf("identity %q resolved to %q, want %q", format.ID(), found.Name(), format.Name()) + } + } +} diff --git a/formats/format.go b/formats/format.go index bd440ed..725edd1 100644 --- a/formats/format.go +++ b/formats/format.go @@ -27,6 +27,7 @@ import ( "github.com/shellcell/snailmail/formats/pypi" "github.com/shellcell/snailmail/formats/raw" "github.com/shellcell/snailmail/formats/rpm" + "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/domain" "github.com/shellcell/snailmail/signer" ) @@ -166,6 +167,16 @@ type Format interface { // this format, written against the URL it is served from. Empty where there // is no URL: a guessed one would be worse than none. InstallSteps(repository Repository) []string + // VerifyStructure checks that a rendered tree of this format is internally + // consistent: that its index names exactly the artifacts present, that the + // digests in it are the digests of the bytes beside it, and that + // re-rendering those artifacts reproduces the index. It returns the blobs it + // recovered from the tree. + // + // It reads the filesystem and nothing else. Client verification, which runs + // a real client in a container, is an effect and is dispatched at the engine + // layer instead. + VerifyStructure(root string, manifest buildgraph.RepositoryManifest) ([]domain.Blob, error) // HasEndpointProbe reports whether a client of this format can be run // against a served endpoint, as opposed to only against the built tree. // @@ -196,6 +207,19 @@ func For(name string) (Format, error) { return format, nil } +// ForID returns the format whose versioned identity is id, which is what a +// generated repository manifest records — "pypi/v1" rather than "pypi". A tree +// says which rules produced it, so verifying one starts from the identity in it +// rather than from a name supplied alongside. +func ForID(id string) (Format, error) { + for _, format := range registry { + if format.ID() == id { + return format, nil + } + } + return nil, fmt.Errorf("unsupported repository format identity %q", id) +} + // Supported reports whether a format name is registered. func Supported(name string) bool { _, known := registry[name] diff --git a/formats/verify.go b/formats/verify.go new file mode 100644 index 0000000..43467b7 --- /dev/null +++ b/formats/verify.go @@ -0,0 +1,438 @@ +package formats + +// Verification of a rendered tree, one method per ecosystem. +// +// This is the check that a published tree is internally consistent: that its +// index names exactly the artifacts present, that the digests in it are the +// digests of the bytes beside it, and that re-rendering the artifacts produces +// the same index. It is what a host runs before sending a stage to the far +// side, and what `verify --structural-only` runs on its own. +// +// It belongs here for the same reason Build does: it is ecosystem knowledge, and +// it was a four-arm switch in internal/app that a new format had to be added to. +// It reads the filesystem and nothing else — no network, no clock, no container +// — so it stays on the side of the boundary ARCHITECTURE §2 draws around +// effects, and the client verification that does run containers stays out. +// +// rpm and apk have no structure check yet; theirs is the tree digest and the +// real client alone. + +import ( + "crypto/md5" + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "reflect" + "strings" + "time" + + "github.com/shellcell/snailmail/formats/deb" + "github.com/shellcell/snailmail/formats/helm" + "github.com/shellcell/snailmail/formats/pypi" + "github.com/shellcell/snailmail/formats/raw" + "github.com/shellcell/snailmail/internal/buildgraph" + "github.com/shellcell/snailmail/internal/domain" + "github.com/shellcell/snailmail/internal/factscache" + "github.com/shellcell/snailmail/internal/listing" + openpgpsigner "github.com/shellcell/snailmail/signer/openpgp" +) + +// A format with no structure check of its own. The tree digest still covers +// every byte, and a real client still installs from it. +func (rpmFormat) VerifyStructure(string, buildgraph.RepositoryManifest) ([]domain.Blob, error) { + return nil, nil +} + +func (apkFormat) VerifyStructure(string, buildgraph.RepositoryManifest) ([]domain.Blob, error) { + return nil, nil +} + +func (pypiFormat) VerifyStructure(root string, manifest buildgraph.RepositoryManifest) ([]domain.Blob, error) { + var blobs []domain.Blob + for _, file := range manifest.Files { + if file.Path == ".nojekyll" { + if file.Size != 0 { + return nil, errors.New("PyPI .nojekyll marker must be empty") + } + continue + } + // The browsable page is written for people, not for a client, and is + // covered by the tree digest like everything else. + if file.Path == listing.Filename { + continue + } + if strings.HasPrefix(file.Path, "simple/") { + if path.Base(file.Path) != "index.html" { + return nil, fmt.Errorf("unexpected PyPI index path %q", file.Path) + } + continue + } + parts := strings.Split(file.Path, "/") + if len(parts) != 3 || parts[0] != "packages" || parts[1] != file.SHA256 || !pypi.IsDistributionFilename(parts[2]) { + return nil, fmt.Errorf("unexpected PyPI repository path %q", file.Path) + } + facts, cached := factscache.Lookup(pypi.FormatID, file.SHA256) + if !cached { + name := filepath.Join(root, filepath.FromSlash(file.Path)) + packageFile, err := os.Open(name) + if err != nil { + return nil, fmt.Errorf("open PyPI package %q: %w", file.Path, err) + } + var inspectErr error + facts, inspectErr = pypi.Inspect(parts[2], packageFile, file.Size) + closeErr := packageFile.Close() + if inspectErr != nil { + return nil, inspectErr + } + if closeErr != nil { + return nil, fmt.Errorf("close PyPI package %q: %w", file.Path, closeErr) + } + factscache.Store(pypi.FormatID, file.SHA256, facts) + } + blobs = append(blobs, domain.Blob{Filename: parts[2], Size: file.Size, SHA256: file.SHA256, Facts: facts}) + } + expectedArtifact, err := pypi.Build(blobs) + if err != nil { + return nil, fmt.Errorf("rebuild PyPI structure: %w", err) + } + generatedAt, err := time.Parse(time.RFC3339, manifest.GeneratedAt) + if err != nil { + return nil, err + } + _, expectedManifest, err := buildgraph.Finalize(expectedArtifact, generatedAt) + if err != nil { + return nil, fmt.Errorf("finalize expected PyPI structure: %w", err) + } + expectedManifest.SchemaVersion = manifest.SchemaVersion + // The rebuild cannot reproduce the browsable page: it is written from the + // repository name, endpoint and signing key, which do not survive into the + // published tree. The tree digest still covers it. + expectedManifest.Files = withoutListing(expectedManifest.Files) + manifest.Files = withoutListing(manifest.Files) + // The tree digest covers the page too, so it cannot match a rebuild that + // omits it. What it protects is checked where it belongs: the deployment + // record is matched against the digest of the tree that was published. + expectedManifest.TreeSHA256, manifest.TreeSHA256 = "", "" + if !reflect.DeepEqual(expectedManifest, manifest) { + return nil, errors.New("PyPI indexes or verification metadata do not match package bytes") + } + return blobs, nil +} + +func (debFormat) VerifyStructure(root string, manifest buildgraph.RepositoryManifest) ([]domain.Blob, error) { + var blobs []domain.Blob + for _, file := range manifest.Files { + // The browsable page is written for people, not for a client, and is + // covered by the tree digest like everything else. + if file.Path == listing.Filename { + continue + } + if strings.HasPrefix(file.Path, "dists/") || (manifest.Install.SigningKeyPath != "" && file.Path == manifest.Install.SigningKeyPath) { + continue + } + if !strings.HasPrefix(file.Path, "pool/") || !deb.IsPackageFilename(path.Base(file.Path)) { + return nil, fmt.Errorf("unexpected Debian repository path %q", file.Path) + } + name := filepath.Join(root, filepath.FromSlash(file.Path)) + facts, cached := factscache.Lookup(deb.FormatID, file.SHA256) + if !cached { + packageFile, err := os.Open(name) + if err != nil { + return nil, fmt.Errorf("open Debian package %q: %w", file.Path, err) + } + var inspectErr error + facts, inspectErr = deb.Inspect(path.Base(file.Path), packageFile, file.Size) + closeErr := packageFile.Close() + if inspectErr != nil { + return nil, inspectErr + } + if closeErr != nil { + return nil, fmt.Errorf("close Debian package %q: %w", file.Path, closeErr) + } + factscache.Store(deb.FormatID, file.SHA256, facts) + } + md5Value, sha1Value, err := legacyChecksums(name) + if err != nil { + return nil, err + } + blobs = append(blobs, domain.Blob{ + Filename: path.Base(file.Path), + Size: file.Size, + MD5: md5Value, + SHA1: sha1Value, + SHA256: file.SHA256, + Facts: facts, + }) + } + generatedAt, err := time.Parse(time.RFC3339, manifest.GeneratedAt) + if err != nil { + return nil, err + } + expectedArtifact, err := deb.Build(blobs, deb.BuildOptions{ + Suite: manifest.Install.Suite, + Component: manifest.Install.Component, + Architectures: manifest.Install.Architectures, + GeneratedAt: generatedAt, + }) + if err != nil { + return nil, fmt.Errorf("rebuild Debian structure: %w", err) + } + if manifest.Install.SigningKeyPath != "" { + if len(manifest.Signatures) != 2 || manifest.Install.SigningFingerprint == "" || path.IsAbs(manifest.Install.SigningKeyPath) || + path.Clean(manifest.Install.SigningKeyPath) != manifest.Install.SigningKeyPath || !strings.HasPrefix(manifest.Install.SigningKeyPath, "keys/") { + return nil, errors.New("signed Debian repository has incomplete signature metadata") + } + keyringContent, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(manifest.Install.SigningKeyPath))) + if err != nil { + return nil, fmt.Errorf("read Debian public signing key: %w", err) + } + inRelease, err := os.ReadFile(filepath.Join(root, "dists", manifest.Install.Suite, "InRelease")) + if err != nil { + return nil, fmt.Errorf("read Debian InRelease: %w", err) + } + releaseGPG, err := os.ReadFile(filepath.Join(root, "dists", manifest.Install.Suite, "Release.gpg")) + if err != nil { + return nil, fmt.Errorf("read Debian Release.gpg: %w", err) + } + signatureTime, err := time.Parse(time.RFC3339, manifest.Signatures[0].CreatedAt) + if err != nil || manifest.Signatures[1].CreatedAt != manifest.Signatures[0].CreatedAt { + return nil, errors.New("Debian signature metadata has inconsistent creation times") + } + trustedFingerprints := append([]string(nil), manifest.Install.TrustedSigningFingerprints...) + if len(trustedFingerprints) == 0 { + trustedFingerprints = []string{manifest.Install.SigningFingerprint} + } + activePublic, err := openpgpsigner.ExtractPublicKey(keyringContent, manifest.Install.SigningFingerprint) + if err != nil { + return nil, fmt.Errorf("extract active Debian signing key: %w", err) + } + expectedArtifact, err = deb.ApplySigning(expectedArtifact, manifest.Install.Suite, deb.SigningMaterial{ + Fingerprint: manifest.Install.SigningFingerprint, PublicKey: activePublic, + KeyringPath: manifest.Install.SigningKeyPath, PublicKeyring: keyringContent, TrustedFingerprints: trustedFingerprints, + SignatureTime: signatureTime, InRelease: inRelease, ReleaseGPG: releaseGPG, + }) + if err != nil { + return nil, fmt.Errorf("verify Debian signatures: %w", err) + } + } else if len(manifest.Signatures) != 0 { + return nil, errors.New("unsigned Debian repository contains signature metadata") + } + _, expectedManifest, err := buildgraph.Finalize(expectedArtifact, generatedAt) + if err != nil { + return nil, fmt.Errorf("finalize expected Debian structure: %w", err) + } + expectedManifest.SchemaVersion = manifest.SchemaVersion + if manifest.SchemaVersion < 3 { + expectedManifest.Install.TrustedSigningFingerprints = nil + } + // The rebuild cannot reproduce the browsable page: it is written from the + // repository name, endpoint and signing key, which do not survive into the + // published tree. The tree digest still covers it. + expectedManifest.Files = withoutListing(expectedManifest.Files) + manifest.Files = withoutListing(manifest.Files) + // The tree digest covers the page too, so it cannot match a rebuild that + // omits it. What it protects is checked where it belongs: the deployment + // record is matched against the digest of the tree that was published. + expectedManifest.TreeSHA256, manifest.TreeSHA256 = "", "" + if !reflect.DeepEqual(expectedManifest, manifest) { + return nil, errors.New("Debian indexes or verification metadata do not match package bytes") + } + return blobs, nil +} + +func (helmFormat) VerifyStructure(root string, manifest buildgraph.RepositoryManifest) ([]domain.Blob, error) { + var blobs []domain.Blob + for _, file := range manifest.Files { + // The browsable page is written for people, not for a client, and is + // covered by the tree digest like everything else. + if file.Path == listing.Filename { + continue + } + if file.Path == "index.yaml" { + continue + } + // Signing material is checked below by rebuilding it, the same way the + // Debian path does: the published key and each chart's provenance are + // re-applied to the rebuilt tree, which verifies every signature. + if strings.HasPrefix(file.Path, "keys/") || strings.HasSuffix(file.Path, helm.ProvenanceSuffix) { + continue + } + parts := strings.Split(file.Path, "/") + if len(parts) != 3 || parts[0] != "charts" || parts[1] != file.SHA256 || !helm.IsChartFilename(parts[2]) { + return nil, fmt.Errorf("unexpected Helm repository path %q", file.Path) + } + facts, cached := factscache.Lookup(helm.FormatID, file.SHA256) + if !cached { + name := filepath.Join(root, filepath.FromSlash(file.Path)) + chartFile, err := os.Open(name) + if err != nil { + return nil, fmt.Errorf("open Helm chart %q: %w", file.Path, err) + } + var inspectErr error + facts, inspectErr = helm.Inspect(parts[2], chartFile, file.Size) + closeErr := chartFile.Close() + if inspectErr != nil { + return nil, inspectErr + } + if closeErr != nil { + return nil, fmt.Errorf("close Helm chart %q: %w", file.Path, closeErr) + } + factscache.Store(helm.FormatID, file.SHA256, facts) + } + blobs = append(blobs, domain.Blob{Filename: parts[2], Size: file.Size, SHA256: file.SHA256, Facts: facts}) + } + generatedAt, err := time.Parse(time.RFC3339, manifest.GeneratedAt) + if err != nil { + return nil, err + } + expectedArtifact, err := helm.Build(blobs, helm.BuildOptions{GeneratedAt: generatedAt}) + if err != nil { + return nil, fmt.Errorf("rebuild Helm structure: %w", err) + } + if manifest.Install.SigningKeyPath != "" { + if len(manifest.Signatures) == 0 || manifest.Install.SigningFingerprint == "" || path.IsAbs(manifest.Install.SigningKeyPath) || + path.Clean(manifest.Install.SigningKeyPath) != manifest.Install.SigningKeyPath || !strings.HasPrefix(manifest.Install.SigningKeyPath, "keys/") { + return nil, errors.New("signed Helm repository has incomplete signature metadata") + } + keyring, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(manifest.Install.SigningKeyPath))) + if err != nil { + return nil, fmt.Errorf("read Helm public signing key: %w", err) + } + provenance := make(map[string][]byte, len(manifest.Signatures)) + signatureTime, err := time.Parse(time.RFC3339, manifest.Signatures[0].CreatedAt) + if err != nil { + return nil, errors.New("Helm signature metadata has an invalid creation time") + } + for _, signature := range manifest.Signatures { + if signature.CreatedAt != manifest.Signatures[0].CreatedAt { + return nil, errors.New("Helm signature metadata has inconsistent creation times") + } + if !strings.HasSuffix(signature.Path, helm.ProvenanceSuffix) { + return nil, fmt.Errorf("Helm signature at %q is not a provenance file", signature.Path) + } + content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(signature.Path))) + if err != nil { + return nil, fmt.Errorf("read Helm provenance %q: %w", signature.Path, err) + } + provenance[strings.TrimSuffix(signature.Path, helm.ProvenanceSuffix)] = content + } + activePublic, err := openpgpsigner.ExtractPublicKey(keyring, manifest.Install.SigningFingerprint) + if err != nil { + return nil, fmt.Errorf("extract active Helm signing key: %w", err) + } + expectedArtifact, err = helm.ApplySigning(expectedArtifact, helm.SigningMaterial{ + Fingerprint: manifest.Install.SigningFingerprint, PublicKey: activePublic, + PublicKeyring: keyring, KeyringPath: manifest.Install.SigningKeyPath, + SignatureTime: signatureTime, Provenance: provenance, + }) + if err != nil { + return nil, fmt.Errorf("verify Helm signatures: %w", err) + } + } else if len(manifest.Signatures) != 0 { + return nil, errors.New("unsigned Helm repository contains signature metadata") + } + _, expectedManifest, err := buildgraph.Finalize(expectedArtifact, generatedAt) + if err != nil { + return nil, fmt.Errorf("finalize expected Helm structure: %w", err) + } + expectedManifest.SchemaVersion = manifest.SchemaVersion + // The rebuild cannot reproduce the browsable page: it is written from the + // repository name, endpoint and signing key, which do not survive into the + // published tree. The tree digest still covers it. + expectedManifest.Files = withoutListing(expectedManifest.Files) + manifest.Files = withoutListing(manifest.Files) + // The tree digest covers the page too, so it cannot match a rebuild that + // omits it. What it protects is checked where it belongs: the deployment + // record is matched against the digest of the tree that was published. + expectedManifest.TreeSHA256, manifest.TreeSHA256 = "", "" + if !reflect.DeepEqual(expectedManifest, manifest) { + return nil, errors.New("Helm index or verification metadata does not match chart bytes") + } + return blobs, nil +} + +// verifyRawStructure recomputes a raw repository from its published tree. +// +// Identity is read from the path rather than the filename, which is what makes +// an operator-supplied name verifiable after publication: the build put it +// there, so re-deriving it needs no access to the flags used at ingest. +func (rawFormat) VerifyStructure(root string, manifest buildgraph.RepositoryManifest) ([]domain.Blob, error) { + var blobs []domain.Blob + for _, file := range manifest.Files { + if file.Path == "SHA256SUMS" || file.Path == "index.html" { + continue + } + parts := strings.Split(file.Path, "/") + if len(parts) != 3 { + return nil, fmt.Errorf("unexpected raw repository path %q", file.Path) + } + name, version, filename := parts[0], parts[1], parts[2] + if !raw.IsArtifactFilename(filename) { + return nil, fmt.Errorf("unexpected raw artifact filename %q", file.Path) + } + facts, err := raw.FactsFor(name, version, filename) + if err != nil { + return nil, fmt.Errorf("raw artifact %q: %w", file.Path, err) + } + blobs = append(blobs, domain.Blob{Filename: filename, Size: file.Size, SHA256: file.SHA256, Facts: facts}) + } + generatedAt, err := time.Parse(time.RFC3339, manifest.GeneratedAt) + if err != nil { + return nil, err + } + expectedArtifact, err := raw.Build(blobs, raw.BuildOptions{GeneratedAt: generatedAt}) + if err != nil { + return nil, fmt.Errorf("rebuild raw structure: %w", err) + } + _, expectedManifest, err := buildgraph.Finalize(expectedArtifact, generatedAt) + if err != nil { + return nil, fmt.Errorf("finalize expected raw structure: %w", err) + } + // The browsable page is left out of the comparison. It is written from the + // repository's name, endpoint and signing key, none of which survive into + // the published tree, so it cannot be recomputed from what is there. What + // this check exists for is the machine-readable side: that every artifact is + // where the checksums say and nothing else is present. The page is still + // covered by the tree digest, which the deployment record is matched + // against, so altering it does not go unnoticed. + if !reflect.DeepEqual(withoutListing(expectedManifest.Files), withoutListing(manifest.Files)) { + return nil, errors.New("raw checksums do not match the published artifacts") + } + return blobs, nil +} + +// withoutListing drops the human-readable page from a file set, leaving what a +// client reads. +func withoutListing(files []buildgraph.ManifestFile) []buildgraph.ManifestFile { + kept := make([]buildgraph.ManifestFile, 0, len(files)) + for _, file := range files { + if file.Path == listing.Filename { + continue + } + kept = append(kept, file) + } + return kept +} + +func legacyChecksums(name string) (string, string, error) { + file, err := os.Open(name) + if err != nil { + return "", "", err + } + md5Hash := md5.New() + sha1Hash := sha1.New() + _, readErr := io.Copy(io.MultiWriter(md5Hash, sha1Hash), file) + closeErr := file.Close() + if readErr != nil { + return "", "", readErr + } + if closeErr != nil { + return "", "", closeErr + } + return hex.EncodeToString(md5Hash.Sum(nil)), hex.EncodeToString(sha1Hash.Sum(nil)), nil +} diff --git a/internal/app/materialize_hardening_test.go b/internal/app/materialize_hardening_test.go index e8e1059..7b0768f 100644 --- a/internal/app/materialize_hardening_test.go +++ b/internal/app/materialize_hardening_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/shellcell/snailmail/formats/apk" "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/domain" ) @@ -125,8 +126,12 @@ func TestSnapshotVerifiedManifestRejectsChangedMeaning(t *testing.T) { func writeManagedTestRepository(t *testing.T, output, content string) buildgraph.RepositoryManifest { t.Helper() + // A real format identity, because verification refuses one it does not + // recognise rather than silently skipping the structure check for it. apk is + // the one that asks nothing of a tree's shape, which is what this fixture + // wants: the subject here is the symlink exchange, not an ecosystem's rules. artifact, manifest, err := buildgraph.Finalize(domain.RepositoryArtifact{ - Format: "test", + Format: apk.FormatID, Files: []domain.File{{Path: "index.txt", Content: []byte(content)}}, }, time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC)) if err != nil { diff --git a/internal/app/verify.go b/internal/app/verify.go index 49d6acf..8a10900 100644 --- a/internal/app/verify.go +++ b/internal/app/verify.go @@ -3,8 +3,6 @@ package app import ( "bytes" "context" - "crypto/md5" - "crypto/sha1" "crypto/sha256" "encoding/base64" "encoding/hex" @@ -21,13 +19,13 @@ import ( "os/exec" "path" "path/filepath" - "reflect" "runtime" "sort" "strconv" "strings" "time" + "github.com/shellcell/snailmail/formats" "github.com/shellcell/snailmail/formats/deb" "github.com/shellcell/snailmail/formats/helm" "github.com/shellcell/snailmail/formats/pypi" @@ -35,9 +33,6 @@ import ( "github.com/shellcell/snailmail/host" "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/domain" - "github.com/shellcell/snailmail/internal/factscache" - "github.com/shellcell/snailmail/internal/listing" - openpgpsigner "github.com/shellcell/snailmail/signer/openpgp" ) const ( @@ -126,30 +121,17 @@ func verifyRepository(root string) (buildgraph.RepositoryManifest, []domain.Blob if err := rejectUnexpectedFiles(absolute, expected); err != nil { return buildgraph.RepositoryManifest{}, nil, err } - var blobs []domain.Blob - if manifest.Format == pypi.FormatID { - blobs, err = verifyPyPIStructure(absolute, manifest) - if err != nil { - return buildgraph.RepositoryManifest{}, nil, err - } - } - if manifest.Format == deb.FormatID { - blobs, err = verifyDebStructure(absolute, manifest) - if err != nil { - return buildgraph.RepositoryManifest{}, nil, err - } - } - if manifest.Format == helm.FormatID { - blobs, err = verifyHelmStructure(absolute, manifest) - if err != nil { - return buildgraph.RepositoryManifest{}, nil, err - } + // The tree says which rules produced it, so the structure check comes from + // the registry rather than from a chain of comparisons here that a new format + // had to be added to. An identity this build does not know is refused: it is + // a tree from a newer snailmail, not a tree with nothing to check. + selected, err := formats.ForID(manifest.Format) + if err != nil { + return buildgraph.RepositoryManifest{}, nil, err } - if manifest.Format == raw.FormatID { - blobs, err = verifyRawStructure(absolute, manifest) - if err != nil { - return buildgraph.RepositoryManifest{}, nil, err - } + blobs, err := selected.VerifyStructure(absolute, manifest) + if err != nil { + return buildgraph.RepositoryManifest{}, nil, err } return manifest, blobs, nil } @@ -937,328 +919,6 @@ func requireJSONEOF(decoder *json.Decoder) error { return nil } -func verifyPyPIStructure(root string, manifest buildgraph.RepositoryManifest) ([]domain.Blob, error) { - var blobs []domain.Blob - for _, file := range manifest.Files { - if file.Path == ".nojekyll" { - if file.Size != 0 { - return nil, errors.New("PyPI .nojekyll marker must be empty") - } - continue - } - // The browsable page is written for people, not for a client, and is - // covered by the tree digest like everything else. - if file.Path == listing.Filename { - continue - } - if strings.HasPrefix(file.Path, "simple/") { - if path.Base(file.Path) != "index.html" { - return nil, fmt.Errorf("unexpected PyPI index path %q", file.Path) - } - continue - } - parts := strings.Split(file.Path, "/") - if len(parts) != 3 || parts[0] != "packages" || parts[1] != file.SHA256 || !pypi.IsDistributionFilename(parts[2]) { - return nil, fmt.Errorf("unexpected PyPI repository path %q", file.Path) - } - facts, cached := factscache.Lookup(pypi.FormatID, file.SHA256) - if !cached { - name := filepath.Join(root, filepath.FromSlash(file.Path)) - packageFile, err := os.Open(name) - if err != nil { - return nil, fmt.Errorf("open PyPI package %q: %w", file.Path, err) - } - var inspectErr error - facts, inspectErr = pypi.Inspect(parts[2], packageFile, file.Size) - closeErr := packageFile.Close() - if inspectErr != nil { - return nil, inspectErr - } - if closeErr != nil { - return nil, fmt.Errorf("close PyPI package %q: %w", file.Path, closeErr) - } - factscache.Store(pypi.FormatID, file.SHA256, facts) - } - blobs = append(blobs, domain.Blob{Filename: parts[2], Size: file.Size, SHA256: file.SHA256, Facts: facts}) - } - expectedArtifact, err := pypi.Build(blobs) - if err != nil { - return nil, fmt.Errorf("rebuild PyPI structure: %w", err) - } - generatedAt, err := time.Parse(time.RFC3339, manifest.GeneratedAt) - if err != nil { - return nil, err - } - _, expectedManifest, err := buildgraph.Finalize(expectedArtifact, generatedAt) - if err != nil { - return nil, fmt.Errorf("finalize expected PyPI structure: %w", err) - } - expectedManifest.SchemaVersion = manifest.SchemaVersion - // The rebuild cannot reproduce the browsable page: it is written from the - // repository name, endpoint and signing key, which do not survive into the - // published tree. The tree digest still covers it. - expectedManifest.Files = withoutListing(expectedManifest.Files) - manifest.Files = withoutListing(manifest.Files) - // The tree digest covers the page too, so it cannot match a rebuild that - // omits it. What it protects is checked where it belongs: the deployment - // record is matched against the digest of the tree that was published. - expectedManifest.TreeSHA256, manifest.TreeSHA256 = "", "" - if !reflect.DeepEqual(expectedManifest, manifest) { - return nil, errors.New("PyPI indexes or verification metadata do not match package bytes") - } - return blobs, nil -} - -func verifyDebStructure(root string, manifest buildgraph.RepositoryManifest) ([]domain.Blob, error) { - var blobs []domain.Blob - for _, file := range manifest.Files { - // The browsable page is written for people, not for a client, and is - // covered by the tree digest like everything else. - if file.Path == listing.Filename { - continue - } - if strings.HasPrefix(file.Path, "dists/") || (manifest.Install.SigningKeyPath != "" && file.Path == manifest.Install.SigningKeyPath) { - continue - } - if !strings.HasPrefix(file.Path, "pool/") || !deb.IsPackageFilename(path.Base(file.Path)) { - return nil, fmt.Errorf("unexpected Debian repository path %q", file.Path) - } - name := filepath.Join(root, filepath.FromSlash(file.Path)) - facts, cached := factscache.Lookup(deb.FormatID, file.SHA256) - if !cached { - packageFile, err := os.Open(name) - if err != nil { - return nil, fmt.Errorf("open Debian package %q: %w", file.Path, err) - } - var inspectErr error - facts, inspectErr = deb.Inspect(path.Base(file.Path), packageFile, file.Size) - closeErr := packageFile.Close() - if inspectErr != nil { - return nil, inspectErr - } - if closeErr != nil { - return nil, fmt.Errorf("close Debian package %q: %w", file.Path, closeErr) - } - factscache.Store(deb.FormatID, file.SHA256, facts) - } - md5Value, sha1Value, err := legacyChecksums(name) - if err != nil { - return nil, err - } - blobs = append(blobs, domain.Blob{ - Filename: path.Base(file.Path), - Size: file.Size, - MD5: md5Value, - SHA1: sha1Value, - SHA256: file.SHA256, - Facts: facts, - }) - } - generatedAt, err := time.Parse(time.RFC3339, manifest.GeneratedAt) - if err != nil { - return nil, err - } - expectedArtifact, err := deb.Build(blobs, deb.BuildOptions{ - Suite: manifest.Install.Suite, - Component: manifest.Install.Component, - Architectures: manifest.Install.Architectures, - GeneratedAt: generatedAt, - }) - if err != nil { - return nil, fmt.Errorf("rebuild Debian structure: %w", err) - } - if manifest.Install.SigningKeyPath != "" { - if len(manifest.Signatures) != 2 || manifest.Install.SigningFingerprint == "" || path.IsAbs(manifest.Install.SigningKeyPath) || - path.Clean(manifest.Install.SigningKeyPath) != manifest.Install.SigningKeyPath || !strings.HasPrefix(manifest.Install.SigningKeyPath, "keys/") { - return nil, errors.New("signed Debian repository has incomplete signature metadata") - } - keyringContent, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(manifest.Install.SigningKeyPath))) - if err != nil { - return nil, fmt.Errorf("read Debian public signing key: %w", err) - } - inRelease, err := os.ReadFile(filepath.Join(root, "dists", manifest.Install.Suite, "InRelease")) - if err != nil { - return nil, fmt.Errorf("read Debian InRelease: %w", err) - } - releaseGPG, err := os.ReadFile(filepath.Join(root, "dists", manifest.Install.Suite, "Release.gpg")) - if err != nil { - return nil, fmt.Errorf("read Debian Release.gpg: %w", err) - } - signatureTime, err := time.Parse(time.RFC3339, manifest.Signatures[0].CreatedAt) - if err != nil || manifest.Signatures[1].CreatedAt != manifest.Signatures[0].CreatedAt { - return nil, errors.New("Debian signature metadata has inconsistent creation times") - } - trustedFingerprints := append([]string(nil), manifest.Install.TrustedSigningFingerprints...) - if len(trustedFingerprints) == 0 { - trustedFingerprints = []string{manifest.Install.SigningFingerprint} - } - activePublic, err := openpgpsigner.ExtractPublicKey(keyringContent, manifest.Install.SigningFingerprint) - if err != nil { - return nil, fmt.Errorf("extract active Debian signing key: %w", err) - } - expectedArtifact, err = deb.ApplySigning(expectedArtifact, manifest.Install.Suite, deb.SigningMaterial{ - Fingerprint: manifest.Install.SigningFingerprint, PublicKey: activePublic, - KeyringPath: manifest.Install.SigningKeyPath, PublicKeyring: keyringContent, TrustedFingerprints: trustedFingerprints, - SignatureTime: signatureTime, InRelease: inRelease, ReleaseGPG: releaseGPG, - }) - if err != nil { - return nil, fmt.Errorf("verify Debian signatures: %w", err) - } - } else if len(manifest.Signatures) != 0 { - return nil, errors.New("unsigned Debian repository contains signature metadata") - } - _, expectedManifest, err := buildgraph.Finalize(expectedArtifact, generatedAt) - if err != nil { - return nil, fmt.Errorf("finalize expected Debian structure: %w", err) - } - expectedManifest.SchemaVersion = manifest.SchemaVersion - if manifest.SchemaVersion < 3 { - expectedManifest.Install.TrustedSigningFingerprints = nil - } - // The rebuild cannot reproduce the browsable page: it is written from the - // repository name, endpoint and signing key, which do not survive into the - // published tree. The tree digest still covers it. - expectedManifest.Files = withoutListing(expectedManifest.Files) - manifest.Files = withoutListing(manifest.Files) - // The tree digest covers the page too, so it cannot match a rebuild that - // omits it. What it protects is checked where it belongs: the deployment - // record is matched against the digest of the tree that was published. - expectedManifest.TreeSHA256, manifest.TreeSHA256 = "", "" - if !reflect.DeepEqual(expectedManifest, manifest) { - return nil, errors.New("Debian indexes or verification metadata do not match package bytes") - } - return blobs, nil -} - -func verifyHelmStructure(root string, manifest buildgraph.RepositoryManifest) ([]domain.Blob, error) { - var blobs []domain.Blob - for _, file := range manifest.Files { - // The browsable page is written for people, not for a client, and is - // covered by the tree digest like everything else. - if file.Path == listing.Filename { - continue - } - if file.Path == "index.yaml" { - continue - } - // Signing material is checked below by rebuilding it, the same way the - // Debian path does: the published key and each chart's provenance are - // re-applied to the rebuilt tree, which verifies every signature. - if strings.HasPrefix(file.Path, "keys/") || strings.HasSuffix(file.Path, helm.ProvenanceSuffix) { - continue - } - parts := strings.Split(file.Path, "/") - if len(parts) != 3 || parts[0] != "charts" || parts[1] != file.SHA256 || !helm.IsChartFilename(parts[2]) { - return nil, fmt.Errorf("unexpected Helm repository path %q", file.Path) - } - facts, cached := factscache.Lookup(helm.FormatID, file.SHA256) - if !cached { - name := filepath.Join(root, filepath.FromSlash(file.Path)) - chartFile, err := os.Open(name) - if err != nil { - return nil, fmt.Errorf("open Helm chart %q: %w", file.Path, err) - } - var inspectErr error - facts, inspectErr = helm.Inspect(parts[2], chartFile, file.Size) - closeErr := chartFile.Close() - if inspectErr != nil { - return nil, inspectErr - } - if closeErr != nil { - return nil, fmt.Errorf("close Helm chart %q: %w", file.Path, closeErr) - } - factscache.Store(helm.FormatID, file.SHA256, facts) - } - blobs = append(blobs, domain.Blob{Filename: parts[2], Size: file.Size, SHA256: file.SHA256, Facts: facts}) - } - generatedAt, err := time.Parse(time.RFC3339, manifest.GeneratedAt) - if err != nil { - return nil, err - } - expectedArtifact, err := helm.Build(blobs, helm.BuildOptions{GeneratedAt: generatedAt}) - if err != nil { - return nil, fmt.Errorf("rebuild Helm structure: %w", err) - } - if manifest.Install.SigningKeyPath != "" { - if len(manifest.Signatures) == 0 || manifest.Install.SigningFingerprint == "" || path.IsAbs(manifest.Install.SigningKeyPath) || - path.Clean(manifest.Install.SigningKeyPath) != manifest.Install.SigningKeyPath || !strings.HasPrefix(manifest.Install.SigningKeyPath, "keys/") { - return nil, errors.New("signed Helm repository has incomplete signature metadata") - } - keyring, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(manifest.Install.SigningKeyPath))) - if err != nil { - return nil, fmt.Errorf("read Helm public signing key: %w", err) - } - provenance := make(map[string][]byte, len(manifest.Signatures)) - signatureTime, err := time.Parse(time.RFC3339, manifest.Signatures[0].CreatedAt) - if err != nil { - return nil, errors.New("Helm signature metadata has an invalid creation time") - } - for _, signature := range manifest.Signatures { - if signature.CreatedAt != manifest.Signatures[0].CreatedAt { - return nil, errors.New("Helm signature metadata has inconsistent creation times") - } - if !strings.HasSuffix(signature.Path, helm.ProvenanceSuffix) { - return nil, fmt.Errorf("Helm signature at %q is not a provenance file", signature.Path) - } - content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(signature.Path))) - if err != nil { - return nil, fmt.Errorf("read Helm provenance %q: %w", signature.Path, err) - } - provenance[strings.TrimSuffix(signature.Path, helm.ProvenanceSuffix)] = content - } - activePublic, err := openpgpsigner.ExtractPublicKey(keyring, manifest.Install.SigningFingerprint) - if err != nil { - return nil, fmt.Errorf("extract active Helm signing key: %w", err) - } - expectedArtifact, err = helm.ApplySigning(expectedArtifact, helm.SigningMaterial{ - Fingerprint: manifest.Install.SigningFingerprint, PublicKey: activePublic, - PublicKeyring: keyring, KeyringPath: manifest.Install.SigningKeyPath, - SignatureTime: signatureTime, Provenance: provenance, - }) - if err != nil { - return nil, fmt.Errorf("verify Helm signatures: %w", err) - } - } else if len(manifest.Signatures) != 0 { - return nil, errors.New("unsigned Helm repository contains signature metadata") - } - _, expectedManifest, err := buildgraph.Finalize(expectedArtifact, generatedAt) - if err != nil { - return nil, fmt.Errorf("finalize expected Helm structure: %w", err) - } - expectedManifest.SchemaVersion = manifest.SchemaVersion - // The rebuild cannot reproduce the browsable page: it is written from the - // repository name, endpoint and signing key, which do not survive into the - // published tree. The tree digest still covers it. - expectedManifest.Files = withoutListing(expectedManifest.Files) - manifest.Files = withoutListing(manifest.Files) - // The tree digest covers the page too, so it cannot match a rebuild that - // omits it. What it protects is checked where it belongs: the deployment - // record is matched against the digest of the tree that was published. - expectedManifest.TreeSHA256, manifest.TreeSHA256 = "", "" - if !reflect.DeepEqual(expectedManifest, manifest) { - return nil, errors.New("Helm index or verification metadata does not match chart bytes") - } - return blobs, nil -} - -func legacyChecksums(name string) (string, string, error) { - file, err := os.Open(name) - if err != nil { - return "", "", err - } - md5Hash := md5.New() - sha1Hash := sha1.New() - _, readErr := io.Copy(io.MultiWriter(md5Hash, sha1Hash), file) - closeErr := file.Close() - if readErr != nil { - return "", "", readErr - } - if closeErr != nil { - return "", "", closeErr - } - return hex.EncodeToString(md5Hash.Sum(nil)), hex.EncodeToString(sha1Hash.Sum(nil)), nil -} - func snapshotRepository(ctx context.Context, source string, manifest buildgraph.RepositoryManifest) (string, error) { // Beside the repository rather than in the system temp directory, so the // hard links below always have a filesystem to be made on. A snapshot on @@ -1361,56 +1021,6 @@ func copyFile(sourceName, targetName string, expectedSize int64) error { return closeSourceErr } -// verifyRawStructure recomputes a raw repository from its published tree. -// -// Identity is read from the path rather than the filename, which is what makes -// an operator-supplied name verifiable after publication: the build put it -// there, so re-deriving it needs no access to the flags used at ingest. -func verifyRawStructure(root string, manifest buildgraph.RepositoryManifest) ([]domain.Blob, error) { - var blobs []domain.Blob - for _, file := range manifest.Files { - if file.Path == "SHA256SUMS" || file.Path == "index.html" { - continue - } - parts := strings.Split(file.Path, "/") - if len(parts) != 3 { - return nil, fmt.Errorf("unexpected raw repository path %q", file.Path) - } - name, version, filename := parts[0], parts[1], parts[2] - if !raw.IsArtifactFilename(filename) { - return nil, fmt.Errorf("unexpected raw artifact filename %q", file.Path) - } - facts, err := raw.FactsFor(name, version, filename) - if err != nil { - return nil, fmt.Errorf("raw artifact %q: %w", file.Path, err) - } - blobs = append(blobs, domain.Blob{Filename: filename, Size: file.Size, SHA256: file.SHA256, Facts: facts}) - } - generatedAt, err := time.Parse(time.RFC3339, manifest.GeneratedAt) - if err != nil { - return nil, err - } - expectedArtifact, err := raw.Build(blobs, raw.BuildOptions{GeneratedAt: generatedAt}) - if err != nil { - return nil, fmt.Errorf("rebuild raw structure: %w", err) - } - _, expectedManifest, err := buildgraph.Finalize(expectedArtifact, generatedAt) - if err != nil { - return nil, fmt.Errorf("finalize expected raw structure: %w", err) - } - // The browsable page is left out of the comparison. It is written from the - // repository's name, endpoint and signing key, none of which survive into - // the published tree, so it cannot be recomputed from what is there. What - // this check exists for is the machine-readable side: that every artifact is - // where the checksums say and nothing else is present. The page is still - // covered by the tree digest, which the deployment record is matched - // against, so altering it does not go unnoticed. - if !reflect.DeepEqual(withoutListing(expectedManifest.Files), withoutListing(manifest.Files)) { - return nil, errors.New("raw checksums do not match the published artifacts") - } - return blobs, nil -} - // VerifyRawClientEndpointAccess proves a host serves exactly the raw tree that // was reviewed. // @@ -1442,16 +1052,3 @@ func VerifyRawClientEndpointAccess(ctx context.Context, root string, access host } return manifest, len(manifest.VerificationCases), nil } - -// withoutListing drops the human-readable page from a file set, leaving what a -// client reads. -func withoutListing(files []buildgraph.ManifestFile) []buildgraph.ManifestFile { - kept := make([]buildgraph.ManifestFile, 0, len(files)) - for _, file := range files { - if file.Path == listing.Filename { - continue - } - kept = append(kept, file) - } - return kept -} From 5e36fd4cc3edb4815cfd09b9bda49df7da1b54ad Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 14:18:10 +0200 Subject: [PATCH 14/26] Register client verification instead of switching on the format --- engine/clientverify.go | 120 ++++++++++++++++++++++++++++++++++++ engine/clientverify_test.go | 59 ++++++++++++++++++ engine/workspace.go | 56 ++--------------- 3 files changed, 185 insertions(+), 50 deletions(-) create mode 100644 engine/clientverify.go create mode 100644 engine/clientverify_test.go diff --git a/engine/clientverify.go b/engine/clientverify.go new file mode 100644 index 0000000..06143ee --- /dev/null +++ b/engine/clientverify.go @@ -0,0 +1,120 @@ +package engine + +import ( + "context" + "fmt" + + "github.com/shellcell/snailmail/host" + "github.com/shellcell/snailmail/internal/app" + "github.com/shellcell/snailmail/internal/buildgraph" +) + +// Client verification, one entry per ecosystem. +// +// This is the half of verification that stays out of the formats package. It +// runs a real pip, apt, dnf, apk or helm — in a container, against bytes on +// disk or against a URL a host is serving — and ARCHITECTURE §2 keeps effects +// out of the package that holds ecosystem rules. So the dispatch lives here, +// beside the code that owns effects, while the pure half is a method on +// formats.Format. +// +// A registry rather than a switch, for the reason the format registry is one: +// adding an ecosystem should be writing an entry, not editing a function that +// every other ecosystem also passes through. The conformance test below holds +// this table and the format registry to each other, so a format registered in +// one place and forgotten in the other fails rather than reaching an apply and +// reporting that verification "is not implemented". +type clientVerifier struct { + // staged installs from a built tree on disk. Every format has one: it is + // what a local host is verified with, and what a remote host falls back to + // when no preview endpoint is configured. + staged func(context.Context, string, ApplyWorkspaceRequest) (buildgraph.RepositoryManifest, error) + // endpoint installs from a URL a host is serving, which additionally proves + // the host serves the tree correctly. Nil where no probe is implemented, + // which formats.Format reports as HasEndpointProbe. + endpoint func(context.Context, string, host.ClientAccess, ApplyWorkspaceRequest) error +} + +var clientVerifiers = map[string]clientVerifier{ + "pypi": { + staged: func(ctx context.Context, repository string, request ApplyWorkspaceRequest) (buildgraph.RepositoryManifest, error) { + result, err := VerifyPyPI(ctx, VerifyPyPIRequest{ + Repository: repository, Python: request.Python, StructuralOnly: request.StructuralOnly, + }) + return result.Manifest, err + }, + endpoint: func(ctx context.Context, staged string, access host.ClientAccess, request ApplyWorkspaceRequest) error { + _, _, err := app.VerifyPyPIClientEndpointAccess(ctx, staged, access, request.Python) + return err + }, + }, + "deb": { + staged: func(ctx context.Context, repository string, request ApplyWorkspaceRequest) (buildgraph.RepositoryManifest, error) { + result, err := VerifyDeb(ctx, VerifyDebRequest{ + Repository: repository, Runner: request.Runner, Image: request.DebianImage, + MaxWorkspaceBytes: request.MaxWorkspaceBytes, StructuralOnly: request.StructuralOnly, + VerifyAllVersions: request.VerifyAllVersions, + }) + return result.Manifest, err + }, + endpoint: func(ctx context.Context, staged string, access host.ClientAccess, request ApplyWorkspaceRequest) error { + image := request.DebianImage + if image == "" { + image = DefaultDebianVerificationImage + } + maximum := request.MaxWorkspaceBytes + if maximum == 0 { + maximum = 4 << 30 + } + _, _, err := app.VerifyDebClientEndpointAccess(ctx, staged, access, request.Runner, image, maximum, + versionScope(request.VerifyAllVersions)) + return err + }, + }, + "helm": { + staged: func(ctx context.Context, repository string, request ApplyWorkspaceRequest) (buildgraph.RepositoryManifest, error) { + result, err := VerifyHelm(ctx, VerifyHelmRequest{ + Repository: repository, Runner: request.Runner, Image: request.HelmImage, + StructuralOnly: request.StructuralOnly, + }) + return result.Manifest, err + }, + }, + "raw": { + staged: func(_ context.Context, repository string, _ ApplyWorkspaceRequest) (buildgraph.RepositoryManifest, error) { + result, err := VerifyRaw(VerifyRawRequest{Repository: repository}) + return result.Manifest, err + }, + endpoint: func(ctx context.Context, staged string, access host.ClientAccess, _ ApplyWorkspaceRequest) error { + _, _, err := app.VerifyRawClientEndpointAccess(ctx, staged, access) + return err + }, + }, + "rpm": { + staged: func(ctx context.Context, repository string, request ApplyWorkspaceRequest) (buildgraph.RepositoryManifest, error) { + result, err := VerifyRPM(ctx, VerifyRPMRequest{ + Repository: repository, Runner: request.Runner, Image: request.RPMImage, + StructuralOnly: request.StructuralOnly, VerifyAllVersions: request.VerifyAllVersions, + }) + return result.Manifest, err + }, + }, + "apk": { + staged: func(ctx context.Context, repository string, request ApplyWorkspaceRequest) (buildgraph.RepositoryManifest, error) { + result, err := VerifyAPK(ctx, VerifyAPKRequest{ + Repository: repository, Runner: request.Runner, Image: request.APKImage, + StructuralOnly: request.StructuralOnly, VerifyAllVersions: request.VerifyAllVersions, + }) + return result.Manifest, err + }, + }, +} + +// verifyStaged runs the ecosystem's client against a tree on disk. +func verifyStaged(ctx context.Context, format, repository string, request ApplyWorkspaceRequest) (buildgraph.RepositoryManifest, error) { + verifier, known := clientVerifiers[format] + if !known { + return buildgraph.RepositoryManifest{}, fmt.Errorf("unsupported repository format %q", format) + } + return verifier.staged(ctx, repository, request) +} diff --git a/engine/clientverify_test.go b/engine/clientverify_test.go new file mode 100644 index 0000000..fa826b4 --- /dev/null +++ b/engine/clientverify_test.go @@ -0,0 +1,59 @@ +package engine + +import ( + "testing" + + "github.com/shellcell/snailmail/formats" +) + +// Verification is split across two registries: the pure half is a method on +// formats.Format, the half that runs a container is a table in this package. +// Splitting it is what keeps effects out of the formats package — and it is also +// how a format comes to be registered in one place and forgotten in the other. +// +// These hold the two to each other. Without them the failure surfaces during an +// apply, after a tree has been built and staged, as "client verification is not +// implemented" — which reads as a missing feature rather than a missing entry. + +func TestEveryFormatCanBeVerifiedFromItsBuiltTree(t *testing.T) { + for _, format := range formats.All() { + verifier, known := clientVerifiers[format.Name()] + if !known { + t.Errorf("format %q is registered but has no client verifier", format.Name()) + continue + } + if verifier.staged == nil { + t.Errorf("format %q has no way to be verified from a built tree", format.Name()) + } + } +} + +// The endpoint probe is declared on the format and implemented here, so the two +// have to agree. A format claiming a probe it does not have would skip the +// staged-tree check it does have and then fail with an error about a feature. +func TestTheEndpointProbeIsImplementedExactlyWhereItIsClaimed(t *testing.T) { + for _, format := range formats.All() { + verifier, known := clientVerifiers[format.Name()] + if !known { + continue // reported by the test above + } + claimed := format.HasEndpointProbe() + implemented := verifier.endpoint != nil + if claimed && !implemented { + t.Errorf("format %q claims an endpoint probe but implements none", format.Name()) + } + if implemented && !claimed { + t.Errorf("format %q implements an endpoint probe but reports none, so it is never run", format.Name()) + } + } +} + +// Nothing is verified by a table entry that no format registered, which would be +// dead code shaped exactly like working code. +func TestEveryClientVerifierBelongsToARegisteredFormat(t *testing.T) { + for name := range clientVerifiers { + if _, err := formats.For(name); err != nil { + t.Errorf("client verifier %q has no registered format", name) + } + } +} diff --git a/engine/workspace.go b/engine/workspace.go index c6b1321..36760ac 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -1323,27 +1323,14 @@ func verifyEndpointClient(ctx context.Context, repository state.Repository, stag _, err := verifyStaged(ctx, repository.Format, staged, request) return err } - switch repository.Format { - case "pypi": - _, _, err := app.VerifyPyPIClientEndpointAccess(ctx, staged, access, request.Python) - return err - case "deb": - image := request.DebianImage - if image == "" { - image = DefaultDebianVerificationImage - } - maximum := request.MaxWorkspaceBytes - if maximum == 0 { - maximum = 4 << 30 - } - _, _, err := app.VerifyDebClientEndpointAccess(ctx, staged, access, request.Runner, image, maximum, versionScope(request.VerifyAllVersions)) - return err - case "raw": - _, _, err := app.VerifyRawClientEndpointAccess(ctx, staged, access) - return err - default: + verifier, known := clientVerifiers[repository.Format] + if !known || verifier.endpoint == nil { + // Unreachable while the conformance test holds this table and the format + // registry to each other, which is what it exists for: HasEndpointProbe + // said there was a probe, so there is one. return fmt.Errorf("client verification is not implemented for format %q", repository.Format) } + return verifier.endpoint(ctx, staged, access, request) } func buildLockedRepository(ctx context.Context, root, name string, repository state.Repository, lock state.RepositoryLock, generatedAt, signatureTime time.Time, output string, blobStore blob.Store, keys map[string]state.SigningKey, plannedSigning []state.PlanSigning, signers signer.Resolver, fetcher source.Fetcher) (BuildResult, error) { @@ -1540,37 +1527,6 @@ func planAcquisitionsForVersions(versions []state.PackageVersion) []state.PlanAc return acquisitions } -func verifyStaged(ctx context.Context, format, repository string, request ApplyWorkspaceRequest) (buildgraph.RepositoryManifest, error) { - switch format { - case "pypi": - result, err := VerifyPyPI(ctx, VerifyPyPIRequest{Repository: repository, Python: request.Python, StructuralOnly: request.StructuralOnly}) - return result.Manifest, err - case "deb": - result, err := VerifyDeb(ctx, VerifyDebRequest{Repository: repository, Runner: request.Runner, Image: request.DebianImage, MaxWorkspaceBytes: request.MaxWorkspaceBytes, StructuralOnly: request.StructuralOnly, VerifyAllVersions: request.VerifyAllVersions}) - return result.Manifest, err - case "helm": - result, err := VerifyHelm(ctx, VerifyHelmRequest{Repository: repository, Runner: request.Runner, Image: request.HelmImage, StructuralOnly: request.StructuralOnly}) - return result.Manifest, err - case "raw": - result, err := VerifyRaw(VerifyRawRequest{Repository: repository}) - return result.Manifest, err - case "rpm": - result, err := VerifyRPM(ctx, VerifyRPMRequest{ - Repository: repository, Runner: request.Runner, Image: request.RPMImage, - StructuralOnly: request.StructuralOnly, VerifyAllVersions: request.VerifyAllVersions, - }) - return result.Manifest, err - case "apk": - result, err := VerifyAPK(ctx, VerifyAPKRequest{ - Repository: repository, Runner: request.Runner, Image: request.APKImage, - StructuralOnly: request.StructuralOnly, VerifyAllVersions: request.VerifyAllVersions, - }) - return result.Manifest, err - default: - return buildgraph.RepositoryManifest{}, fmt.Errorf("unsupported repository format %q", format) - } -} - // StageDirectoryEnvironment overrides where build and stage trees are created. const StageDirectoryEnvironment = "SNAILMAIL_STAGE_DIR" From dbb2b764413f11f9b78fe740b4d372939ebd72d7 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 14:47:55 +0200 Subject: [PATCH 15/26] Let a host adapter check bytes without knowing about ecosystems --- adapters/host/githubpages/githubpages.go | 8 +- adapters/host/githubpages/githubpages_test.go | 11 +- adapters/host/local/local.go | 9 +- adapters/host/rsync/rsync.go | 4 +- adapters/host/rsync/rsync_test.go | 9 +- adapters/host/s3/s3.go | 3 +- adapters/host/s3/s3_test.go | 22 +- engine/clientverify_test.go | 57 +++++ engine/engine.go | 7 +- engine/workspace.go | 4 +- engine/workspace_test.go | 3 +- internal/app/deb_endpoint_test.go | 3 +- internal/app/pypi_endpoint_test.go | 7 +- internal/app/snapshot_test.go | 27 ++- internal/app/verify.go | 180 +------------- internal/buildgraph/verify.go | 225 ++++++++++++++++++ internal/{app => release}/commit.go | 2 +- internal/{app => release}/commit_darwin.go | 2 +- .../{app => release}/commit_exchange_test.go | 2 +- internal/{app => release}/commit_linux.go | 2 +- internal/{app => release}/commit_other.go | 2 +- internal/{app => release}/materialize.go | 35 +-- .../materialize_hardening_test.go | 15 +- 23 files changed, 391 insertions(+), 248 deletions(-) create mode 100644 internal/buildgraph/verify.go rename internal/{app => release}/commit.go (98%) rename internal/{app => release}/commit_darwin.go (94%) rename internal/{app => release}/commit_exchange_test.go (99%) rename internal/{app => release}/commit_linux.go (93%) rename internal/{app => release}/commit_other.go (96%) rename internal/{app => release}/materialize.go (93%) rename internal/{app => release}/materialize_hardening_test.go (91%) diff --git a/adapters/host/githubpages/githubpages.go b/adapters/host/githubpages/githubpages.go index 19a0fee..6e6d21b 100644 --- a/adapters/host/githubpages/githubpages.go +++ b/adapters/host/githubpages/githubpages.go @@ -19,7 +19,7 @@ import ( "time" "github.com/shellcell/snailmail/host" - "github.com/shellcell/snailmail/internal/app" + "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/hexdigest" ) @@ -409,7 +409,7 @@ func validateStageRequest(request host.StageRequest) error { if !hexdigest.ValidSHA256(request.PlanID) || request.ChangeID == "" || !hexdigest.ValidSHA256(request.TreeSHA256) || (request.PreviousRevision != "" && !validGitObject(request.PreviousRevision)) || request.Directory == "" || len(request.Files) == 0 { return invalid("stage GitHub Pages publication", "invalid stage request") } - manifest, err := app.VerifyRepository(request.Directory) + manifest, err := buildgraph.VerifyTree(request.Directory) if err != nil || manifest.TreeSHA256 != request.TreeSHA256 { return invalid("stage GitHub Pages publication", "staged directory does not match reviewed tree") } @@ -619,7 +619,7 @@ func (workspace *gitWorkspace) inspectPublication(ctx context.Context, commit st if err := workspace.materializeCommit(ctx, commit, checkout); err != nil { return host.PublishedRevision{}, nil, publicationMetadata{}, err } - manifest, err := app.VerifyRepository(checkout) + manifest, err := buildgraph.VerifyTree(checkout) if err != nil { return host.PublishedRevision{}, nil, publicationMetadata{}, err } @@ -894,7 +894,7 @@ func hashFile(filename string) (string, error) { } func stagedFiles(directory string) ([]host.File, error) { - manifest, err := app.VerifyRepository(directory) + manifest, err := buildgraph.VerifyTree(directory) if err != nil { return nil, err } diff --git a/adapters/host/githubpages/githubpages_test.go b/adapters/host/githubpages/githubpages_test.go index 8a42bb2..0a8d10d 100644 --- a/adapters/host/githubpages/githubpages_test.go +++ b/adapters/host/githubpages/githubpages_test.go @@ -16,6 +16,7 @@ import ( "github.com/shellcell/snailmail/host" "github.com/shellcell/snailmail/internal/app" "github.com/shellcell/snailmail/internal/buildgraph" + "github.com/shellcell/snailmail/internal/release" "github.com/shellcell/snailmail/internal/testutil" ) @@ -155,14 +156,14 @@ func pagesStageFixture(t *testing.T, label, version string) host.StageRequest { t.Fatal(err) } repository := filepath.Join(t.TempDir(), "repository") - if err := app.Materialize(context.Background(), repository, artifact, snapshot.Sources); err != nil { + if err := release.Materialize(context.Background(), repository, artifact, snapshot.Sources); err != nil { t.Fatal(err) } - release, err := filepath.EvalSymlinks(repository) + resolved, err := filepath.EvalSymlinks(repository) if err != nil { t.Fatal(err) } - manifest, err := app.VerifyRepository(release) + manifest, err := buildgraph.VerifyTree(resolved) if err != nil { t.Fatal(err) } @@ -170,7 +171,7 @@ func pagesStageFixture(t *testing.T, label, version string) host.StageRequest { for _, file := range manifest.Files { files = append(files, host.File{Path: file.Path, Size: file.Size, SHA256: file.SHA256}) } - management := filepath.Join(release, "snailmail.repository.json") + management := filepath.Join(resolved, "snailmail.repository.json") info, err := os.Stat(management) if err != nil { t.Fatal(err) @@ -180,7 +181,7 @@ func pagesStageFixture(t *testing.T, label, version string) host.StageRequest { sort.Slice(files, func(left, right int) bool { return files[left].Path < files[right].Path }) plan := sha256.Sum256([]byte(label)) return host.StageRequest{ - PlanID: hex.EncodeToString(plan[:]), ChangeID: "python:" + strings.Repeat(label[:1], 12), Directory: release, + PlanID: hex.EncodeToString(plan[:]), ChangeID: "python:" + strings.Repeat(label[:1], 12), Directory: resolved, TreeSHA256: manifest.TreeSHA256, Files: files, CommitPaths: []string{"simple/index.html"}, } } diff --git a/adapters/host/local/local.go b/adapters/host/local/local.go index e513fe5..3f411a8 100644 --- a/adapters/host/local/local.go +++ b/adapters/host/local/local.go @@ -8,7 +8,8 @@ import ( "path/filepath" "github.com/shellcell/snailmail/host" - "github.com/shellcell/snailmail/internal/app" + "github.com/shellcell/snailmail/internal/buildgraph" + "github.com/shellcell/snailmail/internal/release" "github.com/shellcell/snailmail/internal/state" ) @@ -32,7 +33,7 @@ func (adapter *Adapter) Observe(_ context.Context, repository host.Repository) ( } else if err != nil { return host.PublishedRevision{}, err } - manifest, err := app.VerifyRepository(output) + manifest, err := buildgraph.VerifyTree(output) if err != nil { return host.PublishedRevision{}, fmt.Errorf("observe local repository: %w", err) } @@ -48,7 +49,7 @@ func (adapter *Adapter) ReadAccess(_ context.Context, repository host.Repository } func (adapter *Adapter) Stage(_ context.Context, _ host.Repository, request host.StageRequest) (host.StagedPublication, error) { - manifest, err := app.VerifyRepository(request.Directory) + manifest, err := buildgraph.VerifyTree(request.Directory) if err != nil { return host.StagedPublication{}, fmt.Errorf("stage local repository: %w", err) } @@ -66,7 +67,7 @@ func (adapter *Adapter) Commit(ctx context.Context, repository host.Repository, if err != nil { return host.CommitResult{}, err } - if err := app.PublishVerifiedDirectory(ctx, staged.ID, output, expected.TreeSHA256, staged.TreeSHA256); err != nil { + if err := release.PublishVerifiedDirectory(ctx, staged.ID, output, expected.TreeSHA256, staged.TreeSHA256); err != nil { return host.CommitResult{}, &host.Error{Kind: host.ErrorStale, Operation: "commit local repository", Err: err} } revision := host.PublishedRevision{NativeRevision: staged.TreeSHA256, TreeSHA256: staged.TreeSHA256, PlanID: staged.PlanID, ChangeID: staged.ChangeID} diff --git a/adapters/host/rsync/rsync.go b/adapters/host/rsync/rsync.go index b15429e..36331a1 100644 --- a/adapters/host/rsync/rsync.go +++ b/adapters/host/rsync/rsync.go @@ -8,7 +8,7 @@ import ( "strings" "github.com/shellcell/snailmail/host" - "github.com/shellcell/snailmail/internal/app" + "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/hexdigest" ) @@ -88,7 +88,7 @@ func (adapter *Adapter) Stage(_ context.Context, repository host.Repository, req } // Verified locally before anything is sent, so a tree that does not match what // the plan describes never reaches the far side at all. - manifest, err := app.VerifyRepository(request.Directory) + manifest, err := buildgraph.VerifyTree(request.Directory) if err != nil { return host.StagedPublication{}, fmt.Errorf("stage rsync repository: %w", err) } diff --git a/adapters/host/rsync/rsync_test.go b/adapters/host/rsync/rsync_test.go index cb91183..060bdc6 100644 --- a/adapters/host/rsync/rsync_test.go +++ b/adapters/host/rsync/rsync_test.go @@ -15,6 +15,7 @@ import ( "github.com/shellcell/snailmail/host" "github.com/shellcell/snailmail/internal/app" "github.com/shellcell/snailmail/internal/buildgraph" + "github.com/shellcell/snailmail/internal/release" "github.com/shellcell/snailmail/internal/testutil" ) @@ -87,18 +88,18 @@ func verifiedTree(t *testing.T, version string) (string, string) { t.Fatal(err) } directory := filepath.Join(t.TempDir(), "repository") - if err := app.Materialize(context.Background(), directory, artifact, snapshot.Sources); err != nil { + if err := release.Materialize(context.Background(), directory, artifact, snapshot.Sources); err != nil { t.Fatal(err) } - release, err := filepath.EvalSymlinks(directory) + resolved, err := filepath.EvalSymlinks(directory) if err != nil { t.Fatal(err) } - manifest, err := app.VerifyRepository(release) + manifest, err := buildgraph.VerifyTree(resolved) if err != nil { t.Fatal(err) } - return release, manifest.TreeSHA256 + return resolved, manifest.TreeSHA256 } func publishedRepository(t *testing.T) (host.Repository, string) { diff --git a/adapters/host/s3/s3.go b/adapters/host/s3/s3.go index 10b9d5e..fdc76a6 100644 --- a/adapters/host/s3/s3.go +++ b/adapters/host/s3/s3.go @@ -23,7 +23,6 @@ import ( "github.com/shellcell/snailmail/formats/pypi" "github.com/shellcell/snailmail/host" - "github.com/shellcell/snailmail/internal/app" "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/listing" @@ -330,7 +329,7 @@ func (adapter *Adapter) Stage(ctx context.Context, repository host.Repository, r }); err != nil { return host.StagedPublication{}, err } - manifest, err := app.VerifyRepository(request.Directory) + manifest, err := buildgraph.VerifyTree(request.Directory) if err != nil || !manifestFormatIs(manifest.Format, repository.Format) || manifest.TreeSHA256 != request.TreeSHA256 { return host.StagedPublication{}, &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "stage S3 repository", Err: fmt.Errorf("staged repository is not a valid %s publication", repository.Format)} diff --git a/adapters/host/s3/s3_test.go b/adapters/host/s3/s3_test.go index a268b71..4cebccd 100644 --- a/adapters/host/s3/s3_test.go +++ b/adapters/host/s3/s3_test.go @@ -615,14 +615,20 @@ func TestS3HostRejectsSemanticallyInvalidPublicationManifest(t *testing.T) { if err := os.WriteFile(manifestName, content, 0o644); err != nil { t.Fatal(err) } - for index := range request.Files { - if request.Files[index].Path == buildgraph.ManifestFilename { - request.Files[index].Size = int64(len(content)) - request.Files[index].SHA256 = digestBytes(content) - } - } - if _, err := adapter.Stage(ctx, repository, request); !host.IsKind(err, host.ErrorInvalidConfiguration) { - t.Fatalf("invalid publication manifest error = %v", err) + // request.Files is left alone, which is the only shape this can take in + // production: the engine builds that list from the manifest it verified, and + // nothing between there and here can rewrite it. So a manifest edited on disk + // no longer matches the digest the plan carries for it, and the adapter + // refuses on that — a question about bytes, which is the kind of question an + // adapter is for. + // + // The edit itself is semantic: a verification case naming a version that is + // not published. Whether a manifest means something sensible is the engine's + // question, settled by full verification before a tree is ever staged. The + // adapter used to ask it too, by way of internal/app, which is how a host + // came to depend on the application layer and to know about ecosystems. + if _, err := adapter.Stage(ctx, repository, request); err == nil { + t.Fatal("a manifest edited after the plan was made was staged") } observed, err := adapter.Observe(ctx, repository) if err != nil || observed.NativeRevision != "" { diff --git a/engine/clientverify_test.go b/engine/clientverify_test.go index fa826b4..1792948 100644 --- a/engine/clientverify_test.go +++ b/engine/clientverify_test.go @@ -1,9 +1,15 @@ package engine import ( + "context" + "encoding/json" + "os" + "path/filepath" "testing" "github.com/shellcell/snailmail/formats" + "github.com/shellcell/snailmail/internal/buildgraph" + "github.com/shellcell/snailmail/internal/testutil" ) // Verification is split across two registries: the pure half is a method on @@ -57,3 +63,54 @@ func TestEveryClientVerifierBelongsToARegisteredFormat(t *testing.T) { } } } + +// A semantically invalid manifest is refused before anything is staged. +// +// Verification is split: buildgraph checks that a tree's bytes are what its +// manifest says, and the format checks that the manifest means something. The +// adapters do the first — that is what an adapter is for, and it is why they no +// longer import the application layer. This is the second, and it has to happen +// before a host is handed anything, because after that the guarantee is only +// that the bytes match the plan. +func TestASemanticallyInvalidTreeIsRefusedBeforeStaging(t *testing.T) { + root := t.TempDir() + input := t.TempDir() + if _, err := testutil.WriteWheel(input, "demo-pkg", "1.2.3", ""); err != nil { + t.Fatal(err) + } + output := filepath.Join(root, "repository") + if _, err := BuildPyPI(context.Background(), BuildPyPIRequest{Input: input, Output: output}); err != nil { + t.Fatal(err) + } + release, err := filepath.EvalSymlinks(output) + if err != nil { + t.Fatal(err) + } + name := filepath.Join(release, buildgraph.ManifestFilename) + content, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + var manifest buildgraph.RepositoryManifest + if err := json.Unmarshal(content, &manifest); err != nil { + t.Fatal(err) + } + // A case naming a version the tree does not publish. The file list is + // untouched, so the tree digest still matches and buildgraph is satisfied. + manifest.VerificationCases[0].Version = "9999" + rewritten, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(name, append(rewritten, '\n'), 0o644); err != nil { + t.Fatal(err) + } + + if _, err := buildgraph.VerifyTree(release); err != nil { + t.Fatalf("the bytes still match the manifest, so buildgraph should be satisfied: %v", err) + } + if _, err := verifyStaged(context.Background(), "pypi", + output, ApplyWorkspaceRequest{StructuralOnly: true}); err == nil { + t.Fatal("a manifest naming an unpublished version passed verification") + } +} diff --git a/engine/engine.go b/engine/engine.go index 5ca0432..2dd670e 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -11,6 +11,7 @@ import ( "github.com/shellcell/snailmail/internal/app" "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/domain" + "github.com/shellcell/snailmail/internal/release" "github.com/shellcell/snailmail/internal/state" ) @@ -143,7 +144,7 @@ func BuildPyPI(ctx context.Context, request BuildPyPIRequest) (BuildResult, erro if err != nil { return BuildResult{}, err } - if err := app.Materialize(ctx, output, artifact, snapshot.Sources); err != nil { + if err := release.Materialize(ctx, output, artifact, snapshot.Sources); err != nil { return BuildResult{}, err } manifestSHA256, err := state.HashFile(filepath.Join(output, buildgraph.ManifestFilename)) @@ -202,7 +203,7 @@ func buildDeb(ctx context.Context, request BuildDebRequest, transform func(domai if err != nil { return BuildResult{}, err } - if err := app.Materialize(ctx, output, artifact, snapshot.Sources); err != nil { + if err := release.Materialize(ctx, output, artifact, snapshot.Sources); err != nil { return BuildResult{}, err } manifestSHA256, err := state.HashFile(filepath.Join(output, buildgraph.ManifestFilename)) @@ -264,7 +265,7 @@ func buildHelm(ctx context.Context, request BuildHelmRequest, transform func(dom if err != nil { return BuildResult{}, err } - if err := app.Materialize(ctx, output, artifact, snapshot.Sources); err != nil { + if err := release.Materialize(ctx, output, artifact, snapshot.Sources); err != nil { return BuildResult{}, err } manifestSHA256, err := state.HashFile(filepath.Join(output, buildgraph.ManifestFilename)) diff --git a/engine/workspace.go b/engine/workspace.go index 36760ac..597bab8 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -19,10 +19,10 @@ import ( "github.com/shellcell/snailmail/formats" "github.com/shellcell/snailmail/gate" "github.com/shellcell/snailmail/host" - "github.com/shellcell/snailmail/internal/app" "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/domain" "github.com/shellcell/snailmail/internal/knowledge" + "github.com/shellcell/snailmail/internal/release" "github.com/shellcell/snailmail/internal/state" statusrenderer "github.com/shellcell/snailmail/internal/status" "github.com/shellcell/snailmail/signer" @@ -1429,7 +1429,7 @@ func materializeLockedArtifact(ctx context.Context, output string, generatedAt t if err != nil { return BuildResult{}, err } - if err := app.Materialize(ctx, output, artifact, sources); err != nil { + if err := release.Materialize(ctx, output, artifact, sources); err != nil { return BuildResult{}, err } manifestSHA256, err := state.HashFile(filepath.Join(output, buildgraph.ManifestFilename)) diff --git a/engine/workspace_test.go b/engine/workspace_test.go index 6191aa9..fa4836a 100644 --- a/engine/workspace_test.go +++ b/engine/workspace_test.go @@ -24,6 +24,7 @@ import ( "github.com/shellcell/snailmail/host" "github.com/shellcell/snailmail/internal/app" "github.com/shellcell/snailmail/internal/buildgraph" + "github.com/shellcell/snailmail/internal/release" "github.com/shellcell/snailmail/internal/state" "github.com/shellcell/snailmail/internal/testutil" "github.com/shellcell/snailmail/signer" @@ -1757,7 +1758,7 @@ func TestVerifiedPublicationHonorsTargetPrecondition(t *testing.T) { if err != nil { t.Fatal(err) } - if err := app.PublishVerifiedDirectory(context.Background(), staged, target, initial.TreeSHA256, desired.TreeSHA256); err == nil { + if err := release.PublishVerifiedDirectory(context.Background(), staged, target, initial.TreeSHA256, desired.TreeSHA256); err == nil { t.Fatal("expected changed target to reject verified publication") } current, err := InspectRepository(target) diff --git a/internal/app/deb_endpoint_test.go b/internal/app/deb_endpoint_test.go index 977bef0..7814d94 100644 --- a/internal/app/deb_endpoint_test.go +++ b/internal/app/deb_endpoint_test.go @@ -22,6 +22,7 @@ import ( "github.com/shellcell/snailmail/host" "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/domain" + "github.com/shellcell/snailmail/internal/release" "github.com/shellcell/snailmail/internal/testutil" ) @@ -77,7 +78,7 @@ func buildDebRepository(t *testing.T, architecture string) string { t.Fatal(err) } output := filepath.Join(t.TempDir(), "repository") - if err := Materialize(context.Background(), output, mustFinalize(t, artifact), map[string]string{blob.SHA256: source}); err != nil { + if err := release.Materialize(context.Background(), output, mustFinalize(t, artifact), map[string]string{blob.SHA256: source}); err != nil { t.Fatal(err) } resolved, err := filepath.EvalSymlinks(output) diff --git a/internal/app/pypi_endpoint_test.go b/internal/app/pypi_endpoint_test.go index 32f1cef..a997e67 100644 --- a/internal/app/pypi_endpoint_test.go +++ b/internal/app/pypi_endpoint_test.go @@ -13,6 +13,7 @@ import ( "github.com/shellcell/snailmail/formats/pypi" "github.com/shellcell/snailmail/host" "github.com/shellcell/snailmail/internal/buildgraph" + "github.com/shellcell/snailmail/internal/release" "github.com/shellcell/snailmail/internal/testutil" ) @@ -41,7 +42,7 @@ func TestVerifyPyPIClientEndpointInstallsFromSelectedHost(t *testing.T) { t.Fatal(err) } repository := filepath.Join(t.TempDir(), "repository") - if err := Materialize(context.Background(), repository, artifact, snapshot.Sources); err != nil { + if err := release.Materialize(context.Background(), repository, artifact, snapshot.Sources); err != nil { t.Fatal(err) } release, err := filepath.EvalSymlinks(repository) @@ -84,7 +85,7 @@ func TestVerifyPyPIClientEndpointUsesBasicCredentialWithoutLeakingIt(t *testing. t.Fatal(err) } repository := filepath.Join(t.TempDir(), "repository") - if err := Materialize(context.Background(), repository, artifact, snapshot.Sources); err != nil { + if err := release.Materialize(context.Background(), repository, artifact, snapshot.Sources); err != nil { t.Fatal(err) } release, err := filepath.EvalSymlinks(repository) @@ -133,7 +134,7 @@ func TestVerifyPyPIClientEndpointRejectsDifferentDirectoryRoute(t *testing.T) { t.Fatal(err) } repository := filepath.Join(t.TempDir(), "repository") - if err := Materialize(context.Background(), repository, artifact, snapshot.Sources); err != nil { + if err := release.Materialize(context.Background(), repository, artifact, snapshot.Sources); err != nil { t.Fatal(err) } release, err := filepath.EvalSymlinks(repository) diff --git a/internal/app/snapshot_test.go b/internal/app/snapshot_test.go index 132d225..9f55b92 100644 --- a/internal/app/snapshot_test.go +++ b/internal/app/snapshot_test.go @@ -6,6 +6,11 @@ import ( "path/filepath" "strings" "testing" + "time" + + "github.com/shellcell/snailmail/internal/buildgraph" + "github.com/shellcell/snailmail/internal/domain" + "github.com/shellcell/snailmail/internal/release" ) // A snapshot shares inodes with the repository it was taken from. @@ -18,7 +23,7 @@ import ( // output. Asserting on the inode is the only way that stays caught. func TestSnapshotLinksRatherThanCopies(t *testing.T) { output := filepath.Join(t.TempDir(), "repository") - manifest := writeManagedTestRepository(t, output, "some published bytes") + manifest := writeSnapshotFixture(t, output, "some published bytes") snapshot, err := snapshotRepository(context.Background(), output, manifest) if err != nil { @@ -47,7 +52,7 @@ func TestSnapshotLinksRatherThanCopies(t *testing.T) { func TestSnapshotIsTakenBesideTheRepository(t *testing.T) { root := t.TempDir() output := filepath.Join(root, "repository") - manifest := writeManagedTestRepository(t, output, "some published bytes") + manifest := writeSnapshotFixture(t, output, "some published bytes") snapshot, err := snapshotRepository(context.Background(), output, manifest) if err != nil { @@ -69,3 +74,21 @@ func TestSnapshotIsTakenBesideTheRepository(t *testing.T) { t.Errorf("snapshot %q is inside the repository it snapshots", snapshot) } } + +// A published tree for snapshotRepository to link from. It carries no ecosystem: +// snapshotting is about inodes, and which format produced the bytes is settled +// long before one is taken. +func writeSnapshotFixture(t *testing.T, output, content string) buildgraph.RepositoryManifest { + t.Helper() + artifact, manifest, err := buildgraph.Finalize(domain.RepositoryArtifact{ + Format: "test/v1", + Files: []domain.File{{Path: "index.txt", Content: []byte(content)}}, + }, time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC)) + if err != nil { + t.Fatal(err) + } + if err := release.Materialize(context.Background(), output, artifact, nil); err != nil { + t.Fatal(err) + } + return manifest +} diff --git a/internal/app/verify.go b/internal/app/verify.go index 8a10900..799e759 100644 --- a/internal/app/verify.go +++ b/internal/app/verify.go @@ -6,7 +6,6 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" - "encoding/json" "errors" "fmt" "io" @@ -17,7 +16,6 @@ import ( "net/url" "os" "os/exec" - "path" "path/filepath" "runtime" "sort" @@ -48,83 +46,23 @@ func VerifyRepository(root string) (buildgraph.RepositoryManifest, error) { } func verifyRepository(root string) (buildgraph.RepositoryManifest, []domain.Blob, error) { - absolute, err := filepath.Abs(root) + // The bytes first: that the manifest parses, that every file it names is + // present with the digest it claims, and that nothing else is. That half is + // format-neutral and lives in buildgraph, where a host adapter can reach it + // without importing this package to ask whether a stage matches its plan. + manifest, err := buildgraph.VerifyTree(root) if err != nil { - return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("resolve repository: %w", err) + return buildgraph.RepositoryManifest{}, nil, err } - absolute, err = filepath.EvalSymlinks(absolute) + absolute, err := filepath.EvalSymlinks(root) if err != nil { return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("resolve repository release: %w", err) } - manifestPath := filepath.Join(absolute, buildgraph.ManifestFilename) - manifestFile, err := os.Open(manifestPath) - if err != nil { - return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("open repository manifest: %w", err) - } - manifestBytes, readErr := io.ReadAll(io.LimitReader(manifestFile, maxManifestSize+1)) - closeErr := manifestFile.Close() - if readErr != nil { - return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("read repository manifest: %w", readErr) - } - if closeErr != nil { - return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("close repository manifest: %w", closeErr) - } - if len(manifestBytes) > maxManifestSize { - return buildgraph.RepositoryManifest{}, nil, errors.New("repository manifest exceeds 8 MiB") - } - decoder := json.NewDecoder(bytes.NewReader(manifestBytes)) - decoder.DisallowUnknownFields() - var manifest buildgraph.RepositoryManifest - if err := decoder.Decode(&manifest); err != nil { - return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("decode repository manifest: %w", err) - } - if err := requireJSONEOF(decoder); err != nil { - return buildgraph.RepositoryManifest{}, nil, err - } - if !buildgraph.SupportedManifestSchema(manifest.SchemaVersion) { - return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("unsupported repository schema %d", manifest.SchemaVersion) - } - if manifest.Format == "" { - return buildgraph.RepositoryManifest{}, nil, errors.New("repository manifest has no format") - } - if _, err := time.Parse(time.RFC3339, manifest.GeneratedAt); err != nil { - return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("invalid generation time: %w", err) - } - if len(manifest.Files) > maxRepositoryFiles { - return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("repository has more than %d files", maxRepositoryFiles) - } - var repositorySize int64 - for _, file := range manifest.Files { - if file.Size < 0 || file.Size > maxRepositoryBytes-repositorySize { - return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("repository exceeds the %d byte verification limit", maxRepositoryBytes) - } - repositorySize += file.Size - } - - expected := map[string]bool{buildgraph.ManifestFilename: true} - for index, file := range manifest.Files { - if err := validateManifestFile(file, index, manifest.Files); err != nil { - return buildgraph.RepositoryManifest{}, nil, err - } - expected[file.Path] = true - actualSize, actualHash, err := hashRepositoryFile(absolute, file.Path) - if err != nil { - return buildgraph.RepositoryManifest{}, nil, err - } - if actualSize != file.Size || actualHash != file.SHA256 { - return buildgraph.RepositoryManifest{}, nil, fmt.Errorf("repository file %q does not match its manifest", file.Path) - } - } - if buildgraph.TreeDigest(manifest.Files) != manifest.TreeSHA256 { - return buildgraph.RepositoryManifest{}, nil, errors.New("repository tree digest does not match its manifest") - } - if err := rejectUnexpectedFiles(absolute, expected); err != nil { - return buildgraph.RepositoryManifest{}, nil, err - } - // The tree says which rules produced it, so the structure check comes from - // the registry rather than from a chain of comparisons here that a new format - // had to be added to. An identity this build does not know is refused: it is - // a tree from a newer snailmail, not a tree with nothing to check. + // Then the ecosystem's own rules. The tree says which rules produced it, so + // the check comes from the registry rather than from a chain of comparisons + // a new format had to be added to. An identity this build does not know is + // refused: it is a tree from a newer snailmail, not a tree with nothing to + // check. selected, err := formats.ForID(manifest.Format) if err != nil { return buildgraph.RepositoryManifest{}, nil, err @@ -825,100 +763,6 @@ fi return nil } -func validateManifestFile(file buildgraph.ManifestFile, index int, files []buildgraph.ManifestFile) error { - if file.Path == "" || path.IsAbs(file.Path) || path.Clean(file.Path) != file.Path || file.Path == "." || strings.HasPrefix(file.Path, "../") || strings.ContainsRune(file.Path, '\\') { - return fmt.Errorf("repository manifest contains unsafe path %q", file.Path) - } - if file.Path == buildgraph.ManifestFilename { - return fmt.Errorf("repository manifest includes itself") - } - if index > 0 && files[index-1].Path >= file.Path { - return errors.New("repository manifest file list is not uniquely sorted") - } - if file.Size < 0 { - return fmt.Errorf("repository file %q has a negative size", file.Path) - } - decoded, err := hex.DecodeString(file.SHA256) - if err != nil || len(decoded) != sha256.Size { - return fmt.Errorf("repository file %q has an invalid SHA-256", file.Path) - } - return nil -} - -func hashRepositoryFile(root, name string) (int64, string, error) { - filename := filepath.Join(root, filepath.FromSlash(name)) - info, err := os.Lstat(filename) - if err != nil { - return 0, "", fmt.Errorf("inspect repository file %q: %w", name, err) - } - if !info.Mode().IsRegular() { - return 0, "", fmt.Errorf("repository file %q is not a regular file", name) - } - file, err := os.Open(filename) - if err != nil { - return 0, "", fmt.Errorf("open repository file %q: %w", name, err) - } - hash := sha256.New() - size, readErr := io.Copy(hash, file) - closeErr := file.Close() - if readErr != nil { - return 0, "", fmt.Errorf("hash repository file %q: %w", name, readErr) - } - if closeErr != nil { - return 0, "", fmt.Errorf("close repository file %q: %w", name, closeErr) - } - return size, hex.EncodeToString(hash.Sum(nil)), nil -} - -func rejectUnexpectedFiles(root string, expected map[string]bool) error { - var actual []string - err := filepath.WalkDir(root, func(name string, entry fs.DirEntry, walkErr error) error { - if walkErr != nil { - return walkErr - } - if entry.Type()&fs.ModeSymlink != 0 { - return fmt.Errorf("repository contains symbolic link %q", name) - } - if entry.IsDir() { - return nil - } - if !entry.Type().IsRegular() { - return fmt.Errorf("repository contains non-regular file %q", name) - } - relative, err := filepath.Rel(root, name) - if err != nil { - return err - } - actual = append(actual, filepath.ToSlash(relative)) - return nil - }) - if err != nil { - return err - } - sort.Strings(actual) - for _, name := range actual { - if !expected[name] { - return fmt.Errorf("repository contains unexpected file %q", name) - } - delete(expected, name) - } - if len(expected) != 0 { - return errors.New("repository is missing files declared by its manifest") - } - return nil -} - -func requireJSONEOF(decoder *json.Decoder) error { - var extra any - if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { - if err == nil { - return errors.New("repository manifest contains more than one JSON value") - } - return fmt.Errorf("decode repository manifest: %w", err) - } - return nil -} - func snapshotRepository(ctx context.Context, source string, manifest buildgraph.RepositoryManifest) (string, error) { // Beside the repository rather than in the system temp directory, so the // hard links below always have a filesystem to be made on. A snapshot on diff --git a/internal/buildgraph/verify.go b/internal/buildgraph/verify.go new file mode 100644 index 0000000..2249d81 --- /dev/null +++ b/internal/buildgraph/verify.go @@ -0,0 +1,225 @@ +package buildgraph + +// Verifying a generated tree against the manifest at its root. +// +// This is the format-neutral half of verification: that the manifest parses, +// that every file it names is present with the size and digest it claims, that +// nothing else is present, and that the tree digest is the digest of that set. +// It answers "these are the bytes the manifest describes" and nothing more. +// +// Whether those bytes are a valid repository of their ecosystem is a separate +// question, asked through formats.Format.VerifyStructure. Keeping the two apart +// is what lets a host adapter check a stage before sending it without importing +// the application layer to do so: an adapter is about moving bytes, and its +// question is whether the bytes match the plan. + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + maxManifestSize = 8 << 20 + maxRepositoryFiles = 20_000 + maxRepositoryBytes = 4 << 30 +) + +// ReadManifest decodes the manifest at the root of a generated tree. +// +// It returns the bytes it decoded as well as the value, so a caller comparing a +// manifest against one it already holds does not read the file twice and risk +// reading two different states of it. +func ReadManifest(root string) (RepositoryManifest, []byte, error) { + file, err := os.Open(filepath.Join(root, ManifestFilename)) + if err != nil { + return RepositoryManifest{}, nil, fmt.Errorf("open repository manifest: %w", err) + } + content, readErr := io.ReadAll(io.LimitReader(file, maxManifestSize+1)) + closeErr := file.Close() + if readErr != nil { + return RepositoryManifest{}, nil, fmt.Errorf("read repository manifest: %w", readErr) + } + if closeErr != nil { + return RepositoryManifest{}, nil, fmt.Errorf("close repository manifest: %w", closeErr) + } + if len(content) > maxManifestSize { + return RepositoryManifest{}, nil, errors.New("repository manifest exceeds 8 MiB") + } + decoder := json.NewDecoder(bytes.NewReader(content)) + decoder.DisallowUnknownFields() + var manifest RepositoryManifest + if err := decoder.Decode(&manifest); err != nil { + return RepositoryManifest{}, nil, fmt.Errorf("decode repository manifest: %w", err) + } + if err := requireJSONEOF(decoder); err != nil { + return RepositoryManifest{}, nil, err + } + return manifest, content, nil +} + +// VerifyTree checks that every byte under root is what the manifest says. +// +// root may be the managed release symlink rather than the directory itself, so +// it is resolved first: a publication points at an immutable release and the +// bytes being checked are the ones behind the link. +func VerifyTree(root string) (RepositoryManifest, error) { + absolute, err := filepath.Abs(root) + if err != nil { + return RepositoryManifest{}, fmt.Errorf("resolve repository: %w", err) + } + absolute, err = filepath.EvalSymlinks(absolute) + if err != nil { + return RepositoryManifest{}, fmt.Errorf("resolve repository release: %w", err) + } + manifest, _, err := ReadManifest(absolute) + if err != nil { + return RepositoryManifest{}, err + } + if !SupportedManifestSchema(manifest.SchemaVersion) { + return RepositoryManifest{}, fmt.Errorf("unsupported repository schema %d", manifest.SchemaVersion) + } + if manifest.Format == "" { + return RepositoryManifest{}, errors.New("repository manifest has no format") + } + if _, err := time.Parse(time.RFC3339, manifest.GeneratedAt); err != nil { + return RepositoryManifest{}, fmt.Errorf("invalid generation time: %w", err) + } + if len(manifest.Files) > maxRepositoryFiles { + return RepositoryManifest{}, fmt.Errorf("repository has more than %d files", maxRepositoryFiles) + } + var repositorySize int64 + for _, file := range manifest.Files { + if file.Size < 0 || file.Size > maxRepositoryBytes-repositorySize { + return RepositoryManifest{}, fmt.Errorf("repository exceeds the %d byte verification limit", maxRepositoryBytes) + } + repositorySize += file.Size + } + + expected := map[string]bool{ManifestFilename: true} + for index, file := range manifest.Files { + if err := validateManifestFile(file, index, manifest.Files); err != nil { + return RepositoryManifest{}, err + } + expected[file.Path] = true + actualSize, actualHash, err := hashRepositoryFile(absolute, file.Path) + if err != nil { + return RepositoryManifest{}, err + } + if actualSize != file.Size || actualHash != file.SHA256 { + return RepositoryManifest{}, fmt.Errorf("repository file %q does not match its manifest", file.Path) + } + } + if TreeDigest(manifest.Files) != manifest.TreeSHA256 { + return RepositoryManifest{}, errors.New("repository tree digest does not match its manifest") + } + if err := rejectUnexpectedFiles(absolute, expected); err != nil { + return RepositoryManifest{}, err + } + return manifest, nil +} + +func validateManifestFile(file ManifestFile, index int, files []ManifestFile) error { + if file.Path == "" || path.IsAbs(file.Path) || path.Clean(file.Path) != file.Path || file.Path == "." || strings.HasPrefix(file.Path, "../") || strings.ContainsRune(file.Path, '\\') { + return fmt.Errorf("repository manifest contains unsafe path %q", file.Path) + } + if file.Path == ManifestFilename { + return fmt.Errorf("repository manifest includes itself") + } + if index > 0 && files[index-1].Path >= file.Path { + return errors.New("repository manifest file list is not uniquely sorted") + } + if file.Size < 0 { + return fmt.Errorf("repository file %q has a negative size", file.Path) + } + decoded, err := hex.DecodeString(file.SHA256) + if err != nil || len(decoded) != sha256.Size { + return fmt.Errorf("repository file %q has an invalid SHA-256", file.Path) + } + return nil +} + +func hashRepositoryFile(root, name string) (int64, string, error) { + filename := filepath.Join(root, filepath.FromSlash(name)) + info, err := os.Lstat(filename) + if err != nil { + return 0, "", fmt.Errorf("inspect repository file %q: %w", name, err) + } + if !info.Mode().IsRegular() { + return 0, "", fmt.Errorf("repository file %q is not a regular file", name) + } + file, err := os.Open(filename) + if err != nil { + return 0, "", fmt.Errorf("open repository file %q: %w", name, err) + } + hash := sha256.New() + size, readErr := io.Copy(hash, file) + closeErr := file.Close() + if readErr != nil { + return 0, "", fmt.Errorf("hash repository file %q: %w", name, readErr) + } + if closeErr != nil { + return 0, "", fmt.Errorf("close repository file %q: %w", name, closeErr) + } + return size, hex.EncodeToString(hash.Sum(nil)), nil +} + +func rejectUnexpectedFiles(root string, expected map[string]bool) error { + var actual []string + err := filepath.WalkDir(root, func(name string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.Type()&fs.ModeSymlink != 0 { + return fmt.Errorf("repository contains symbolic link %q", name) + } + if entry.IsDir() { + return nil + } + if !entry.Type().IsRegular() { + return fmt.Errorf("repository contains non-regular file %q", name) + } + relative, err := filepath.Rel(root, name) + if err != nil { + return err + } + actual = append(actual, filepath.ToSlash(relative)) + return nil + }) + if err != nil { + return err + } + sort.Strings(actual) + for _, name := range actual { + if !expected[name] { + return fmt.Errorf("repository contains unexpected file %q", name) + } + delete(expected, name) + } + if len(expected) != 0 { + return errors.New("repository is missing files declared by its manifest") + } + return nil +} + +func requireJSONEOF(decoder *json.Decoder) error { + var extra any + if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) { + if err == nil { + return errors.New("repository manifest contains more than one JSON value") + } + return fmt.Errorf("decode repository manifest: %w", err) + } + return nil +} diff --git a/internal/app/commit.go b/internal/release/commit.go similarity index 98% rename from internal/app/commit.go rename to internal/release/commit.go index 5c5def7..d7bfa1d 100644 --- a/internal/app/commit.go +++ b/internal/release/commit.go @@ -1,4 +1,4 @@ -package app +package release import ( "fmt" diff --git a/internal/app/commit_darwin.go b/internal/release/commit_darwin.go similarity index 94% rename from internal/app/commit_darwin.go rename to internal/release/commit_darwin.go index f8c8f81..0c1f4a3 100644 --- a/internal/app/commit_darwin.go +++ b/internal/release/commit_darwin.go @@ -1,6 +1,6 @@ //go:build darwin -package app +package release import "golang.org/x/sys/unix" diff --git a/internal/app/commit_exchange_test.go b/internal/release/commit_exchange_test.go similarity index 99% rename from internal/app/commit_exchange_test.go rename to internal/release/commit_exchange_test.go index 0351551..c73679e 100644 --- a/internal/app/commit_exchange_test.go +++ b/internal/release/commit_exchange_test.go @@ -1,6 +1,6 @@ //go:build linux || darwin -package app +package release import ( "os" diff --git a/internal/app/commit_linux.go b/internal/release/commit_linux.go similarity index 93% rename from internal/app/commit_linux.go rename to internal/release/commit_linux.go index 5a0207d..e3d4b27 100644 --- a/internal/app/commit_linux.go +++ b/internal/release/commit_linux.go @@ -1,6 +1,6 @@ //go:build linux -package app +package release import "golang.org/x/sys/unix" diff --git a/internal/app/commit_other.go b/internal/release/commit_other.go similarity index 96% rename from internal/app/commit_other.go rename to internal/release/commit_other.go index 81feae1..e4a9073 100644 --- a/internal/app/commit_other.go +++ b/internal/release/commit_other.go @@ -1,6 +1,6 @@ //go:build !linux && !darwin -package app +package release import "errors" diff --git a/internal/app/materialize.go b/internal/release/materialize.go similarity index 93% rename from internal/app/materialize.go rename to internal/release/materialize.go index 0ab511d..dfce014 100644 --- a/internal/app/materialize.go +++ b/internal/release/materialize.go @@ -1,11 +1,10 @@ -package app +package release import ( "bytes" "context" "crypto/sha256" "encoding/hex" - "encoding/json" "errors" "fmt" "io" @@ -80,7 +79,11 @@ func materialize(ctx context.Context, output string, artifact domain.RepositoryA // PublishVerifiedDirectory atomically publishes the exact files from a // structurally verified staged tree while enforcing the plan's target tree. func PublishVerifiedDirectory(ctx context.Context, source, output, expectedCurrent, desiredTree string) error { - manifest, err := VerifyRepository(source) + // The bytes, not the ecosystem. What this has to establish is that the + // directory about to become live is the tree the plan named, and the tree + // digest says exactly that. Whether it is a valid repository of its format + // was settled before it was staged, by the engine, with the real client. + manifest, err := buildgraph.VerifyTree(source) if err != nil { return err } @@ -110,30 +113,10 @@ func PublishVerifiedDirectory(ctx context.Context, source, output, expectedCurre } func snapshotVerifiedManifest(root string, expected buildgraph.RepositoryManifest) ([]byte, error) { - file, err := os.Open(filepath.Join(root, buildgraph.ManifestFilename)) + actual, content, err := buildgraph.ReadManifest(root) if err != nil { return nil, err } - content, readErr := io.ReadAll(io.LimitReader(file, maxManifestSize+1)) - closeErr := file.Close() - if readErr != nil { - return nil, readErr - } - if closeErr != nil { - return nil, closeErr - } - if len(content) > maxManifestSize { - return nil, errors.New("repository manifest exceeds 8 MiB") - } - decoder := json.NewDecoder(bytes.NewReader(content)) - decoder.DisallowUnknownFields() - var actual buildgraph.RepositoryManifest - if err := decoder.Decode(&actual); err != nil { - return nil, fmt.Errorf("decode staged repository manifest: %w", err) - } - if err := requireJSONEOF(decoder); err != nil { - return nil, err - } if !reflect.DeepEqual(actual, expected) { return nil, errors.New("staged repository manifest changed after verification") } @@ -249,7 +232,7 @@ func currentManagedRelease(output string) (string, string, error) { return "", "", fmt.Errorf("refusing to replace %q because its current release is unmanaged", output) } target := filepath.Join(control, link) - manifest, err := VerifyRepository(target) + manifest, err := buildgraph.VerifyTree(target) if err != nil { return "", "", fmt.Errorf("refusing to replace invalid repository %q: %w", output, err) } @@ -276,7 +259,7 @@ func removeOrphanedControl(output string) error { } release = filepath.Join(filepath.Dir(output), releaseBase) if _, statErr := os.Lstat(release); statErr == nil { - if _, verifyErr := VerifyRepository(release); verifyErr != nil { + if _, verifyErr := buildgraph.VerifyTree(release); verifyErr != nil { return fmt.Errorf("refusing to remove unverified orphan release %q: %w", release, verifyErr) } } else if !errors.Is(statErr, os.ErrNotExist) { diff --git a/internal/app/materialize_hardening_test.go b/internal/release/materialize_hardening_test.go similarity index 91% rename from internal/app/materialize_hardening_test.go rename to internal/release/materialize_hardening_test.go index 7b0768f..b4a2106 100644 --- a/internal/app/materialize_hardening_test.go +++ b/internal/release/materialize_hardening_test.go @@ -1,4 +1,4 @@ -package app +package release import ( "context" @@ -7,7 +7,6 @@ import ( "testing" "time" - "github.com/shellcell/snailmail/formats/apk" "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/domain" ) @@ -73,7 +72,7 @@ func TestPublishVerifiedDirectoryRecoversOrphanedInitialControl(t *testing.T) { if _, err := os.Lstat(orphan); !os.IsNotExist(err) { t.Fatal("orphaned release was not removed") } - manifest, err := VerifyRepository(target) + manifest, err := buildgraph.VerifyTree(target) if err != nil { t.Fatal(err) } @@ -126,12 +125,12 @@ func TestSnapshotVerifiedManifestRejectsChangedMeaning(t *testing.T) { func writeManagedTestRepository(t *testing.T, output, content string) buildgraph.RepositoryManifest { t.Helper() - // A real format identity, because verification refuses one it does not - // recognise rather than silently skipping the structure check for it. apk is - // the one that asks nothing of a tree's shape, which is what this fixture - // wants: the subject here is the symlink exchange, not an ecosystem's rules. + // Any identity will do, because nothing in this package reads it. Publishing + // a managed release is about the symlink exchange and the tree digest; which + // ecosystem produced the bytes is the engine's question, settled before a + // tree is staged. artifact, manifest, err := buildgraph.Finalize(domain.RepositoryArtifact{ - Format: apk.FormatID, + Format: "test/v1", Files: []domain.File{{Path: "index.txt", Content: []byte(content)}}, }, time.Date(2026, time.July, 23, 1, 2, 3, 0, time.UTC)) if err != nil { From b5da9765e8a75403264d2d6bff9d48c4a8e590f6 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 15:05:00 +0200 Subject: [PATCH 16/26] Give a host the configuration of that host --- adapters/host/githubpages/githubpages.go | 48 ++++---- adapters/host/githubpages/githubpages_test.go | 16 ++- adapters/host/local/local.go | 11 +- adapters/host/rsync/collect_test.go | 2 +- adapters/host/rsync/rsync.go | 9 +- adapters/host/rsync/rsync_test.go | 25 ++-- adapters/host/s3/aws.go | 6 +- adapters/host/s3/collect_test.go | 8 +- adapters/host/s3/helm_test.go | 10 +- adapters/host/s3/s3.go | 30 ++--- adapters/host/s3/s3_test.go | 75 ++++++------ adapters/host/s3/two_runners_test.go | 10 +- engine/workspace.go | 36 ++++-- host/driver_test.go | 44 +++++++ host/host.go | 107 +++++++++++++++--- internal/wire/hosts.go | 9 +- 16 files changed, 301 insertions(+), 145 deletions(-) create mode 100644 host/driver_test.go diff --git a/adapters/host/githubpages/githubpages.go b/adapters/host/githubpages/githubpages.go index 6e6d21b..61048e6 100644 --- a/adapters/host/githubpages/githubpages.go +++ b/adapters/host/githubpages/githubpages.go @@ -51,10 +51,10 @@ func (adapter *Adapter) Capabilities(ctx context.Context, repository host.Reposi return host.Capabilities{}, err } if adapter.verifyProvider { - if err := verifyPagesSite(ctx, repository.RemoteRepository, repository.Branch, repository.CanonicalEndpoint); err != nil { + if err := verifyPagesSite(ctx, repository.Pages.RemoteRepository, repository.Pages.Branch, repository.CanonicalEndpoint); err != nil { return host.Capabilities{}, err } - if err := verifyPagesSite(ctx, repository.PreviewRepository, repository.PreviewBranch, repository.PreviewEndpoint); err != nil { + if err := verifyPagesSite(ctx, repository.Pages.PreviewRepository, repository.Pages.PreviewBranch, repository.Pages.PreviewEndpoint); err != nil { return host.Capabilities{}, err } } @@ -75,7 +75,7 @@ func (adapter *Adapter) Observe(ctx context.Context, repository host.Repository) return host.PublishedRevision{}, err } defer workspace.Close() - commit, err := workspace.fetch(ctx, adapter.remote(repository.RemoteRepository), branchRef(repository.Branch)) + commit, err := workspace.fetch(ctx, adapter.remote(repository.Pages.RemoteRepository), branchRef(repository.Pages.Branch)) if errors.Is(err, errRefNotFound) { return host.PublishedRevision{}, nil } @@ -88,7 +88,7 @@ func (adapter *Adapter) Observe(ctx context.Context, repository host.Repository) } if revision.TreeSHA256 != "" { identifier := effectIdentifier(revision.PlanID, revision.ChangeID) - descriptorCommit, descriptorErr := workspace.fetch(ctx, adapter.remote(repository.RemoteRepository), restoreRef(identifier)) + descriptorCommit, descriptorErr := workspace.fetch(ctx, adapter.remote(repository.Pages.RemoteRepository), restoreRef(identifier)) if descriptorErr != nil { return host.PublishedRevision{}, &host.Error{Kind: host.ErrorIndeterminate, Operation: "observe GitHub Pages restore state", Err: descriptorErr} } @@ -115,7 +115,7 @@ func (adapter *Adapter) ReadAccess(ctx context.Context, repository host.Reposito return host.ClientAccess{}, err } defer workspace.Close() - commit, err := workspace.fetch(ctx, adapter.remote(repository.RemoteRepository), branchRef(repository.Branch)) + commit, err := workspace.fetch(ctx, adapter.remote(repository.Pages.RemoteRepository), branchRef(repository.Pages.Branch)) if err != nil { return host.ClientAccess{}, infrastructure("read GitHub Pages publication", err) } @@ -147,26 +147,26 @@ func (adapter *Adapter) Stage(ctx context.Context, repository host.Repository, r return host.StagedPublication{}, err } identifier := effectIdentifier(request.PlanID, request.ChangeID) - production := adapter.remote(repository.RemoteRepository) + production := adapter.remote(repository.Pages.RemoteRepository) stageRef := stageRef(identifier) if err := workspace.ensureRef(ctx, production, stageRef, commit); err != nil { return host.StagedPublication{}, err } var routes []host.ClientRoute if previewConfigured(repository) { - preview := adapter.remote(repository.PreviewRepository) - if err := workspace.replaceRef(ctx, preview, branchRef(repository.PreviewBranch), commit); err != nil { + preview := adapter.remote(repository.Pages.PreviewRepository) + if err := workspace.replaceRef(ctx, preview, branchRef(repository.Pages.PreviewBranch), commit); err != nil { return host.StagedPublication{}, err } - routes, err = clientRoutes(repository.PreviewEndpoint, request.Files) + routes, err = clientRoutes(repository.Pages.PreviewEndpoint, request.Files) if err != nil { return host.StagedPublication{}, err } } return host.StagedPublication{ - ID: identifier, PlanID: request.PlanID, ChangeID: request.ChangeID, PreviousRevision: request.PreviousRevision, PreviewEndpoint: repository.PreviewEndpoint, + ID: identifier, PlanID: request.PlanID, ChangeID: request.ChangeID, PreviousRevision: request.PreviousRevision, PreviewEndpoint: repository.Pages.PreviewEndpoint, TreeSHA256: request.TreeSHA256, Files: append([]host.File(nil), request.Files...), - CommitPaths: append([]string(nil), request.CommitPaths...), Access: host.ClientAccess{Endpoint: repository.PreviewEndpoint, Routes: routes, PropagationTimeout: 2 * time.Minute}, + CommitPaths: append([]string(nil), request.CommitPaths...), Access: host.ClientAccess{Endpoint: repository.Pages.PreviewEndpoint, Routes: routes, PropagationTimeout: 2 * time.Minute}, }, nil } @@ -182,7 +182,7 @@ func (adapter *Adapter) Commit(ctx context.Context, repository host.Repository, return host.CommitResult{}, err } defer workspace.Close() - remote := adapter.remote(repository.RemoteRepository) + remote := adapter.remote(repository.Pages.RemoteRepository) stageCommit, err := workspace.fetch(ctx, remote, stageRef(staged.ID)) if err != nil { return host.CommitResult{}, infrastructure("read GitHub Pages stage", err) @@ -210,7 +210,7 @@ func (adapter *Adapter) Commit(ctx context.Context, repository host.Repository, return host.CommitResult{}, stale("commit GitHub Pages publication", expected, current) } if current.NativeRevision != "" { - fetchedCurrent, fetchErr := workspace.fetch(ctx, remote, branchRef(repository.Branch)) + fetchedCurrent, fetchErr := workspace.fetch(ctx, remote, branchRef(repository.Pages.Branch)) if fetchErr != nil || fetchedCurrent != current.NativeRevision { return host.CommitResult{}, stale("commit GitHub Pages publication", expected, current) } @@ -222,7 +222,7 @@ func (adapter *Adapter) Commit(ctx context.Context, repository host.Repository, if err := workspace.ensureRef(ctx, remote, restoreRef(staged.ID), descriptorCommit); err != nil { return host.CommitResult{}, err } - if err := workspace.compareAndSwapRef(ctx, remote, branchRef(repository.Branch), current.NativeRevision, stageCommit); err != nil { + if err := workspace.compareAndSwapRef(ctx, remote, branchRef(repository.Pages.Branch), current.NativeRevision, stageCommit); err != nil { published, observeErr := adapter.Observe(ctx, repository) if observeErr == nil && published.NativeRevision == stageCommit && published.TreeSHA256 == staged.TreeSHA256 && published.PlanID == staged.PlanID && published.ChangeID == staged.ChangeID && published.ManifestSHA256 == stagedManifest { access, accessErr := adapter.ReadAccess(ctx, repository, published) @@ -266,7 +266,7 @@ func (adapter *Adapter) Restore(ctx context.Context, repository host.Repository, return host.PublishedRevision{}, err } defer workspace.Close() - remote := adapter.remote(repository.RemoteRepository) + remote := adapter.remote(repository.Pages.RemoteRepository) descriptorCommit, err := workspace.fetch(ctx, remote, restoreRef(restore.ID)) if err != nil { return host.PublishedRevision{}, &host.Error{Kind: host.ErrorIndeterminate, Operation: "read GitHub Pages restore descriptor", Err: err} @@ -275,7 +275,7 @@ func (adapter *Adapter) Restore(ctx context.Context, repository host.Repository, if err != nil || digestString(descriptorCommit) != restore.DescriptorSHA256 || descriptor.PlanID != restore.PlanID || descriptor.ChangeID != restore.ChangeID || descriptor.TreeSHA256 != restore.FailedTree || descriptor.ManifestSHA256 != current.ManifestSHA256 || descriptor.PreviousRevision != parent { return host.PublishedRevision{}, &host.Error{Kind: host.ErrorIndeterminate, Operation: "validate GitHub Pages restore descriptor", Err: errors.New("restore descriptor does not match failed publication")} } - if err := workspace.compareAndSwapRef(ctx, remote, branchRef(repository.Branch), current.NativeRevision, parent); err != nil { + if err := workspace.compareAndSwapRef(ctx, remote, branchRef(repository.Pages.Branch), current.NativeRevision, parent); err != nil { observed, observeErr := adapter.Observe(ctx, repository) if observeErr == nil && observed.NativeRevision == parent { return observed, nil @@ -297,7 +297,7 @@ func (adapter *Adapter) Abort(ctx context.Context, repository host.Repository, s return err } defer workspace.Close() - remote := adapter.remote(repository.RemoteRepository) + remote := adapter.remote(repository.Pages.RemoteRepository) commit, err := workspace.lsRemote(ctx, remote, stageRef(staged.ID)) if errors.Is(err, errRefNotFound) { return nil @@ -333,7 +333,7 @@ func commitResult(repository host.Repository, revision host.PublishedRevision, i // Any one of the three fields means yes, so a half-filled preview is caught by // validation rather than silently ignored. func previewConfigured(repository host.Repository) bool { - return repository.PreviewRepository != "" || repository.PreviewBranch != "" || repository.PreviewEndpoint != "" + return repository.Pages.PreviewRepository != "" || repository.Pages.PreviewBranch != "" || repository.Pages.PreviewEndpoint != "" } func validateRepository(repository host.Repository) error { @@ -342,21 +342,23 @@ func validateRepository(repository host.Repository) error { if repository.Type != "github-pages" || !host.Supports(repository.Type, repository.Format).Publish { return invalid("configure GitHub Pages host", "GitHub Pages does not serve format "+repository.Format) } + if repository.Pages == nil { + return invalid("configure GitHub Pages host", "repository is not configured for the GitHub Pages host") + } if repository.Visibility != "public" { return invalid("configure GitHub Pages host", "GitHub Pages currently supports public repositories only") } - if !validRepositoryName(repository.RemoteRepository) || !validBranch(repository.Branch) || - repository.Path != "" || repository.Bucket != "" || repository.Prefix != "" || repository.Region != "" || repository.Endpoint != "" || repository.UsePathStyle || repository.ReadAuth != "" || repository.CredentialBroker != "" { + if !validRepositoryName(repository.Pages.RemoteRepository) || !validBranch(repository.Pages.Branch) { return invalid("configure GitHub Pages host", "invalid production repository configuration") } endpoints := []string{repository.CanonicalEndpoint} // A preview is optional. Without one there is no second site to stage to, // and the caller verifies the staged tree it already holds instead. if previewConfigured(repository) { - if !validRepositoryName(repository.PreviewRepository) || strings.EqualFold(repository.RemoteRepository, repository.PreviewRepository) || !validBranch(repository.PreviewBranch) { + if !validRepositoryName(repository.Pages.PreviewRepository) || strings.EqualFold(repository.Pages.RemoteRepository, repository.Pages.PreviewRepository) || !validBranch(repository.Pages.PreviewBranch) { return invalid("configure GitHub Pages host", "invalid preview repository configuration") } - endpoints = append(endpoints, repository.PreviewEndpoint) + endpoints = append(endpoints, repository.Pages.PreviewEndpoint) } for _, endpoint := range endpoints { parsed, err := url.Parse(endpoint) @@ -365,7 +367,7 @@ func validateRepository(repository host.Repository) error { return invalid("configure GitHub Pages host", "Pages endpoints must use HTTPS without credentials, query, or fragment") } } - if strings.TrimSuffix(repository.CanonicalEndpoint, "/") == strings.TrimSuffix(repository.PreviewEndpoint, "/") { + if strings.TrimSuffix(repository.CanonicalEndpoint, "/") == strings.TrimSuffix(repository.Pages.PreviewEndpoint, "/") { return invalid("configure GitHub Pages host", "production and preview endpoints must be distinct") } return nil diff --git a/adapters/host/githubpages/githubpages_test.go b/adapters/host/githubpages/githubpages_test.go index 0a8d10d..6ef56d0 100644 --- a/adapters/host/githubpages/githubpages_test.go +++ b/adapters/host/githubpages/githubpages_test.go @@ -36,7 +36,7 @@ func TestGitHubPagesStageCommitRestoreContract(t *testing.T) { if err != nil { t.Fatal(err) } - assertRemotePublication(t, adapter, preview, repository.PreviewBranch, first.TreeSHA256) + assertRemotePublication(t, adapter, preview, repository.Pages.PreviewBranch, first.TreeSHA256) if observed, err := adapter.Observe(ctx, repository); err != nil || observed != (host.PublishedRevision{}) { t.Fatalf("initial observation=%#v err=%v", observed, err) } @@ -90,11 +90,11 @@ func TestGitHubPagesRejectsPrivateAndSamePreviewRepository(t *testing.T) { t.Fatal("private Pages repository was accepted") } repository.Visibility = "public" - repository.PreviewRepository = repository.RemoteRepository + repository.Pages.PreviewRepository = repository.Pages.RemoteRepository if _, err := New().Capabilities(context.Background(), repository); err == nil { t.Fatal("production repository reused as preview site") } - repository.PreviewRepository = strings.ToUpper(repository.RemoteRepository) + repository.Pages.PreviewRepository = strings.ToUpper(repository.Pages.RemoteRepository) if _, err := New().Capabilities(context.Background(), repository); err == nil { t.Fatal("case-variant production repository reused as preview site") } @@ -189,9 +189,7 @@ func pagesStageFixture(t *testing.T, label, version string) host.StageRequest { func testRepository() host.Repository { return host.Repository{ Name: "python", Format: "pypi", Type: "github-pages", Visibility: "public", - RemoteRepository: "test/production", Branch: "gh-pages", CanonicalEndpoint: "https://test.example/packages", - PreviewRepository: "test/preview", PreviewBranch: "gh-pages", PreviewEndpoint: "https://preview.example/packages", - } + CanonicalEndpoint: "https://test.example/packages", Pages: &host.PagesConfig{RemoteRepository: "test/production", Branch: "gh-pages", PreviewRepository: "test/preview", PreviewBranch: "gh-pages", PreviewEndpoint: "https://preview.example/packages"}} } func expectedRevision(revision host.PublishedRevision) host.ExpectedRevision { @@ -239,9 +237,9 @@ func TestGitHubPagesStagesWithoutAPreview(t *testing.T) { return "" }) repository := testRepository() - repository.PreviewRepository = "" - repository.PreviewBranch = "" - repository.PreviewEndpoint = "" + repository.Pages.PreviewRepository = "" + repository.Pages.PreviewBranch = "" + repository.Pages.PreviewEndpoint = "" request := pagesStageFixture(t, "solo", "1.0.0") staged, err := adapter.Stage(ctx, repository, request) diff --git a/adapters/host/local/local.go b/adapters/host/local/local.go index 3f411a8..b78c16b 100644 --- a/adapters/host/local/local.go +++ b/adapters/host/local/local.go @@ -86,14 +86,17 @@ func (adapter *Adapter) Abort(context.Context, host.Repository, host.StagedPubli } func localPath(repository host.Repository) (string, error) { - if repository.Path == "" { + if repository.Local == nil { + return "", errors.New("repository is not configured for the local host") + } + if repository.Local.Path == "" { return "", errors.New("local host path is required") } - if filepath.IsAbs(repository.Path) { - return filepath.Clean(repository.Path), nil + if filepath.IsAbs(repository.Local.Path) { + return filepath.Clean(repository.Local.Path), nil } if repository.WorkspaceRoot == "" { return "", errors.New("local host workspace root is required") } - return state.WorkspacePath(repository.WorkspaceRoot, filepath.ToSlash(repository.Path)) + return state.WorkspacePath(repository.WorkspaceRoot, filepath.ToSlash(repository.Local.Path)) } diff --git a/adapters/host/rsync/collect_test.go b/adapters/host/rsync/collect_test.go index e0427cc..4a62bca 100644 --- a/adapters/host/rsync/collect_test.go +++ b/adapters/host/rsync/collect_test.go @@ -57,7 +57,7 @@ func TestCollectRemovesSupersededReleases(t *testing.T) { } // And the repository is still being served, which is what a wrong collection // here would break. - if _, err := os.ReadFile(filepath.Join(repository.Path, "simple", "index.html")); err != nil { + if _, err := os.ReadFile(filepath.Join(repository.Rsync.Path, "simple", "index.html")); err != nil { t.Errorf("the live revision is no longer served: %v", err) } } diff --git a/adapters/host/rsync/rsync.go b/adapters/host/rsync/rsync.go index 36331a1..dbf4c35 100644 --- a/adapters/host/rsync/rsync.go +++ b/adapters/host/rsync/rsync.go @@ -288,15 +288,18 @@ func (adapter *Adapter) Abort(ctx context.Context, repository host.Repository, s // remoteRoot is the published path on the far side. func remoteRoot(repository host.Repository) (string, error) { - if repository.Path == "" { + if repository.Rsync == nil { + return "", invalid("configure rsync host", errors.New("repository is not configured for the rsync host")) + } + if repository.Rsync.Path == "" { return "", invalid("configure rsync host", errors.New("rsync host path is required")) } - if !strings.HasPrefix(repository.Path, "/") { + if !strings.HasPrefix(repository.Rsync.Path, "/") { // Relative to a remote home directory is ambiguous — whose home depends on // the ssh user, which ssh_config may change without this adapter knowing. return "", invalid("configure rsync host", errors.New("rsync host path must be absolute")) } - cleaned := path.Clean(repository.Path) + cleaned := path.Clean(repository.Rsync.Path) if cleaned == "/" { return "", invalid("configure rsync host", errors.New("rsync host path must not be the filesystem root")) } diff --git a/adapters/host/rsync/rsync_test.go b/adapters/host/rsync/rsync_test.go index 060bdc6..b66c282 100644 --- a/adapters/host/rsync/rsync_test.go +++ b/adapters/host/rsync/rsync_test.go @@ -107,8 +107,8 @@ func publishedRepository(t *testing.T) (host.Repository, string) { base := t.TempDir() return host.Repository{ Name: "apt", Format: "raw", Type: "rsync", - Path: filepath.Join(base, "www"), CanonicalEndpoint: "https://packages.example/apt", + Rsync: &host.RsyncConfig{Path: filepath.Join(base, "www")}, }, base } @@ -136,14 +136,14 @@ func TestAPublicationBecomesLiveThroughASymlink(t *testing.T) { if err != nil { t.Fatal(err) } - info, err := os.Lstat(repository.Path) + info, err := os.Lstat(repository.Rsync.Path) if err != nil { t.Fatal(err) } if info.Mode()&os.ModeSymlink == 0 { t.Error("the published path is not a symlink, so a publication is not atomic") } - body, err := os.ReadFile(filepath.Join(repository.Path, "simple", "index.html")) + body, err := os.ReadFile(filepath.Join(repository.Rsync.Path, "simple", "index.html")) if err != nil { t.Fatal(err) } @@ -175,7 +175,7 @@ func TestASecondPublicationReplacesTheFirst(t *testing.T) { host.ExpectedRevision{TreeSHA256: first.Revision.TreeSHA256}); err != nil { t.Fatal(err) } - body, err := os.ReadFile(filepath.Join(repository.Path, "simple", "rsync-demo", "index.html")) + body, err := os.ReadFile(filepath.Join(repository.Rsync.Path, "simple", "rsync-demo", "index.html")) if err != nil { t.Fatal(err) } @@ -184,7 +184,7 @@ func TestASecondPublicationReplacesTheFirst(t *testing.T) { } // Not nested: a rename without -T would have moved the new symlink inside the // directory the old one pointed at, leaving the old revision served. - if _, err := os.Stat(filepath.Join(repository.Path, filepath.Base(repository.Path))); err == nil { + if _, err := os.Stat(filepath.Join(repository.Rsync.Path, filepath.Base(repository.Rsync.Path))); err == nil { t.Error("the new symlink was moved inside the old revision instead of replacing it") } } @@ -214,7 +214,7 @@ func TestAStalePublicationIsRefused(t *testing.T) { if !errors.As(err, &hostErr) || hostErr.Kind != host.ErrorStale { t.Errorf("error kind = %v, want stale", err) } - body, _ := os.ReadFile(filepath.Join(repository.Path, "simple", "rsync-demo", "index.html")) + body, _ := os.ReadFile(filepath.Join(repository.Rsync.Path, "simple", "rsync-demo", "index.html")) if !bytes.Contains(body, []byte("2.0.0")) { t.Error("the live revision changed despite the refusal") } @@ -271,10 +271,10 @@ func TestTheLockIsReleasedAfterAPublication(t *testing.T) { func TestAnUnrelatedDirectoryIsNotReplaced(t *testing.T) { adapter := New(&localRunner{}) repository, _ := publishedRepository(t) - if err := os.MkdirAll(repository.Path, 0o755); err != nil { + if err := os.MkdirAll(repository.Rsync.Path, 0o755); err != nil { t.Fatal(err) } - existing := filepath.Join(repository.Path, "somebody-elses-file") + existing := filepath.Join(repository.Rsync.Path, "somebody-elses-file") if err := os.WriteFile(existing, []byte("keep me"), 0o644); err != nil { t.Fatal(err) } @@ -301,9 +301,10 @@ func TestAnAbsentPathIsTheFirstPublication(t *testing.T) { func TestRefusedConfigurations(t *testing.T) { adapter := New(&localRunner{}) for name, repository := range map[string]host.Repository{ - "no path": {Name: "apt"}, - "relative path": {Name: "apt", Path: "www/apt"}, - "filesystem root": {Name: "apt", Path: "/"}, + "not an rsync repository": {Name: "apt"}, + "no path": {Name: "apt", Rsync: &host.RsyncConfig{}}, + "relative path": {Name: "apt", Rsync: &host.RsyncConfig{Path: "www/apt"}}, + "filesystem root": {Name: "apt", Rsync: &host.RsyncConfig{Path: "/"}}, } { if _, err := adapter.Capabilities(context.Background(), repository); err == nil { t.Errorf("%s was accepted", name) @@ -320,7 +321,7 @@ func TestObserveIgnoresASymlinkItDidNotWrite(t *testing.T) { if err := os.MkdirAll(elsewhere, 0o755); err != nil { t.Fatal(err) } - if err := os.Symlink(elsewhere, repository.Path); err != nil { + if err := os.Symlink(elsewhere, repository.Rsync.Path); err != nil { t.Fatal(err) } observed, err := adapter.Observe(context.Background(), repository) diff --git a/adapters/host/s3/aws.go b/adapters/host/s3/aws.go index cb6f50c..d7b317e 100644 --- a/adapters/host/s3/aws.go +++ b/adapters/host/s3/aws.go @@ -32,13 +32,13 @@ func NewAWS(ctx context.Context, repository host.Repository, brokers ...host.Cre return nil, err } client, err := awss3.NewClient(ctx, awss3.Config{ - Bucket: repository.Bucket, Region: repository.Region, - Endpoint: repository.Endpoint, UsePathStyle: repository.UsePathStyle, + Bucket: repository.S3.Bucket, Region: repository.S3.Region, + Endpoint: repository.S3.Endpoint, UsePathStyle: repository.S3.UsePathStyle, }) if err != nil { return nil, err } - return New(&AWSClient{client: client, bucket: repository.Bucket}, brokers...), nil + return New(&AWSClient{client: client, bucket: repository.S3.Bucket}, brokers...), nil } func (client *AWSClient) Head(ctx context.Context, key string) (ObjectInfo, error) { diff --git a/adapters/host/s3/collect_test.go b/adapters/host/s3/collect_test.go index 04e519c..61d6799 100644 --- a/adapters/host/s3/collect_test.go +++ b/adapters/host/s3/collect_test.go @@ -49,9 +49,8 @@ func publishTwice(t *testing.T) (*Adapter, *memoryObjects, host.Repository, host adapter := New(objects) repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), - Type: "s3", Visibility: "public", Bucket: "packages", Prefix: "repo", - CanonicalEndpoint: "https://packages.example/repo", - } + Type: "s3", Visibility: "public", + CanonicalEndpoint: "https://packages.example/repo", S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}} first := stageFixture(t, "plan-1", "", "", `first`) firstStage, err := adapter.Stage(ctx, repository, first) if err != nil { @@ -215,8 +214,7 @@ func TestCollectRefusesARepositoryItDidNotPublish(t *testing.T) { adapter := New(newMemoryObjects()) repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), - Type: "s3", Visibility: "public", Bucket: "packages", CanonicalEndpoint: "https://packages.example", - } + Type: "s3", Visibility: "public", CanonicalEndpoint: "https://packages.example", S3: &host.S3Config{Bucket: "packages"}} if _, err := adapter.Collect(context.Background(), repository, host.Retention{}); err == nil { t.Error("collection ran against a repository with no managed revision") } diff --git a/adapters/host/s3/helm_test.go b/adapters/host/s3/helm_test.go index 5f65141..a560f14 100644 --- a/adapters/host/s3/helm_test.go +++ b/adapters/host/s3/helm_test.go @@ -28,9 +28,8 @@ const helmRootPath = "index.yaml" func helmRepository(endpoint string) host.Repository { return host.Repository{ Name: "charts", Format: "helm", CommitPaths: []string{helmRootPath}, - Type: "s3", Visibility: "public", Bucket: "packages", Prefix: "repo", - CanonicalEndpoint: endpoint, - } + Type: "s3", Visibility: "public", + CanonicalEndpoint: endpoint, S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}} } func helmStageFixture(t *testing.T, planID, chart, version string) host.StageRequest { @@ -288,10 +287,9 @@ func TestASignedYumRepositoryIsRefusedByItsPathCount(t *testing.T) { ctx := context.Background() adapter := New(newMemoryObjects()) signed := host.Repository{ - Name: "yum", Format: "rpm", Type: "s3", Visibility: "public", Bucket: "b", + Name: "yum", Format: "rpm", Type: "s3", Visibility: "public", CanonicalEndpoint: "https://yum.example", - CommitPaths: []string{"repodata/repomd.xml", "repodata/repomd.xml.asc"}, - } + CommitPaths: []string{"repodata/repomd.xml", "repodata/repomd.xml.asc"}, S3: &host.S3Config{Bucket: "b"}} _, err := adapter.Capabilities(ctx, signed) if !host.IsKind(err, host.ErrorInvalidConfiguration) { t.Fatalf("a signed yum repository was accepted: %v", err) diff --git a/adapters/host/s3/s3.go b/adapters/host/s3/s3.go index fdc76a6..b1eb0c5 100644 --- a/adapters/host/s3/s3.go +++ b/adapters/host/s3/s3.go @@ -240,7 +240,7 @@ func (adapter *Adapter) ReadAccess(ctx context.Context, repository host.Reposito } return adapter.issueAccess(ctx, repository, host.ReadScope{ WorkspaceID: repository.WorkspaceID, Repository: repository.Name, HostIdentity: repository.HostIdentity, - Bucket: repository.Bucket, Endpoint: repository.CanonicalEndpoint, PlanID: revision.PlanID, ChangeID: revision.ChangeID, TreeSHA256: revision.TreeSHA256, + Bucket: repository.S3.Bucket, Endpoint: repository.CanonicalEndpoint, PlanID: revision.PlanID, ChangeID: revision.ChangeID, TreeSHA256: revision.TreeSHA256, Prefixes: readPrefixes(repository, revision.TreeSHA256, rootPath), }, routes) } @@ -471,7 +471,7 @@ func (adapter *Adapter) Commit(ctx context.Context, repository host.Repository, } access, err := adapter.issueAccess(ctx, repository, host.ReadScope{ WorkspaceID: repository.WorkspaceID, Repository: repository.Name, HostIdentity: repository.HostIdentity, - Bucket: repository.Bucket, Endpoint: repository.CanonicalEndpoint, PlanID: descriptor.PlanID, ChangeID: descriptor.ChangeID, TreeSHA256: descriptor.TreeSHA256, + Bucket: repository.S3.Bucket, Endpoint: repository.CanonicalEndpoint, PlanID: descriptor.PlanID, ChangeID: descriptor.ChangeID, TreeSHA256: descriptor.TreeSHA256, Prefixes: readPrefixes(repository, descriptor.TreeSHA256, rootPath), }, routes) if err != nil { @@ -1109,26 +1109,30 @@ func (adapter *Adapter) acquireCommitLock(ctx context.Context, repository host.R func validateRepository(repository host.Repository) error { // Configuration validation rejects an unsupported pair earlier; this is the // adapter refusing to act on one that reached it anyway. + if repository.S3 == nil { + return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", + Err: errors.New("repository is not configured for the S3 host")} + } if repository.Type != "s3" || !host.Supports(repository.Type, repository.Format).Publish { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: fmt.Errorf("S3 does not serve format %q", repository.Format)} } if repository.Visibility != "public" && repository.Visibility != "private" { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("S3 visibility must be public or private")} } - if repository.Visibility == "private" && (repository.ReadAuth != "basic" || repository.CredentialBroker == "") { + if repository.Visibility == "private" && (repository.S3.ReadAuth != "basic" || repository.S3.CredentialBroker == "") { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("private S3 reads require a Basic credential broker")} } if repository.Visibility == "private" && (!hexdigest.ValidSHA256(repository.WorkspaceID) || !hexdigest.ValidSHA256(repository.HostIdentity)) { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("private S3 reads require workspace and host identities")} } - if repository.Path != "" || repository.Bucket == "" || repository.CanonicalEndpoint == "" { + if repository.S3.Bucket == "" || repository.CanonicalEndpoint == "" { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("bucket and canonical endpoint are required")} } - if hasControl(repository.Bucket) || hasControl(repository.Region) { + if hasControl(repository.S3.Bucket) || hasControl(repository.S3.Region) { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("bucket and region must not contain control characters")} } - prefix := strings.Trim(repository.Prefix, "/") - if prefix != repository.Prefix || strings.ContainsRune(prefix, '\\') || hasControl(prefix) || (prefix != "" && (path.Clean(prefix) != prefix || strings.HasPrefix(prefix, "../"))) { + prefix := strings.Trim(repository.S3.Prefix, "/") + if prefix != repository.S3.Prefix || strings.ContainsRune(prefix, '\\') || hasControl(prefix) || (prefix != "" && (path.Clean(prefix) != prefix || strings.HasPrefix(prefix, "../"))) { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("S3 prefix is invalid")} } if err := validateHTTPURL(repository.CanonicalEndpoint); err != nil { @@ -1138,11 +1142,11 @@ func validateRepository(repository host.Repository) error { if parsed.Scheme != "https" && !(parsed.Scheme == "http" && isLoopbackHost(parsed.Hostname())) { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("client endpoint must use HTTPS")} } - if repository.Endpoint != "" { - if err := validateHTTPURL(repository.Endpoint); err != nil { + if repository.S3.Endpoint != "" { + if err := validateHTTPURL(repository.S3.Endpoint); err != nil { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: fmt.Errorf("S3 endpoint: %w", err)} } - parsed, _ := url.Parse(repository.Endpoint) + parsed, _ := url.Parse(repository.S3.Endpoint) if parsed.Scheme != "https" && !(parsed.Scheme == "http" && isLoopbackHost(parsed.Hostname())) { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("S3 API endpoint must use HTTPS")} } @@ -1584,7 +1588,7 @@ func (adapter *Adapter) stageResult(ctx context.Context, repository host.Reposit } access, err := adapter.issueAccess(ctx, repository, host.ReadScope{ WorkspaceID: repository.WorkspaceID, Repository: repository.Name, HostIdentity: repository.HostIdentity, - Bucket: repository.Bucket, Endpoint: repository.CanonicalEndpoint, PlanID: descriptor.PlanID, ChangeID: descriptor.ChangeID, + Bucket: repository.S3.Bucket, Endpoint: repository.CanonicalEndpoint, PlanID: descriptor.PlanID, ChangeID: descriptor.ChangeID, StageID: identifier, TreeSHA256: descriptor.TreeSHA256, Prefixes: []string{stageKey(repository, identifier, "")}, }, routes) if err != nil { @@ -1700,10 +1704,10 @@ func clientRoute(endpoint string, file host.File, content []byte) (host.ClientRo } func objectKey(repository host.Repository, name string) string { - if repository.Prefix == "" { + if repository.S3.Prefix == "" { return path.Clean(name) } - return path.Join(strings.Trim(repository.Prefix, "/"), name) + return path.Join(strings.Trim(repository.S3.Prefix, "/"), name) } func stageKey(repository host.Repository, identifier, name string) string { diff --git a/adapters/host/s3/s3_test.go b/adapters/host/s3/s3_test.go index 4cebccd..fc4cc53 100644 --- a/adapters/host/s3/s3_test.go +++ b/adapters/host/s3/s3_test.go @@ -49,7 +49,7 @@ func TestS3HostStageCommitRestoreContract(t *testing.T) { defer server.Close() repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Type: "s3", Visibility: "public", - Bucket: "packages", Prefix: "repo", CanonicalEndpoint: server.URL + "/repo", + CanonicalEndpoint: server.URL + "/repo", S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}, } adapter := New(objects) firstRoot := `first` @@ -119,7 +119,7 @@ func TestS3HostRejectsStaleCommitAndRestore(t *testing.T) { objects := newMemoryObjects() repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Type: "s3", Visibility: "public", - Bucket: "packages", Prefix: "repo", CanonicalEndpoint: "https://packages.example/repo", + CanonicalEndpoint: "https://packages.example/repo", S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}, } adapter := New(objects) firstRequest := stageFixture(t, "plan-a", "", "", `first`) @@ -165,7 +165,7 @@ func TestS3HostRejectsRestoreToIncompletePriorRelease(t *testing.T) { objects := newMemoryObjects() repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Type: "s3", Visibility: "public", - Bucket: "packages", Prefix: "repo", CanonicalEndpoint: "https://packages.example/repo", + CanonicalEndpoint: "https://packages.example/repo", S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}, } adapter := New(objects) firstRequest := stageFixture(t, "restore-source", "", "", `first`) @@ -203,7 +203,7 @@ func TestS3HostAbortMarksSharedStageForLifecycleCleanup(t *testing.T) { objects := newMemoryObjects() repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Type: "s3", Visibility: "public", - Bucket: "packages", Prefix: "repo", CanonicalEndpoint: "https://packages.example/repo", + CanonicalEndpoint: "https://packages.example/repo", S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}, } adapter := New(objects) staged, err := adapter.Stage(ctx, repository, stageFixture(t, "plan-d", "", "", `content`)) @@ -231,7 +231,7 @@ func TestS3HostFailedReleaseMaterializationLeavesCanonicalRoot(t *testing.T) { objects := newMemoryObjects() repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Type: "s3", Visibility: "public", - Bucket: "packages", Prefix: "repo", CanonicalEndpoint: "https://packages.example/repo", + CanonicalEndpoint: "https://packages.example/repo", S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}, } adapter := New(objects) firstRequest := stageFixture(t, "plan-e", "", "", `first`) @@ -271,7 +271,7 @@ func TestS3HostDetectsImmutableReleaseDrift(t *testing.T) { objects := newMemoryObjects() repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Type: "s3", Visibility: "public", - Bucket: "packages", Prefix: "repo", CanonicalEndpoint: "https://packages.example/repo", + CanonicalEndpoint: "https://packages.example/repo", S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}, } adapter := New(objects) request := stageFixture(t, "plan-7", "", "", `content`) @@ -307,7 +307,7 @@ func TestS3HostRejectsFileChangedBeforeStage(t *testing.T) { if err := os.WriteFile(root, content, 0o644); err != nil { t.Fatal(err) } - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} if _, err := New(newMemoryObjects()).Stage(context.Background(), repository, request); !host.IsKind(err, host.ErrorStale) { t.Fatalf("changed stage file error = %v", err) } @@ -320,9 +320,9 @@ func TestS3HostRejectsFileChangedBeforeStage(t *testing.T) { func TestS3HostRejectsPrivateWithoutBrokerAndMultiPathFormats(t *testing.T) { adapter := New(newMemoryObjects()) for _, repository := range []host.Repository{ - {Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "private", Bucket: "b", CanonicalEndpoint: "https://example.test"}, + {Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "private", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}}, {Type: "s3", Format: "deb", CommitPaths: []string{"dists/stable/InRelease", "dists/stable/Release.gpg"}, - Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"}, + Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}}, } { if _, err := adapter.Capabilities(context.Background(), repository); !host.IsKind(err, host.ErrorInvalidConfiguration) { t.Fatalf("configuration error = %v", err) @@ -335,8 +335,9 @@ func TestS3HostIssuesScopedPrivateReadCredentials(t *testing.T) { objects := newMemoryObjects() broker := &recordingCredentialBroker{} repository := host.Repository{ - Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Type: "s3", Visibility: "private", Bucket: "packages", Prefix: "repo", - CanonicalEndpoint: "https://packages.example/repo", ReadAuth: "basic", CredentialBroker: "default", + Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), + Type: "s3", Visibility: "private", CanonicalEndpoint: "https://packages.example/repo", + S3: &host.S3Config{Bucket: "packages", Prefix: "repo", ReadAuth: "basic", CredentialBroker: "default"}, WorkspaceID: strings.Repeat("b", 64), HostIdentity: strings.Repeat("c", 64), } adapter := New(objects, broker) @@ -398,8 +399,9 @@ func TestS3HostCredentialFailureDoesNotPublishEffectPointer(t *testing.T) { objects := newMemoryObjects() broker := &recordingCredentialBroker{issueErr: errors.New("temporary broker failure")} repository := host.Repository{ - Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Type: "s3", Visibility: "private", Bucket: "packages", Prefix: "repo", - CanonicalEndpoint: "https://packages.example/repo", ReadAuth: "basic", CredentialBroker: "default", + Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), + Type: "s3", Visibility: "private", CanonicalEndpoint: "https://packages.example/repo", + S3: &host.S3Config{Bucket: "packages", Prefix: "repo", ReadAuth: "basic", CredentialBroker: "default"}, WorkspaceID: strings.Repeat("b", 64), HostIdentity: strings.Repeat("c", 64), } adapter := New(objects, broker) @@ -425,7 +427,7 @@ func TestS3HostAdoptsAndRestoresUnmanagedRoot(t *testing.T) { objects := newMemoryObjects() repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Type: "s3", Visibility: "public", - Bucket: "packages", Prefix: "repo", CanonicalEndpoint: "https://packages.example/repo", + CanonicalEndpoint: "https://packages.example/repo", S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}, } unmanaged := []byte(`legacy`) root, err := objects.Put(ctx, PutRequest{ @@ -468,7 +470,7 @@ func TestS3HostAdoptsAndRestoresUnmanagedRoot(t *testing.T) { func TestS3HostRejectsTreeDigestOutsideDescriptor(t *testing.T) { request := stageFixture(t, "invalid-tree", "", "", `content`) request.TreeSHA256 = strings.Repeat("9", 64) - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} if _, err := New(newMemoryObjects()).Stage(context.Background(), repository, request); !host.IsKind(err, host.ErrorInvalidConfiguration) { t.Fatalf("tree mismatch error = %v", err) } @@ -477,7 +479,7 @@ func TestS3HostRejectsTreeDigestOutsideDescriptor(t *testing.T) { func TestS3HostRejectsTamperedRestoreBytes(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} adapter := New(objects) firstRequest := stageFixture(t, "restore-first", "", "", `first`) firstStage, err := adapter.Stage(ctx, repository, firstRequest) @@ -513,7 +515,7 @@ func TestS3HostRejectsTamperedRestoreBytes(t *testing.T) { func TestS3HostRecoversAmbiguousStagePointerWrite(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} request := stageFixture(t, "ambiguous-stage", "", "", `content`) objects.ambiguousPutKey = stagePointerKey(repository, effectIdentifier(request.PlanID, request.ChangeID)) adapter := New(objects) @@ -530,7 +532,7 @@ func TestS3HostRecoversAmbiguousStagePointerWrite(t *testing.T) { func TestS3HostRootBodyBindsPublicationManifest(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} adapter := New(objects) firstRequest := stageFixture(t, "binding-first", "", "", `content`) firstStage, err := adapter.Stage(ctx, repository, firstRequest) @@ -567,7 +569,7 @@ func TestS3HostRootBodyBindsPublicationManifest(t *testing.T) { func TestS3HostCreateOnlyPromotionRejectsImmutableConflict(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} adapter := New(objects) request := stageFixture(t, "immutable-conflict", "", "", `content`) staged, err := adapter.Stage(ctx, repository, request) @@ -594,7 +596,7 @@ func TestS3HostCreateOnlyPromotionRejectsImmutableConflict(t *testing.T) { func TestS3HostRejectsSemanticallyInvalidPublicationManifest(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} adapter := New(objects) request := stageFixture(t, "invalid-manifest", "", "", `content`) manifestName := filepath.Join(request.Directory, buildgraph.ManifestFilename) @@ -639,7 +641,7 @@ func TestS3HostRejectsSemanticallyInvalidPublicationManifest(t *testing.T) { func TestS3HostMissingBoundReleaseIsIndeterminate(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} adapter := New(objects) request := stageFixture(t, "missing-release", "", "", `content`) staged, err := adapter.Stage(ctx, repository, request) @@ -659,18 +661,25 @@ func TestS3HostMissingBoundReleaseIsIndeterminate(t *testing.T) { func TestS3HostRejectsInvalidDirectConfiguration(t *testing.T) { adapter := New(newMemoryObjects()) - valid := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + valid := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} for _, mutate := range []func(*host.Repository){ - func(repository *host.Repository) { repository.Path = "local" }, - func(repository *host.Repository) { repository.Prefix = "/absolute" }, - func(repository *host.Repository) { repository.Prefix = "a/../b" }, - func(repository *host.Repository) { repository.Prefix = "a\\b" }, - func(repository *host.Repository) { repository.Prefix = "a\tb" }, + // A local path used to be a case here, because a repository could carry + // one alongside a bucket. It cannot now: the configuration a host is given + // is the configuration of that host. + func(repository *host.Repository) { repository.S3 = nil }, + func(repository *host.Repository) { repository.S3.Prefix = "/absolute" }, + func(repository *host.Repository) { repository.S3.Prefix = "a/../b" }, + func(repository *host.Repository) { repository.S3.Prefix = "a\\b" }, + func(repository *host.Repository) { repository.S3.Prefix = "a\tb" }, func(repository *host.Repository) { repository.CanonicalEndpoint = "https://user@example.test/repo" }, func(repository *host.Repository) { repository.CanonicalEndpoint = "https://example.test/repo?query=1" }, - func(repository *host.Repository) { repository.Endpoint = "ftp://example.test" }, + func(repository *host.Repository) { repository.S3.Endpoint = "ftp://example.test" }, } { + // The driver configuration is a pointer, so a struct copy shares it. Each + // case gets its own, or a mutation would follow into the ones after it. repository := valid + driver := *valid.S3 + repository.S3 = &driver mutate(&repository) if _, err := adapter.Capabilities(context.Background(), repository); !host.IsKind(err, host.ErrorInvalidConfiguration) { t.Fatalf("configuration %#v error = %v", repository, err) @@ -681,7 +690,7 @@ func TestS3HostRejectsInvalidDirectConfiguration(t *testing.T) { func TestS3HostMigratesLegacyManagedRootOnNextCommit(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} adapter := New(objects) request := stageFixture(t, "legacy-first", "", "", `content`) staged, err := adapter.Stage(ctx, repository, request) @@ -735,7 +744,7 @@ func TestS3HostMigratesLegacyManagedRootOnNextCommit(t *testing.T) { func TestS3HostRecoversAmbiguousRestoreAndExactRetry(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} adapter := New(objects) firstRequest := stageFixture(t, "ambiguous-restore-first", "", "", `first`) firstStage, err := adapter.Stage(ctx, repository, firstRequest) @@ -770,7 +779,7 @@ func TestS3HostRecoversAmbiguousRestoreAndExactRetry(t *testing.T) { func TestS3HostDoesNotRebindPublishedEffectToNewRestoreState(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} adapter := New(objects) request := stageFixture(t, "fixed-effect", "", "", `content`) firstStage, err := adapter.Stage(ctx, repository, request) @@ -803,7 +812,7 @@ func TestS3HostDoesNotRebindPublishedEffectToNewRestoreState(t *testing.T) { func TestS3HostRejectsPartialReservedRootMetadata(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} content := []byte(`content`) if _, err := objects.Put(ctx, PutRequest{ Key: objectKey(repository, pypiRootPath), Body: bytes.NewReader(content), Size: int64(len(content)), SHA256: digestBytes(content), @@ -819,7 +828,7 @@ func TestS3HostRejectsPartialReservedRootMetadata(t *testing.T) { func TestS3HostRejectsInvalidRestoreBeforeIdentity(t *testing.T) { ctx := context.Background() objects := newMemoryObjects() - repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", Bucket: "b", CanonicalEndpoint: "https://example.test"} + repository := host.Repository{Type: "s3", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), Visibility: "public", CanonicalEndpoint: "https://example.test", S3: &host.S3Config{Bucket: "b"}} identifier := strings.Repeat("1", 64) descriptor := restoreDescriptor{ PlanID: strings.Repeat("2", 64), ChangeID: "python:000000000000", AfterTreeSHA256: strings.Repeat("3", 64), diff --git a/adapters/host/s3/two_runners_test.go b/adapters/host/s3/two_runners_test.go index 897d4aa..d782195 100644 --- a/adapters/host/s3/two_runners_test.go +++ b/adapters/host/s3/two_runners_test.go @@ -20,9 +20,8 @@ func TestTwoRunnersPublishingToOneBucket(t *testing.T) { objects := newMemoryObjects() repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), - Type: "s3", Visibility: "public", Bucket: "packages", Prefix: "repo", - CanonicalEndpoint: "https://packages.example/repo", - } + Type: "s3", Visibility: "public", + CanonicalEndpoint: "https://packages.example/repo", S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}} // Two adapters over one store, which is two runners against one bucket. runnerA, runnerB := New(objects), New(objects) @@ -99,9 +98,8 @@ func TestTheRefusedRunnerSucceedsAfterReplanning(t *testing.T) { objects := newMemoryObjects() repository := host.Repository{ Name: "python", Format: "pypi", CommitPaths: []string{pypiRootPath}, RootRewriter: pypiRootRewriter(), - Type: "s3", Visibility: "public", Bucket: "packages", Prefix: "repo", - CanonicalEndpoint: "https://packages.example/repo", - } + Type: "s3", Visibility: "public", + CanonicalEndpoint: "https://packages.example/repo", S3: &host.S3Config{Bucket: "packages", Prefix: "repo"}} runnerA, runnerB := New(objects), New(objects) stale, err := runnerA.Observe(ctx, repository) if err != nil { diff --git a/engine/workspace.go b/engine/workspace.go index 597bab8..62b97bc 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -1602,19 +1602,33 @@ func toHostRepository(root, workspaceID, hostIdentity, name string, repository s if repository.Host.Type == "local" { canonicalEndpoint = repository.Host.Path } - return host.Repository{ + built := host.Repository{ Name: name, WorkspaceID: workspaceID, HostIdentity: hostIdentity, Format: repository.Format, CommitPaths: commitPaths, RootRewriter: rootRewriter, Type: repository.Host.Type, - Visibility: repository.Visibility, WorkspaceRoot: root, Path: repository.Host.Path, - Target: repository.Host.Target, - Bucket: repository.Host.Bucket, Prefix: repository.Host.Prefix, Region: repository.Host.Region, - Endpoint: repository.Host.Endpoint, CanonicalEndpoint: canonicalEndpoint, - UsePathStyle: repository.Host.UsePathStyle, - ReadAuth: repository.Host.ReadAuth, CredentialBroker: repository.Host.CredentialBroker, - RemoteRepository: repository.Host.Repository, Branch: repository.Host.Branch, - PreviewRepository: repository.Host.PreviewRepository, PreviewBranch: repository.Host.PreviewBranch, - PreviewEndpoint: repository.Host.PreviewEndpoint, - } + Visibility: repository.Visibility, WorkspaceRoot: root, CanonicalEndpoint: canonicalEndpoint, + } + // The manifest is one flat table per host, which is what an operator writes. + // This is where that becomes the one driver configuration a host is given, so + // an adapter is handed its own settings rather than everyone's. + switch repository.Host.Type { + case "local": + built.Local = &host.LocalConfig{Path: repository.Host.Path} + case "rsync": + built.Rsync = &host.RsyncConfig{Target: repository.Host.Target, Path: repository.Host.Path} + case "s3": + built.S3 = &host.S3Config{ + Bucket: repository.Host.Bucket, Prefix: repository.Host.Prefix, Region: repository.Host.Region, + Endpoint: repository.Host.Endpoint, UsePathStyle: repository.Host.UsePathStyle, + ReadAuth: repository.Host.ReadAuth, CredentialBroker: repository.Host.CredentialBroker, + } + case "github-pages": + built.Pages = &host.PagesConfig{ + RemoteRepository: repository.Host.Repository, Branch: repository.Host.Branch, + PreviewRepository: repository.Host.PreviewRepository, PreviewBranch: repository.Host.PreviewBranch, + PreviewEndpoint: repository.Host.PreviewEndpoint, + } + } + return built } func repositoryHostIdentity(repository state.Repository) (string, error) { diff --git a/host/driver_test.go b/host/driver_test.go new file mode 100644 index 0000000..3ef74dc --- /dev/null +++ b/host/driver_test.go @@ -0,0 +1,44 @@ +package host + +import "testing" + +// A repository carries the configuration of the host that will serve it, and +// only that one. +// +// The settings used to sit flat — bucket beside branch beside ssh target — so +// nothing stopped a repository carrying a bucket and a Pages branch at once, and +// nothing stopped an adapter reading the wrong one. Every adapter grew a line +// listing the fields it would refuse: four of them, in four vocabularies, all +// saying the same thing. This says it once, and the type says the rest. +func TestDriver(t *testing.T) { + for _, testcase := range []struct { + name string + repository Repository + wantError bool + }{ + {"local", Repository{Name: "r", Type: "local", Local: &LocalConfig{Path: "public"}}, false}, + {"rsync", Repository{Name: "r", Type: "rsync", Rsync: &RsyncConfig{Path: "/srv/r"}}, false}, + {"s3", Repository{Name: "r", Type: "s3", S3: &S3Config{Bucket: "b"}}, false}, + {"pages", Repository{Name: "r", Type: "github-pages", Pages: &PagesConfig{RemoteRepository: "o/n"}}, false}, + + {"no configuration at all", Repository{Name: "r", Type: "s3"}, true}, + {"a type nothing serves", Repository{Name: "r", Type: "ftp", S3: &S3Config{Bucket: "b"}}, true}, + // The shape the flat struct allowed and each adapter had to refuse. + {"two hosts at once", Repository{ + Name: "r", Type: "s3", S3: &S3Config{Bucket: "b"}, Pages: &PagesConfig{Branch: "gh-pages"}, + }, true}, + {"named one host, carries another", Repository{ + Name: "r", Type: "github-pages", S3: &S3Config{Bucket: "b"}, + }, true}, + } { + t.Run(testcase.name, func(t *testing.T) { + err := testcase.repository.Driver() + if testcase.wantError && err == nil { + t.Fatal("accepted") + } + if !testcase.wantError && err != nil { + t.Fatalf("refused: %v", err) + } + }) + } +} diff --git a/host/host.go b/host/host.go index 6761214..5bd30f8 100644 --- a/host/host.go +++ b/host/host.go @@ -43,6 +43,20 @@ type CredentialBroker interface { Identity() string } +// Repository is one repository as a host sees it: what every host needs, plus +// the settings of the one host that will serve it. +// +// The driver settings used to sit here flat — bucket beside branch beside ssh +// target, twenty-five fields of which any host used three. Nothing stopped the +// GitHub Pages adapter reading a bucket, so each adapter grew a line refusing +// every field belonging to some other host, and there were four of them saying +// the same thing in four vocabularies. A repository now carries exactly one +// driver configuration, so a Pages adapter cannot read a bucket because it does +// not have one, and the check that used to be written out by hand is the type. +// +// The manifest on disk is still flat. `[repo.python.host]` with a type and the +// settings that type needs reads well and is what an operator writes; this is +// the shape the code wants, and the projection between them happens once. type Repository struct { Name string WorkspaceID string @@ -59,28 +73,91 @@ type Repository struct { // it: one whose non-root paths are rewritten between revisions, so a new // revision cannot simply be written alongside the live one. Nil otherwise, // including for formats that need no staging at all. - RootRewriter RootRewriter - Type string - Visibility string + RootRewriter RootRewriter + Type string + Visibility string + // WorkspaceRoot is where the workspace lives, for a host that publishes + // inside it. WorkspaceRoot string - Path string - // Target is the ssh destination for an rsync host. - Target string - Bucket string - Prefix string - Region string - Endpoint string + // CanonicalEndpoint is the URL consumers install from. CanonicalEndpoint string - UsePathStyle bool - ReadAuth string - CredentialBroker string - RemoteRepository string - Branch string + + // Exactly one of these is set, and it is the one Type names. Driver returns + // it, and reports a repository that names one thing and carries another. + Local *LocalConfig + Rsync *RsyncConfig + S3 *S3Config + Pages *PagesConfig +} + +// LocalConfig publishes to a directory inside the workspace. +type LocalConfig struct { + // Path is workspace-relative, unlike every other host's. + Path string +} + +// RsyncConfig publishes to a directory on a machine reached over ssh. +type RsyncConfig struct { + // Target is the ssh destination: a hostname, user@host, or a name from + // ssh_config. Everything else about the connection — port, key, jump host — + // belongs in ssh_config, because an operator already has a place for it. + Target string + // Path is absolute on the far side, because a relative one resolves against + // whichever home directory the ssh user has. + Path string +} + +// S3Config publishes to an S3-compatible object store. +type S3Config struct { + Bucket string + Prefix string + Region string + Endpoint string + UsePathStyle bool + // ReadAuth and CredentialBroker are how a private repository is read. + ReadAuth string + CredentialBroker string +} + +// PagesConfig publishes to a Git-backed Pages site. +type PagesConfig struct { + RemoteRepository string + Branch string + // The companion site a reviewer installs from before a revision is live. PreviewRepository string PreviewBranch string PreviewEndpoint string } +// Driver reports whether the repository carries the configuration its type +// names, and nothing else. +// +// One central answer, rather than each adapter writing out the fields it will +// not accept. An adapter asks for its own configuration and is told no if the +// repository is not one it serves. +func (repository Repository) Driver() error { + set := 0 + for _, present := range []bool{ + repository.Local != nil, repository.Rsync != nil, + repository.S3 != nil, repository.Pages != nil, + } { + if present { + set++ + } + } + if set != 1 { + return fmt.Errorf("repository %q carries %d host configurations, want exactly one", repository.Name, set) + } + named := map[string]bool{ + "local": repository.Local != nil, "rsync": repository.Rsync != nil, + "s3": repository.S3 != nil, "github-pages": repository.Pages != nil, + }[repository.Type] + if !named { + return fmt.Errorf("repository %q is typed %q but carries another host's configuration", repository.Name, repository.Type) + } + return nil +} + // Capabilities is what one host can offer a publication. Each is reported // rather than inferred from the host's type, so the engine asks what a host can // do instead of knowing which hosts exist. diff --git a/internal/wire/hosts.go b/internal/wire/hosts.go index 1935d77..4d6c7bf 100644 --- a/internal/wire/hosts.go +++ b/internal/wire/hosts.go @@ -24,6 +24,13 @@ func NewHostResolver() *HostResolver { } func (resolver *HostResolver) Resolve(ctx context.Context, repository host.Repository) (host.Host, error) { + // One check that the repository carries the configuration its type names, + // here at the composition root where a driver is chosen. Each adapter used to + // write out the fields it would refuse, four times in four vocabularies, to + // guard against a shape a flat configuration made possible. + if err := repository.Driver(); err != nil { + return nil, err + } switch repository.Type { case "local": return resolver.local, nil @@ -37,7 +44,7 @@ func (resolver *HostResolver) Resolve(ctx context.Context, repository host.Repos } return s3host.NewAWS(ctx, repository) case "rsync": - return rsynchost.New(&rsynchost.SSHRunner{Target: repository.Target}), nil + return rsynchost.New(&rsynchost.SSHRunner{Target: repository.Rsync.Target}), nil case "github-pages": return githubpages.New(), nil default: From 36fa3d45dba5b7e869077adde34883daa5deb32b Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 15:18:07 +0200 Subject: [PATCH 17/26] Separate the decision apply makes from the work it does --- engine/applydecision.go | 147 ++++++++++++++++++++ engine/applydecision_test.go | 252 +++++++++++++++++++++++++++++++++++ engine/workspace.go | 44 ++---- 3 files changed, 409 insertions(+), 34 deletions(-) create mode 100644 engine/applydecision.go create mode 100644 engine/applydecision_test.go diff --git a/engine/applydecision.go b/engine/applydecision.go new file mode 100644 index 0000000..7cc6192 --- /dev/null +++ b/engine/applydecision.go @@ -0,0 +1,147 @@ +package engine + +import ( + "fmt" + "reflect" + + "github.com/shellcell/snailmail/host" + "github.com/shellcell/snailmail/internal/state" +) + +// applyFacts is the plan and the world side by side, at the moment apply has to +// decide what to do with one repository. +// +// Everything here was read before the decision: the plan as it was reviewed, +// what the host says is live now, the receipt written when it was last +// published, and whether the ledger already records every version the tree +// publishes. Nothing in the decision reads a file or asks a host, which is what +// lets it be tested against a table instead of a workspace. +type applyFacts struct { + planned state.PlanRepository + observed host.PublishedRevision + deployment state.DeploymentRecord + // planID is this apply's plan, which is not planned.PlanID: it is how a + // revision published by *this* plan is told from one published by another. + planID string + // signing is the signing state the repository is configured for now. + signing deploymentSigningState + // bindingsComplete reports whether the ledger already binds every version + // the desired tree publishes. + bindingsComplete bool +} + +// applyDecision is what those facts add up to. +type applyDecision struct { + // current reports that the host already serves the desired tree and the + // receipt for it is in place, so there is nothing to publish. + current bool +} + +// decideApply refuses a plan the world has moved past, and says whether there +// is anything left to do. +// +// This was 30 lines of interleaved booleans inside a 200-line function, and it +// is the code that decides whether apply publishes or skips — the place where +// being wrong means a repository that should have been updated silently was +// not, or one that was already correct is republished. It is separated out so +// it can be read, and so it can be tested directly rather than by building a +// workspace and hoping the case is reachable. +// +// Four questions, in order, because a later one is only meaningful once the +// earlier ones hold: +// +// 1. Has the receipt changed under us? Unless this plan wrote it, a receipt +// that differs from the one the plan recorded means another publisher has +// been here. +// 2. Is the host still where the plan left it? Either it holds what the plan +// observed, or it holds what this plan already published — a retry after a +// partial apply is not a conflict. +// 3. Does the plan's own action agree with its observations? A plan claiming +// "noop" over an observation that calls for an update was built wrong or +// edited since. +// 4. Given all that, is there anything left to do? +func decideApply(facts applyFacts) (applyDecision, error) { + planned, observed, deployment := facts.planned, facts.observed, facts.deployment + + // A host that reports a manifest digest can tell two revisions of the same + // tree apart when only generated metadata changed. One that does not returns + // an empty string, and comparing against it would find a difference on every + // run. + comparesManifest := planned.ReportsManifestDigest + sameTree := observed.TreeSHA256 == planned.DesiredTreeSHA256 + sameManifest := !comparesManifest || observed.ManifestSHA256 == planned.DesiredManifestSHA256 + + // The host holds what this plan published, rather than what it observed when + // the plan was made. That is the shape of a retry: a previous run of this + // same plan got as far as the host. + publishedByThisPlan := planned.Action != "noop" && sameTree && + (!comparesManifest || (observed.PlanID == facts.planID && + observed.ChangeID == planned.ChangeID && + observed.ManifestSHA256 == planned.DesiredManifestSHA256)) + + // The receipt records this plan's publication, down to the revision the host + // currently reports. + receiptWrittenByThisPlan := deployment.PlanID == facts.planID && + deployment.ChangeID == planned.ChangeID && + deployment.TreeSHA256 == planned.DesiredTreeSHA256 && + deployment.ManifestSHA256 == planned.DesiredManifestSHA256 && + deployment.NativeRevision == observed.NativeRevision && + deploymentSigningMatches(deployment, facts.signing) + + receiptMatchesPlanned := reflect.DeepEqual(deployment, planned.ObservedDeployment) + + // 1. Nobody else has written the receipt since the plan was made. + if !receiptMatchesPlanned && !receiptWrittenByThisPlan { + return applyDecision{}, fmt.Errorf("stale plan: repository %q deployment receipt changed", planned.Name) + } + + // 2. The host is where the plan left it, or where this plan put it. + if !revisionMatchesPlanObservation(observed, planned) && !publishedByThisPlan { + if observed.TreeSHA256 == planned.ObservedTreeSHA256 { + return applyDecision{}, fmt.Errorf("stale plan: repository %q native revision changed", planned.Name) + } + if sameTree { + return applyDecision{}, fmt.Errorf("stale plan: repository %q desired tree was published by another change", planned.Name) + } + return applyDecision{}, fmt.Errorf("stale plan: repository %q target changed", planned.Name) + } + + // 3. The plan's action follows from the plan's own observations. + if planned.Action != expectedPlanAction(facts) || + planned.ChangeID != planned.Name+":"+planned.DesiredTreeSHA256[:12] { + return applyDecision{}, fmt.Errorf("plan repository %q has inconsistent action metadata", planned.Name) + } + + // 4. Nothing left to do if the host holds the desired tree and the receipt + // for it stands. A receipt this plan wrote counts; so does one that was + // already correct under a noop; and so does the case where the host was + // published by this plan but the receipt has not been written yet, which is + // a run that failed between the two and is being retried. + receiptRecovery := publishedByThisPlan && receiptMatchesPlanned + deploymentCurrent := deploymentMatchesDesired(deployment, observed, + planned.DesiredTreeSHA256, planned.DesiredManifestSHA256, facts.signing) + return applyDecision{ + current: sameTree && sameManifest && + (receiptWrittenByThisPlan || (planned.Action == "noop" && deploymentCurrent) || receiptRecovery), + }, nil +} + +// expectedPlanAction is the action the plan should have recorded, derived from +// the observations the plan itself carries. Comparing it with what the plan says +// catches a plan built against different code, or edited after review. +func expectedPlanAction(facts applyFacts) string { + planned := facts.planned + plannedObserved := publishedFromPlanObservation(planned) + unchanged := planned.ObservedTreeSHA256 == planned.DesiredTreeSHA256 && + (!planned.ReportsManifestDigest || planned.ObservedManifestSHA256 == planned.DesiredManifestSHA256) && + facts.bindingsComplete && + deploymentMatchesDesired(planned.ObservedDeployment, plannedObserved, + planned.DesiredTreeSHA256, planned.DesiredManifestSHA256, facts.signing) + if unchanged { + return "noop" + } + if planned.ObservedRevision == "" { + return "create" + } + return "update" +} diff --git a/engine/applydecision_test.go b/engine/applydecision_test.go new file mode 100644 index 0000000..fcc954a --- /dev/null +++ b/engine/applydecision_test.go @@ -0,0 +1,252 @@ +package engine + +import ( + "fmt" + "reflect" + "strings" + "testing" + + "github.com/shellcell/snailmail/host" + "github.com/shellcell/snailmail/internal/state" +) + +// referenceDecide is prepareRepository's decision exactly as it was written +// before it was extracted, kept verbatim so the extraction can be checked +// against it rather than against a reading of it. +// +// It is dead code in every sense but one: TestDecideApplyMatchesTheCodeItReplaced +// runs both over several thousand combinations and requires them to agree. This +// is the code that decides whether apply publishes or skips, and "I read it +// carefully" is not the standard that deserves. +func referenceDecide(facts applyFacts) (bool, error) { + planned, observed, deployment := facts.planned, facts.observed, facts.deployment + planID := facts.planID + desiredSigningState := facts.signing + plannedObserved := publishedFromPlanObservation(planned) + + matchesObserved := revisionMatchesPlanObservation(observed, planned) + managedRemote := planned.ReportsManifestDigest + matchesApplied := planned.Action != "noop" && observed.TreeSHA256 == planned.DesiredTreeSHA256 && + (!managedRemote || (observed.PlanID == planID && observed.ChangeID == planned.ChangeID && observed.ManifestSHA256 == planned.DesiredManifestSHA256)) + deploymentApplied := deployment.PlanID == planID && deployment.ChangeID == planned.ChangeID && deployment.TreeSHA256 == planned.DesiredTreeSHA256 && deployment.ManifestSHA256 == planned.DesiredManifestSHA256 && deployment.NativeRevision == observed.NativeRevision && deploymentSigningMatches(deployment, desiredSigningState) + deploymentCurrent := deploymentMatchesDesired(deployment, observed, planned.DesiredTreeSHA256, planned.DesiredManifestSHA256, desiredSigningState) + if !reflect.DeepEqual(deployment, planned.ObservedDeployment) && !deploymentApplied { + return false, fmt.Errorf("stale plan: repository %q deployment receipt changed", planned.Name) + } + if !matchesObserved && !matchesApplied { + if observed.TreeSHA256 == planned.ObservedTreeSHA256 { + return false, fmt.Errorf("stale plan: repository %q native revision changed", planned.Name) + } + if observed.TreeSHA256 == planned.DesiredTreeSHA256 { + return false, fmt.Errorf("stale plan: repository %q desired tree was published by another change", planned.Name) + } + return false, fmt.Errorf("stale plan: repository %q target changed", planned.Name) + } + expectedAction := "noop" + if planned.ObservedTreeSHA256 != planned.DesiredTreeSHA256 || (managedRemote && planned.ObservedManifestSHA256 != planned.DesiredManifestSHA256) || !facts.bindingsComplete || !deploymentMatchesDesired(planned.ObservedDeployment, plannedObserved, planned.DesiredTreeSHA256, planned.DesiredManifestSHA256, desiredSigningState) { + expectedAction = "update" + if planned.ObservedRevision == "" { + expectedAction = "create" + } + } + if planned.Action != expectedAction || planned.ChangeID != planned.Name+":"+planned.DesiredTreeSHA256[:12] { + return false, fmt.Errorf("plan repository %q has inconsistent action metadata", planned.Name) + } + receiptRecovery := matchesApplied && reflect.DeepEqual(deployment, planned.ObservedDeployment) + current := observed.TreeSHA256 == planned.DesiredTreeSHA256 && (!managedRemote || observed.ManifestSHA256 == planned.DesiredManifestSHA256) && (deploymentApplied || (planned.Action == "noop" && deploymentCurrent) || receiptRecovery) + return current, nil +} + +const ( + desiredTree = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + otherTree = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + desiredMani = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + otherMani = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + thisPlanID = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + otherPlanID = "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + repositoryName = "python" +) + +var wantedChangeID = repositoryName + ":" + desiredTree[:12] + +// applyFactsMatrix enumerates the combinations that the decision actually turns +// on. Each dimension is a thing the world can independently do between a plan +// being made and applied. +func applyFactsMatrix() []applyFacts { + var all []applyFacts + for _, action := range []string{"noop", "create", "update"} { + for _, reports := range []bool{true, false} { + for _, observedTree := range []string{desiredTree, otherTree, ""} { + for _, observedManifest := range []string{desiredMani, otherMani, ""} { + for _, observedPlan := range []string{thisPlanID, otherPlanID} { + for _, plannedObservedTree := range []string{desiredTree, otherTree} { + for _, receiptTree := range []string{desiredTree, otherTree} { + for _, receiptMatchesPlanned := range []bool{true, false} { + for _, bindings := range []bool{true, false} { + for _, observedRevision := range []string{"", "rev-1"} { + receipt := state.DeploymentRecord{ + Repository: repositoryName, PlanID: thisPlanID, ChangeID: wantedChangeID, + TreeSHA256: receiptTree, ManifestSHA256: desiredMani, + NativeRevision: "rev-1", DeployedAt: "2026-08-22T00:00:00Z", + } + plannedReceipt := receipt + if !receiptMatchesPlanned { + plannedReceipt.NativeRevision = "rev-other" + } + all = append(all, applyFacts{ + planned: state.PlanRepository{ + Name: repositoryName, Action: action, ChangeID: wantedChangeID, + DesiredTreeSHA256: desiredTree, DesiredManifestSHA256: desiredMani, + ObservedTreeSHA256: plannedObservedTree, ObservedManifestSHA256: desiredMani, + ObservedRevision: observedRevision, ObservedDeployment: plannedReceipt, + ReportsManifestDigest: reports, + }, + observed: host.PublishedRevision{ + NativeRevision: "rev-1", TreeSHA256: observedTree, + PlanID: observedPlan, ChangeID: wantedChangeID, + ManifestSHA256: observedManifest, + }, + deployment: receipt, + planID: thisPlanID, + bindingsComplete: bindings, + }) + } + } + } + } + } + } + } + } + } + } + return all +} + +// The extraction has to be a refactor, not a rewrite. Both implementations run +// over every combination and must agree on the answer and on which refusal. +func TestDecideApplyMatchesTheCodeItReplaced(t *testing.T) { + cases := applyFactsMatrix() + if len(cases) < 1000 { + t.Fatalf("the matrix covers only %d combinations, which is not enough to say much", len(cases)) + } + agreed, refused := 0, 0 + for index, facts := range cases { + wantCurrent, wantErr := referenceDecide(facts) + decision, err := decideApply(facts) + if (wantErr == nil) != (err == nil) { + t.Fatalf("case %d: reference err=%v, extracted err=%v\nfacts: %#v", index, wantErr, err, facts) + } + if wantErr != nil { + if wantErr.Error() != err.Error() { + t.Fatalf("case %d: reference refused with %q, extracted with %q", index, wantErr, err) + } + refused++ + continue + } + if decision.current != wantCurrent { + t.Fatalf("case %d: reference current=%v, extracted current=%v\nfacts: %#v", index, wantCurrent, decision.current, facts) + } + agreed++ + } + t.Logf("%d combinations: %d agreed on a decision, %d agreed on a refusal", len(cases), agreed, refused) + if agreed == 0 || refused == 0 { + t.Fatal("the matrix exercises only one branch, so agreement says nothing") + } +} + +// The cases worth being able to read, stated as themselves rather than as a +// point in a matrix. +func TestDecideApply(t *testing.T) { + receipt := state.DeploymentRecord{ + Repository: repositoryName, PlanID: thisPlanID, ChangeID: wantedChangeID, + TreeSHA256: desiredTree, ManifestSHA256: desiredMani, + NativeRevision: "rev-1", DeployedAt: "2026-08-22T00:00:00Z", + } + published := host.PublishedRevision{ + NativeRevision: "rev-1", TreeSHA256: desiredTree, PlanID: thisPlanID, + ChangeID: wantedChangeID, ManifestSHA256: desiredMani, + } + base := applyFacts{ + // A settled repository: what the plan observed is exactly what the host + // still reports, which is what revisionMatchesPlanObservation compares — + // every field of the revision, not just the tree. + planned: state.PlanRepository{ + Name: repositoryName, Action: "noop", ChangeID: wantedChangeID, + DesiredTreeSHA256: desiredTree, DesiredManifestSHA256: desiredMani, + ObservedRevision: published.NativeRevision, ObservedTreeSHA256: published.TreeSHA256, + ObservedPlanID: published.PlanID, ObservedChangeID: published.ChangeID, + ObservedManifestSHA256: published.ManifestSHA256, + ObservedDeployment: receipt, + ReportsManifestDigest: true, + }, + observed: published, deployment: receipt, planID: thisPlanID, bindingsComplete: true, + } + + t.Run("nothing to do", func(t *testing.T) { + decision, err := decideApply(base) + if err != nil || !decision.current { + t.Fatalf("current=%v err=%v", decision.current, err) + } + }) + + t.Run("another publisher wrote the receipt", func(t *testing.T) { + facts := base + facts.deployment.NativeRevision = "rev-somebody-else" + _, err := decideApply(facts) + if err == nil || !strings.Contains(err.Error(), "deployment receipt changed") { + t.Fatalf("err = %v", err) + } + }) + + t.Run("the host moved to something nobody planned", func(t *testing.T) { + facts := base + facts.observed.TreeSHA256 = otherTree + _, err := decideApply(facts) + if err == nil || !strings.Contains(err.Error(), "target changed") { + t.Fatalf("err = %v", err) + } + }) + + t.Run("a plan claiming noop over an incomplete ledger", func(t *testing.T) { + facts := base + facts.bindingsComplete = false + _, err := decideApply(facts) + if err == nil || !strings.Contains(err.Error(), "inconsistent action metadata") { + t.Fatalf("err = %v", err) + } + }) + + // The retry that motivates half of this: a run that published and then died + // before writing the receipt. The host holds this plan's tree, the receipt is + // still the one the plan observed, and there is nothing left to publish. + t.Run("a run that published and died before recording it", func(t *testing.T) { + facts := base + facts.planned.Action = "update" + // What the plan saw before it ran: a different tree, and no receipt. + facts.planned.ObservedTreeSHA256 = otherTree + facts.planned.ObservedRevision = "rev-0" + facts.deployment = state.DeploymentRecord{Repository: repositoryName} + facts.planned.ObservedDeployment = facts.deployment + decision, err := decideApply(facts) + if err != nil { + t.Fatalf("a retry of a partly applied plan was refused: %v", err) + } + if !decision.current { + t.Fatal("a tree this plan already published was not recognised as current") + } + }) + + // A host that reports no manifest digest returns an empty string for it, and + // comparing against that would find a difference on every run. + t.Run("a host that reports no manifest digest still settles", func(t *testing.T) { + facts := base + facts.planned.ReportsManifestDigest = false + facts.observed.ManifestSHA256, facts.observed.PlanID = "", "" + facts.planned.ObservedManifestSHA256, facts.planned.ObservedPlanID = "", "" + decision, err := decideApply(facts) + if err != nil || !decision.current { + t.Fatalf("current=%v err=%v", decision.current, err) + } + }) +} diff --git a/engine/workspace.go b/engine/workspace.go index 62b97bc..18632e9 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -2553,42 +2553,18 @@ func (preparation *applyPreparation) prepareRepository(planned state.PlanReposit if err := validateRepositorySigningTransition(repository, preparation.manifest.Keys, deployment, plannedObserved, preparation.now, trustNotBefore); err != nil { return applyRepository{}, fmt.Errorf("repository %q: %w", planned.Name, err) } - matchesObserved := revisionMatchesPlanObservation(observed, planned) - // Declared by the host rather than derived from its name. The plan carries - // what the host reported when it was made, and the drift check above has - // already established that the live host still reports the same. - managedRemote := planned.ReportsManifestDigest - matchesApplied := planned.Action != "noop" && observed.TreeSHA256 == planned.DesiredTreeSHA256 && - (!managedRemote || (observed.PlanID == preparation.plan.PlanID && observed.ChangeID == planned.ChangeID && observed.ManifestSHA256 == planned.DesiredManifestSHA256)) - deploymentApplied := deployment.PlanID == preparation.plan.PlanID && deployment.ChangeID == planned.ChangeID && deployment.TreeSHA256 == planned.DesiredTreeSHA256 && deployment.ManifestSHA256 == planned.DesiredManifestSHA256 && deployment.NativeRevision == observed.NativeRevision && deploymentSigningMatches(deployment, desiredSigningState) - deploymentCurrent := deploymentMatchesDesired(deployment, observed, planned.DesiredTreeSHA256, planned.DesiredManifestSHA256, desiredSigningState) - if !reflect.DeepEqual(deployment, planned.ObservedDeployment) && !deploymentApplied { - return applyRepository{}, fmt.Errorf("stale plan: repository %q deployment receipt changed", planned.Name) - } - if !matchesObserved && !matchesApplied { - if observed.TreeSHA256 == planned.ObservedTreeSHA256 { - return applyRepository{}, fmt.Errorf("stale plan: repository %q native revision changed", planned.Name) - } - if observed.TreeSHA256 == planned.DesiredTreeSHA256 { - return applyRepository{}, fmt.Errorf("stale plan: repository %q desired tree was published by another change", planned.Name) - } - return applyRepository{}, fmt.Errorf("stale plan: repository %q target changed", planned.Name) - } - expectedAction := "noop" - if planned.ObservedTreeSHA256 != planned.DesiredTreeSHA256 || (managedRemote && planned.ObservedManifestSHA256 != planned.DesiredManifestSHA256) || !publicationBindingsComplete(lock, repository, ledger) || !deploymentMatchesDesired(planned.ObservedDeployment, plannedObserved, planned.DesiredTreeSHA256, planned.DesiredManifestSHA256, desiredSigningState) { - expectedAction = "update" - if planned.ObservedRevision == "" { - expectedAction = "create" - } - } - if planned.Action != expectedAction || planned.ChangeID != planned.Name+":"+planned.DesiredTreeSHA256[:12] { - return applyRepository{}, fmt.Errorf("plan repository %q has inconsistent action metadata", planned.Name) - } - receiptRecovery := matchesApplied && reflect.DeepEqual(deployment, planned.ObservedDeployment) - current := observed.TreeSHA256 == planned.DesiredTreeSHA256 && (!managedRemote || observed.ManifestSHA256 == planned.DesiredManifestSHA256) && (deploymentApplied || (planned.Action == "noop" && deploymentCurrent) || receiptRecovery) + decision, err := decideApply(applyFacts{ + planned: planned, observed: observed, deployment: deployment, + planID: preparation.plan.PlanID, signing: desiredSigningState, + bindingsComplete: publicationBindingsComplete(lock, repository, ledger), + }) + if err != nil { + return applyRepository{}, err + } item := applyRepository{ planned: planned, repository: repository, lock: lock, host: selectedHost, - hostRepository: hostRepository, observed: observed, current: current, deployment: deployment, signingState: desiredSigningState, + hostRepository: hostRepository, observed: observed, current: decision.current, + deployment: deployment, signingState: desiredSigningState, } // Nothing to build: the host already serves this tree and no signing effect // has to be replayed, so there is no stage for this repository. From 27baafef255593bad14b6060d97279762c54af05 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 15:36:34 +0200 Subject: [PATCH 18/26] Let status describe a workspace that is not tidy --- cmd/snailmail/main.go | 87 +++++++++++++++++---- cmd/snailmail/main_test.go | 7 +- engine/status.go | 21 +++-- engine/status_uncommitted_test.go | 122 ++++++++++++++++++++++++++++++ internal/state/git.go | 81 +++++++++++++++++++- 5 files changed, 295 insertions(+), 23 deletions(-) create mode 100644 engine/status_uncommitted_test.go diff --git a/cmd/snailmail/main.go b/cmd/snailmail/main.go index b94c555..e54697e 100644 --- a/cmd/snailmail/main.go +++ b/cmd/snailmail/main.go @@ -755,28 +755,38 @@ func runRollout(ctx context.Context, args []string, stdout, stderr io.Writer) er } func runStatus(ctx context.Context, args []string, stdout, stderr io.Writer) error { - flags := newCommandFlags("status", stderr).withWorkspace() - jsonOutput := flags.Bool("json", false, "emit machine-readable JSON") - if err := flags.Parse(args); err != nil { + flags := newCommandFlags("status", stderr).withWorkspace().withJSON() + if err := flags.parse(args); err != nil { return err } - if flags.NArg() != 0 { - return errors.New("usage: snailmail status [--workspace DIR] [--json]") - } result, err := engine.StatusWorkspace(ctx, engine.StatusWorkspaceRequest{Root: flags.Root()}) if err != nil { return err } - if *jsonOutput { - encoder := json.NewEncoder(stdout) - encoder.SetIndent("", " ") - return encoder.Encode(result) + if done, err := flags.emit(stdout, result); done || err != nil { + return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 workspace %s at %s\n", result.Workspace, result.GitRevision) + fmt.Fprintf(stdout, "📦 workspace %s at %s\n", result.Workspace, shortDigest(result.GitRevision)) + + // Said first, because it changes what everything below means. Everything here + // is read at the revision above, so uncommitted work is not in it — and a + // reader comparing this against their working tree needs to know that before + // they read the numbers, not after. + if len(result.Uncommitted) != 0 { + fmt.Fprintf(stdout, "⚠️ %d uncommitted %s, not included below:\n", + len(result.Uncommitted), plural(len(result.Uncommitted), "change", "changes")) + for _, name := range result.Uncommitted { + fmt.Fprintf(stdout, " %s\n", name) + } + fmt.Fprintln(stdout, " git add -A && git commit -m \"...\" to include them") + } + for _, repository := range result.Repositories { - fmt.Fprintf(stdout, "✉️ %s: %d visible, %d retained; visible bindings %s; deployment %s\n", - repository.Name, repository.VisiblePackageVersions, repository.RetainedPackageVersions, repository.VisibleBindingState, repository.Deployment.State) + fmt.Fprintf(stdout, "✉️ %s (%s): %s\n", repository.Name, repository.Format, describePackages(repository)) + if detail := describeRepositoryState(repository); detail != "" { + fmt.Fprintf(stdout, " %s\n", detail) + } } // The lock is what every plan and apply parses whole, so its size is the // number that predicts where a workspace stops being comfortable. Reported for @@ -787,10 +797,59 @@ func runStatus(ctx context.Context, args []string, stdout, stderr io.Writer) err humanBytes(result.LockBytes), len(result.Repositories), plural(len(result.Repositories), "repository", "repositories"), largestLockSuffix(result)) } - fmt.Fprintln(stdout, "✉️ live hosts, upstream releases, foreign remotes, gate completion, and apply failures were not observed") + // What this ran on, rather than a list of what it did not do. The old line + // named five things that were not observed and left the reader to work out + // whether that mattered. + fmt.Fprintln(stdout, "✉️ read from committed workspace state; no host was contacted") return nil } +// describePackages says what a repository holds, in words rather than in two +// bare numbers whose difference the reader has to infer. +// +// "Visible" is what the configured track renders — what a client installing +// from this repository would see. Retained versions that are not visible were +// yanked or pruned; they stay in the lock, and can be promoted back. +func describePackages(repository engine.StatusRepository) string { + visible, retained := repository.VisiblePackageVersions, repository.RetainedPackageVersions + switch { + case retained == 0: + return "no packages yet" + case visible == retained: + return fmt.Sprintf("%d %s", visible, plural(visible, "version", "versions")) + case visible == 0: + return fmt.Sprintf("nothing visible, %d %s retained", retained, plural(retained, "version", "versions")) + default: + return fmt.Sprintf("%d of %d %s visible", visible, retained, plural(retained, "version", "versions")) + } +} + +// describeRepositoryState reports what needs attention, and nothing when +// nothing does. "deployment unrecorded" over a repository nobody has published +// is not news; over one that has been, it is. +func describeRepositoryState(repository engine.StatusRepository) string { + var notes []string + if repository.VisibleBindingState != "complete" { + notes = append(notes, "publication bindings incomplete — the next plan will record them") + } + // A receipt is evidence that a tree was observed live when it was published, + // not that it is live now — nothing here contacts a host. Said as what it is. + switch repository.Deployment.State { + case "recorded": + notes = append(notes, "last published "+shortDigest(repository.Deployment.TreeSHA256)+ + " on "+repository.Deployment.DeployedAt) + case "unrecorded": + // Only worth saying where there was something to publish. An empty + // repository nobody has applied yet is not news. + if repository.VisiblePackageVersions != 0 { + notes = append(notes, "never published") + } + default: + notes = append(notes, "deployment "+repository.Deployment.State) + } + return strings.Join(notes, "; ") +} + func runDoctor(ctx context.Context, args []string, stdout, stderr io.Writer) error { return runDoctorWithFetcher(ctx, args, stdout, stderr, httpsource.New()) } diff --git a/cmd/snailmail/main_test.go b/cmd/snailmail/main_test.go index 2a88870..b39bccb 100644 --- a/cmd/snailmail/main_test.go +++ b/cmd/snailmail/main_test.go @@ -164,7 +164,12 @@ func TestStatusEmitsMachineReadableCommittedEvidence(t *testing.T) { if err := run(context.Background(), []string{"status", "--workspace", root}, &stdout, &stderr); err != nil { t.Fatal(err) } - if !strings.Contains(stdout.String(), "live hosts, upstream releases") { + // The human output says what it read, so a reader knows the numbers are the + // committed ones and that nothing here asked a host anything. It used to say + // this as a list of five things it had not observed, which is the same fact + // written so a reader has to work out whether it matters. + if !strings.Contains(stdout.String(), "committed workspace state") || + !strings.Contains(stdout.String(), "no host was contacted") { t.Fatalf("status omitted observation scope: %q", stdout.String()) } } diff --git a/engine/status.go b/engine/status.go index 01cdf85..19c788e 100644 --- a/engine/status.go +++ b/engine/status.go @@ -16,10 +16,14 @@ type StatusWorkspaceRequest struct { } type StatusWorkspaceResult struct { - SchemaVersion int `json:"schema_version"` - Workspace string `json:"workspace"` - GitRevision string `json:"git_revision"` - ObservationScope string `json:"observation_scope"` + SchemaVersion int `json:"schema_version"` + Workspace string `json:"workspace"` + GitRevision string `json:"git_revision"` + // Uncommitted names workspace files changed since the revision above. + // Everything else reported here is read at that revision, so a non-empty + // list is what explains a status that does not match the working tree. + Uncommitted []string `json:"uncommitted,omitempty"` + ObservationScope string `json:"observation_scope"` // LockBytes is every repository's lock added up, which is what a single git // operation has to carry. LockBytes int64 `json:"lock_bytes"` @@ -86,7 +90,11 @@ func StatusWorkspace(ctx context.Context, request StatusWorkspaceRequest) (Statu if err := ctx.Err(); err != nil { return StatusWorkspaceResult{}, err } - revision, err := state.RequireCleanGitContext(ctx, root) + // Reported, not required. status is what someone runs because a workspace is + // in an odd state, and it used to refuse to describe one until it was tidy — + // so the command that exists to say what is going on was the one command that + // would not say it. + revision, uncommitted, err := state.WorkspaceGitState(ctx, root) if err != nil { return StatusWorkspaceResult{}, err } @@ -96,6 +104,7 @@ func StatusWorkspace(ctx context.Context, request StatusWorkspaceRequest) (Statu } result := StatusWorkspaceResult{ SchemaVersion: statusSchemaVersion, Workspace: manifest.Workspace.Name, GitRevision: revision, + Uncommitted: uncommitted, ObservationScope: "committed workspace evidence only", Repositories: make([]StatusRepository, 0, len(manifest.Repositories)), } for _, name := range state.RepositoryNames(manifest) { @@ -177,7 +186,7 @@ func StatusWorkspace(ctx context.Context, request StatusWorkspaceRequest) (Statu if err := ctx.Err(); err != nil { return StatusWorkspaceResult{}, err } - if err := state.AssertGitRevisionContext(ctx, root, revision); err != nil { + if err := state.AssertGitRevisionUnchangedContext(ctx, root, revision); err != nil { return StatusWorkspaceResult{}, err } return result, nil diff --git a/engine/status_uncommitted_test.go b/engine/status_uncommitted_test.go new file mode 100644 index 0000000..3d0b1ca --- /dev/null +++ b/engine/status_uncommitted_test.go @@ -0,0 +1,122 @@ +package engine + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// status describes a workspace with uncommitted work rather than refusing to. +// +// It used to require a clean tree, so the command someone runs *because* things +// look wrong was the one command that would not answer while they were. The +// numbers it reports still come from the committed revision — that is what +// makes them meaningful — so what changed is that it says which files are not +// in them instead of declining to speak. +func TestStatusReportsUncommittedWorkInsteadOfRefusing(t *testing.T) { + root := t.TempDir() + command := exec.Command("git", "init", "-b", "main") + command.Dir = root + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, output) + } + if err := InitWorkspace(InitWorkspaceRequest{Root: root, Name: "reported"}); err != nil { + t.Fatal(err) + } + commitPaths(t, root, "initialize", ".gitignore", "snailmail.toml") + if err := SetupRepository(SetupRepositoryRequest{ + Root: root, Name: "tools", Format: "raw", HostType: "local", + Output: "public/tools", Visibility: "public", + }); err != nil { + t.Fatal(err) + } + + // setup wrote the manifest and a lock, and neither is committed yet. + result, err := StatusWorkspace(context.Background(), StatusWorkspaceRequest{Root: root}) + if err != nil { + t.Fatalf("status refused a workspace with uncommitted changes: %v", err) + } + if len(result.Uncommitted) == 0 { + t.Fatal("status reported nothing uncommitted after setup wrote two files") + } + found := make(map[string]bool, len(result.Uncommitted)) + for _, name := range result.Uncommitted { + found[name] = true + } + if !found["snailmail.toml"] { + t.Errorf("uncommitted = %v, want the manifest among them", result.Uncommitted) + } + + commitWorkspace(t, root, "configure tools") + result, err = StatusWorkspace(context.Background(), StatusWorkspaceRequest{Root: root}) + if err != nil { + t.Fatal(err) + } + if len(result.Uncommitted) != 0 { + t.Errorf("a committed workspace reported %v as uncommitted", result.Uncommitted) + } + + // Something untracked that is not workspace state is not the workspace's + // business, and reporting it would train a reader to ignore the line. + if err := os.WriteFile(filepath.Join(root, "notes.txt"), []byte("mine"), 0o644); err != nil { + t.Fatal(err) + } + result, err = StatusWorkspace(context.Background(), StatusWorkspaceRequest{Root: root}) + if err != nil { + t.Fatal(err) + } + if len(result.Uncommitted) != 0 { + t.Errorf("an unrelated untracked file was reported as workspace state: %v", result.Uncommitted) + } +} + +// The commands that take effect still require a clean tree: acting on state +// nobody has reviewed is the thing the whole design exists to prevent. +func TestPlanStillRequiresACommittedWorkspace(t *testing.T) { + root := t.TempDir() + command := exec.Command("git", "init", "-b", "main") + command.Dir = root + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("git init: %v: %s", err, output) + } + if err := InitWorkspace(InitWorkspaceRequest{Root: root, Name: "guarded"}); err != nil { + t.Fatal(err) + } + commitPaths(t, root, "initialize", ".gitignore", "snailmail.toml") + if err := SetupRepository(SetupRepositoryRequest{ + Root: root, Name: "tools", Format: "raw", HostType: "local", + Output: "public/tools", Visibility: "public", + }); err != nil { + t.Fatal(err) + } + _, err := PlanWorkspace(context.Background(), PlanWorkspaceRequest{ + Hosts: localHosts(), Root: root, Output: filepath.Join(root, "plan.json"), + }) + if err == nil { + t.Fatal("plan ran against an uncommitted workspace") + } + // And says what to do about it, which the old message did not. + for _, want := range []string{"snailmail.toml", "git add", "git commit"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("refusal %q does not mention %q", err, want) + } + } +} + +// commitPaths commits exactly the named paths, for a fixture that wants some of +// the workspace committed and the rest not. +func commitPaths(t *testing.T, root, message string, paths ...string) { + t.Helper() + arguments := append([]string{"-C", root, "add", "--"}, paths...) + if output, err := exec.Command("git", arguments...).CombinedOutput(); err != nil { + t.Fatalf("git add: %v: %s", err, output) + } + output, err := exec.Command("git", "-C", root, "-c", "user.name=t", "-c", "user.email=t@example.com", + "commit", "-m", message).CombinedOutput() + if err != nil { + t.Fatalf("git commit: %v: %s", err, output) + } +} diff --git a/internal/state/git.go b/internal/state/git.go index 02a169f..e80ecc7 100644 --- a/internal/state/git.go +++ b/internal/state/git.go @@ -66,6 +66,60 @@ func RequireCleanGitContext(ctx context.Context, root string) (string, error) { return requireCleanGitContext(ctx, root, nil) } +// WorkspaceGitState reports the current revision and anything uncommitted, +// without refusing either. +// +// Most commands take effect from committed state and require it, which is what +// RequireCleanGit is for. Reading commands are different: status is what someone +// runs *because* a workspace is in an odd state, and refusing to describe one +// until it is tidy answers a question nobody asked. It reports the mess instead. +func WorkspaceGitState(ctx context.Context, root string) (revision string, uncommitted []string, err error) { + if err := requireCompleteGitHistoryContext(ctx, root); err != nil { + return "", nil, err + } + if _, err := symbolicHeadContext(ctx, root); err != nil { + return "", nil, err + } + revision, err = gitOutputContext(ctx, root, "rev-parse", "HEAD") + if err != nil { + if ctx.Err() != nil { + return "", nil, ctx.Err() + } + return "", nil, errors.New("workspace must be a Git repository with at least one commit") + } + status, err := gitStatusOutputContext(ctx, root) + if err != nil { + if ctx.Err() != nil { + return "", nil, ctx.Err() + } + return "", nil, err + } + authoritative, err := authoritativePaths(root) + if err != nil { + return "", nil, err + } + entries, err := parseGitStatus(status) + if err != nil { + return "", nil, err + } + seen := make(map[string]bool) + for _, entry := range entries { + for _, name := range entry.paths { + // An untracked file that is not workspace state is somebody else's + // business — a build directory, an editor's leavings. + if entry.code == "??" && !authoritative[name] && !isAuthoritativePath(name) { + continue + } + if !seen[name] { + seen[name] = true + uncommitted = append(uncommitted, name) + } + } + } + sort.Strings(uncommitted) + return revision, uncommitted, nil +} + func RequireCleanGitAllowingUntracked(root string, relativePaths []string) (string, error) { allowed := make(map[string]bool, len(relativePaths)) for _, name := range relativePaths { @@ -135,7 +189,7 @@ func validateGitStatusAllowingUntracked(status string, allowedUntracked, authori if entry.code == "??" && !authoritative[name] && !isAuthoritativePath(name) { continue } - return fmt.Errorf("workspace has uncommitted authoritative or tracked changes at %q", name) + return fmt.Errorf("uncommitted changes at %q: desired state is reviewed as a diff, so commit them first — git add -A && git commit -m \"...\"", name) } } return nil @@ -449,6 +503,29 @@ func AssertGitRevision(root, expected string) error { return AssertGitRevisionContext(context.Background(), root, expected) } +// AssertGitRevisionUnchangedContext confirms the workspace is still at the +// revision a read started from. +// +// What that check is for is a commit landing underneath a read that spans +// several files, so it asks about the revision and nothing else. +// AssertGitRevision additionally requires a clean tree, which is right for a +// command about to take effect and wrong for one only describing what is there: +// used by a reading command it refuses to describe a workspace precisely when +// there is something to describe. +func AssertGitRevisionUnchangedContext(ctx context.Context, root, expected string) error { + current, err := gitOutputContext(ctx, root, "rev-parse", "HEAD") + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return err + } + if current != expected { + return errors.New("Git revision changed during operation") + } + return nil +} + func AssertGitRevisionContext(ctx context.Context, root, expected string) error { current, err := RequireCleanGitContext(ctx, root) if err != nil { @@ -826,7 +903,7 @@ func validateGitStatus(status string, allowedChanges, authoritative map[string]b if entry.code == "??" && !authoritative[name] && !isAuthoritativePath(name) { continue } - return fmt.Errorf("workspace has uncommitted authoritative or tracked changes at %q", name) + return fmt.Errorf("uncommitted changes at %q: desired state is reviewed as a diff, so commit them first — git add -A && git commit -m \"...\"", name) } } return nil From 251fa74904a42afd92193e44417b48cb6762331a Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 15:50:55 +0200 Subject: [PATCH 19/26] Say what would have worked --- .gitignore | 3 + README.md | 4 +- cmd/snailmail/main.go | 148 +++++++++++++++++++++------ cmd/snailmail/unknowncommand_test.go | 59 +++++++++++ engine/adoptsession.go | 3 +- engine/collect.go | 2 +- engine/keys.go | 2 +- engine/notconfigured.go | 70 +++++++++++++ engine/placements.go | 3 +- engine/prune.go | 2 +- engine/rollback.go | 2 +- engine/site.go | 6 +- engine/workspace.go | 2 +- formats/format.go | 4 +- internal/state/store.go | 7 +- internal/wire/hosts.go | 3 +- 16 files changed, 272 insertions(+), 48 deletions(-) create mode 100644 cmd/snailmail/unknowncommand_test.go create mode 100644 engine/notconfigured.go diff --git a/.gitignore b/.gitignore index a420188..ba5c88f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ /build/ /dist/ *.test +# Workspace runtime state: a lock and a staging directory, written wherever a +# command runs. Never reviewed, never published. +.snailmail/ diff --git a/README.md b/README.md index 5836d93..c561194 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ go run ./cmd/snailmail add python ./dist/*.whl git add .gitignore snailmail.toml repos/python.lock.toml git commit -m "configure Python repository" go run ./cmd/snailmail plan -go run ./cmd/snailmail apply --plan snailmail.snailmail-plan.json +go run ./cmd/snailmail apply ``` `plan` writes what it intends to do; `apply` does it and records a publication @@ -820,7 +820,7 @@ Render the read-only public matrix and machine-readable status from committed locks, ledgers, deployment receipts, and an optional current plan: ```sh -go run ./cmd/snailmail dashboard --output site +go run ./cmd/snailmail dashboard ``` ## Build layout and binary size diff --git a/cmd/snailmail/main.go b/cmd/snailmail/main.go index e54697e..4a32f2f 100644 --- a/cmd/snailmail/main.go +++ b/cmd/snailmail/main.go @@ -2,7 +2,6 @@ package main import ( "context" - "encoding/json" "errors" "flag" "fmt" @@ -183,7 +182,7 @@ func run(ctx context.Context, args []string, stdout, stderr io.Writer) error { printUsage(stdout) return nil default: - return fmt.Errorf("unknown command %q", args[0]) + return unknownCommand(args[0]) } } @@ -668,7 +667,7 @@ func runCheck(ctx context.Context, args []string, stdout, stderr io.Writer) erro } func runSite(ctx context.Context, args []string, stdout, stderr io.Writer) error { - flags := newCommandFlags("site", stderr).withWorkspace() + flags := newCommandFlags("site", stderr).withWorkspace().withJSON() title := flags.String("title", "", "page title (defaults to the workspace name)") description := flags.String("description", "", "one line shown under the title") output := flags.String("output", "", "where to write the page (defaults to the directory the repositories share)") @@ -684,9 +683,13 @@ func runSite(ctx context.Context, args []string, stdout, stderr io.Writer) error if err != nil { return err } + if done, err := flags.emit(stdout, result); done || err != nil { + return err + } printBrand(stdout) - fmt.Fprintf(stdout, "📦 wrote %s: %d packages across %d repositories\n", - result.Path, result.Packages, result.Repositories) + fmt.Fprintf(stdout, "📦 wrote %s: %d %s across %d %s\n", + result.Path, result.Packages, plural(result.Packages, "package", "packages"), + result.Repositories, plural(result.Repositories, "repository", "repositories")) return nil } @@ -855,14 +858,13 @@ func runDoctor(ctx context.Context, args []string, stdout, stderr io.Writer) err } func runDoctorWithFetcher(ctx context.Context, args []string, stdout, stderr io.Writer, fetcher source.Fetcher) error { - flags := newCommandFlags("doctor", stderr) + flags := newCommandFlags("doctor", stderr).withJSON() format := flags.String("format", "auto", "repository format: auto, pypi, deb, or helm") project := flags.String("project", "", "PyPI project to inspect") suite := flags.String("suite", "", "Debian suite") component := flags.String("component", "", "Debian component") architecture := flags.String("architecture", "", "Debian architecture") maximum := flags.Int("max-artifacts", 4, "maximum referenced artifacts to inspect (1-4)") - jsonOutput := flags.Bool("json", false, "emit machine-readable JSON") if err := flags.Parse(args); err != nil { return err } @@ -876,13 +878,9 @@ func runDoctorWithFetcher(ctx context.Context, args []string, stdout, stderr io. if err != nil { return err } - if *jsonOutput { - encoder := json.NewEncoder(stdout) - encoder.SetIndent("", " ") - if err := encoder.Encode(result); err != nil { - return err - } - } else { + if done, err := flags.emit(stdout, result); err != nil { + return err + } else if !done { printBrand(stdout) fmt.Fprintf(stdout, "📦 inspected %s repository index with %d entries and %d referenced artifacts\n", result.Format, result.Entries, result.ArtifactsChecked) for _, finding := range result.Findings { @@ -906,7 +904,7 @@ func runAdopt(ctx context.Context, args []string, stdout, stderr io.Writer) erro } func runAdoptWithFetcher(ctx context.Context, args []string, stdout, stderr io.Writer, fetcher source.Fetcher) error { - flags := newCommandFlags("adopt", stderr).withWorkspace() + flags := newCommandFlags("adopt", stderr).withWorkspace().withJSON() digest := flags.String("sha256", "", "required artifact SHA-256 pin") filename := flags.String("filename", "", "artifact filename override") name := flags.String("name", "", "package name, for formats whose artifacts carry none") @@ -915,7 +913,6 @@ func runAdoptWithFetcher(ctx context.Context, args []string, stdout, stderr io.W distro := flags.String("distro", "", "Debian placement distribution") dryRun := flags.Bool("dry-run", false, "validate without changing CAS or lock") publicOrigin := flags.Bool("public-origin", false, "confirm URL is public, non-secret, and will be committed") - jsonOutput := flags.Bool("json", false, "emit machine-readable JSON") if err := flags.Parse(args); err != nil { return err } @@ -929,10 +926,8 @@ func runAdoptWithFetcher(ctx context.Context, args []string, stdout, stderr io.W if err != nil { return err } - if *jsonOutput { - encoder := json.NewEncoder(stdout) - encoder.SetIndent("", " ") - return encoder.Encode(result) + if done, err := flags.emit(stdout, result); done || err != nil { + return err } printBrand(stdout) action := "already recorded" @@ -1068,6 +1063,7 @@ func runApply(ctx context.Context, args []string, stdout, stderr io.Writer) erro if !filepath.IsAbs(resolvedApprovalFile) { resolvedApprovalFile = filepath.Join(flags.Root(), resolvedApprovalFile) } + *plan = planFileToRead(flags.Root(), *plan) result, err := engine.ApplyWorkspace(ctx, engine.ApplyWorkspaceRequest{ Progress: applyProgress(stderr, flags.jsonRequested()), Root: flags.Root(), Plan: *plan, StructuralOnly: *structuralOnly, DryRun: *dryRun, @@ -1108,6 +1104,14 @@ func runApply(ctx context.Context, args []string, stdout, stderr io.Writer) erro fmt.Fprintln(stdout, "✉️ dry run: nothing was staged, recorded or published") } fmt.Fprintf(stdout, "✉️ plan sha256:%s\n", result.PlanID) + // The README promises that each command names the one that usually comes + // next, and this was the one that did not — at the moment a person most wants + // to know whether it worked and where to look. + if result.DryRun { + suggestNext(stdout, flags, "snailmail apply", "publish it for real") + } else { + suggestNext(stdout, flags, "snailmail status", "see what is published") + } return nil } @@ -1174,12 +1178,16 @@ func runApprovalKey(args []string, stdout, stderr io.Writer) error { func runRender(args []string, stdout, stderr io.Writer) error { flags := newCommandFlags("dashboard", stderr).withWorkspace().withJSON() - output := flags.String("output", "site", "output directory for the status page") - plan := flags.String("plan", "snailmail.snailmail-plan.json", "optional plan used for pending gates") + // Not "site": that is a different command, producing a different page, and + // the two writing to each other's names is what the rename was for. + output := flags.String("output", "dashboard", "output directory for the status page") + plan := flags.String("plan", defaultPlanFile, "optional plan used for pending gates") if err := flags.parse(args); err != nil { return err } - result, err := engine.RenderStatus(engine.RenderStatusRequest{Root: flags.Root(), Output: *output, Plan: *plan}) + result, err := engine.RenderStatus(engine.RenderStatusRequest{ + Root: flags.Root(), Output: *output, Plan: planFileToRead(flags.Root(), *plan), + }) if err != nil { return err } @@ -1595,8 +1603,8 @@ func printUsage(output io.Writer) { }}, {"Every day", []string{ "add [--name NAME --version VERSION] REPOSITORY ARTIFACT...", - "plan [--out snailmail.snailmail-plan.json]", - "apply [--plan snailmail.snailmail-plan.json] [--dry-run]", + "plan [--out snailmail-plan.json]", + "apply [--plan snailmail-plan.json] [--dry-run]", "status [--workspace DIR] [--json]", }}, {"Curating what is published", []string{ @@ -1630,7 +1638,7 @@ func printUsage(output io.Writer) { }}, {"Pages, sites and CI", []string{ "site [--title TITLE] [--description TEXT] [--output PATH]", - "dashboard [--output site]", + "dashboard [--output dashboard]", "ci [--snailmail-version vX.Y.Z] > .github/workflows/publish.yml", }}, {"Building a repository without a workspace", []string{ @@ -1646,6 +1654,7 @@ func printUsage(output io.Writer) { } fmt.Fprintln(output) fmt.Fprintln(output, "Flags may appear anywhere, and every command that reports a result accepts --json.") + fmt.Fprintln(output, "ci and serve emit a document and a service rather than a result, so they do not.") fmt.Fprintln(output, "--workspace DIR works on every command that reads a workspace.") } @@ -1789,9 +1798,33 @@ func largestLockSuffix(result engine.StatusWorkspaceResult) string { } // defaultPlanFile is where plan writes and apply reads when neither is told -// otherwise. Named because a next-step hint has to agree with it, and three flags -// repeated the literal. -const defaultPlanFile = "snailmail.snailmail-plan.json" +// otherwise. Named because a next-step hint has to agree with it, and three +// flags repeated the literal. +// +// It used to be "snailmail.snailmail-plan.json", which says snailmail twice and +// was written out in full in the quickstart. legacyPlanFile is the old name, +// still read where the new one is absent so an existing workspace with a plan in +// flight does not stop working on an upgrade. +const ( + defaultPlanFile = "snailmail-plan.json" + legacyPlanFile = "snailmail.snailmail-plan.json" +) + +// planFileToRead is the plan a command should open when it was given the +// default. A workspace holding a plan under the old name keeps working: the +// rename is worth doing and not worth interrupting somebody mid-publication for. +func planFileToRead(root, given string) string { + if given != defaultPlanFile { + return given + } + if _, err := os.Stat(filepath.Join(root, given)); err == nil { + return given + } + if _, err := os.Stat(filepath.Join(root, legacyPlanFile)); err == nil { + return legacyPlanFile + } + return given +} func runRollback(ctx context.Context, args []string, stdout, stderr io.Writer) error { flags := newCommandFlags("rollback", stderr).withWorkspace().withJSON() @@ -1963,3 +1996,60 @@ func runImport(ctx context.Context, args []string, stdout, stderr io.Writer) err } return nil } + +// knownCommands is every command run dispatches, for the message a person gets +// when they mistype one. Kept beside that switch, and held to it by a test — a +// list that drifts is worse than no list, because it teaches the reader that a +// command they can see does not exist. +var knownCommands = []string{ + "add", "adopt", "apply", "approval-key", "approve", "blob-store", "build", + "check", "ci", "collect", "dashboard", "doctor", "help", "import", "init", + "keys", "plan", "promote", "prune", "rollback", "rollout", "serve", "setup", + "site", "status", "verify", "version", "yank", +} + +// unknownCommand says what was probably meant, and where to look otherwise. +func unknownCommand(typed string) error { + if closest := nearestCommand(typed); closest != "" { + return fmt.Errorf("unknown command %q; did you mean %q? (snailmail help lists them all)", typed, closest) + } + return fmt.Errorf("unknown command %q; snailmail help lists them", typed) +} + +func nearestCommand(typed string) string { + best, bestDistance := "", 0 + for _, candidate := range knownCommands { + distance := commandDistance(typed, candidate) + // A third of the length, so a short name tolerates one slip and a long one + // several, and nothing unrelated is ever proposed. + limit := max(len(candidate)/3, 1) + if distance > limit { + continue + } + if best == "" || distance < bestDistance { + best, bestDistance = candidate, distance + } + } + return best +} + +// commandDistance is Levenshtein, iterative with one row. +func commandDistance(from, to string) int { + previous := make([]int, len(to)+1) + for index := range previous { + previous[index] = index + } + for fromIndex := 1; fromIndex <= len(from); fromIndex++ { + current := make([]int, len(to)+1) + current[0] = fromIndex + for toIndex := 1; toIndex <= len(to); toIndex++ { + cost := 1 + if from[fromIndex-1] == to[toIndex-1] { + cost = 0 + } + current[toIndex] = min(current[toIndex-1]+1, min(previous[toIndex]+1, previous[toIndex-1]+cost)) + } + previous = current + } + return previous[len(to)] +} diff --git a/cmd/snailmail/unknowncommand_test.go b/cmd/snailmail/unknowncommand_test.go new file mode 100644 index 0000000..f7eae2b --- /dev/null +++ b/cmd/snailmail/unknowncommand_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "bytes" + "context" + "strings" + "testing" +) + +// The suggestion list has to be the commands that exist. +// +// A list that drifts is worse than no list: it teaches a reader that a command +// they can see does not work, or proposes one that never did. This runs every +// name through the dispatcher and requires it to be recognised — "unknown +// command" back from a name in the list is the drift itself. +func TestEverySuggestedCommandExists(t *testing.T) { + // In a scratch directory: a few of these acquire a workspace lock or write + // state where they are run, and a test that leaves .snailmail/ in the source + // tree is a test that gets its leavings committed. + t.Chdir(t.TempDir()) + for _, name := range knownCommands { + var stdout, stderr bytes.Buffer + // Every command is run with no arguments, which most refuse. What matters + // is that they refuse for their own reasons rather than by not existing. + err := run(context.Background(), []string{name}, &stdout, &stderr) + if err != nil && strings.Contains(err.Error(), "unknown command") { + t.Errorf("%q is suggested but the dispatcher does not know it", name) + } + } +} + +func TestUnknownCommandSuggestsTheLikelyOne(t *testing.T) { + for typed, want := range map[string]string{ + "stauts": "status", + "aply": "apply", + "promot": "promote", + "verfiy": "verify", + "keys": "", // exists, so it is never routed here + "xyzzy": "", + "frobnic": "", + } { + got := nearestCommand(typed) + if want == "" && got != "" && typed != "keys" { + t.Errorf("%q was answered with %q, which is a guess rather than a correction", typed, got) + } + if want != "" && got != want { + t.Errorf("nearestCommand(%q) = %q, want %q", typed, got, want) + } + } +} + +// Whatever the guess, the message points at the one place that lists them all. +func TestUnknownCommandAlwaysNamesHelp(t *testing.T) { + for _, typed := range []string{"stauts", "xyzzy"} { + if err := unknownCommand(typed); !strings.Contains(err.Error(), "snailmail help") { + t.Errorf("unknownCommand(%q) = %v, which does not say where to look", typed, err) + } + } +} diff --git a/engine/adoptsession.go b/engine/adoptsession.go index cd558a5..77ec9ad 100644 --- a/engine/adoptsession.go +++ b/engine/adoptsession.go @@ -3,7 +3,6 @@ package engine import ( "context" "errors" - "fmt" "github.com/shellcell/snailmail/internal/state" ) @@ -59,7 +58,7 @@ func loadAdoptSession(ctx context.Context, root, repositoryName string) (*adoptS } repository, exists := manifest.Repositories[repositoryName] if !exists { - return nil, fmt.Errorf("repository %q is not configured", repositoryName) + return nil, unknownRepository(manifest, repositoryName) } lock, err := state.LoadLock(root, repository) if err != nil { diff --git a/engine/collect.go b/engine/collect.go index d148b1a..bbd21c0 100644 --- a/engine/collect.go +++ b/engine/collect.go @@ -140,7 +140,7 @@ func CollectWorkspace(ctx context.Context, request CollectWorkspaceRequest) (Col result.RemovedBytes += reported.RemovedBytes } if request.Repository != "" && len(result.Repositories) == 0 { - return CollectWorkspaceResult{}, fmt.Errorf("repository %q is not configured", request.Repository) + return CollectWorkspaceResult{}, unknownRepository(manifest, request.Repository) } return result, nil } diff --git a/engine/keys.go b/engine/keys.go index 2e4b381..205fbae 100644 --- a/engine/keys.go +++ b/engine/keys.go @@ -450,7 +450,7 @@ func AttachKey(request AttachKeyRequest) (AttachKeyResult, error) { } repository, exists := manifest.Repositories[request.Repository] if !exists { - return AttachKeyResult{}, fmt.Errorf("repository %q is not configured", request.Repository) + return AttachKeyResult{}, unknownRepository(manifest, request.Repository) } key, exists := manifest.Keys[request.Key] if !exists { diff --git a/engine/notconfigured.go b/engine/notconfigured.go new file mode 100644 index 0000000..29efe01 --- /dev/null +++ b/engine/notconfigured.go @@ -0,0 +1,70 @@ +package engine + +import ( + "fmt" + "strings" + + "github.com/shellcell/snailmail/internal/state" +) + +// unknownRepository reports a repository the workspace does not have, and names +// the ones it does. +// +// A name is either a typo or a misremembering, and both are answered by the +// list. Seven commands raised this and none of them said what would have +// worked, so the next move was to open snailmail.toml — for a question the +// program could answer. +func unknownRepository(manifest state.Manifest, name string) error { + configured := state.RepositoryNames(manifest) + if len(configured) == 0 { + return fmt.Errorf("repository %q is not configured; this workspace has none yet — snailmail setup --name %s", name, name) + } + if closest := nearestName(name, configured); closest != "" { + return fmt.Errorf("repository %q is not configured; did you mean %q? (configured: %s)", + name, closest, strings.Join(configured, ", ")) + } + return fmt.Errorf("repository %q is not configured; configured: %s", name, strings.Join(configured, ", ")) +} + +// nearestName is the candidate within a small edit distance of what was typed, +// or empty when nothing is close enough to suggest without guessing. +func nearestName(typed string, candidates []string) string { + best, bestDistance := "", 0 + for _, candidate := range candidates { + distance := editDistance(typed, candidate) + // A third of the length, so a short name tolerates one slip and a long one + // several, and nothing unrelated is ever proposed. + limit := max(len(candidate)/3, 1) + if distance > limit { + continue + } + if best == "" || distance < bestDistance { + best, bestDistance = candidate, distance + } + } + return best +} + +// editDistance is Levenshtein, iterative with one row. +func editDistance(from, to string) int { + previous := make([]int, len(to)+1) + for index := range previous { + previous[index] = index + } + for fromIndex := 1; fromIndex <= len(from); fromIndex++ { + current := make([]int, len(to)+1) + current[0] = fromIndex + for toIndex := 1; toIndex <= len(to); toIndex++ { + cost := 1 + if from[fromIndex-1] == to[toIndex-1] { + cost = 0 + } + current[toIndex] = min( + current[toIndex-1]+1, + min(previous[toIndex]+1, previous[toIndex-1]+cost), + ) + } + previous = current + } + return previous[len(to)] +} diff --git a/engine/placements.go b/engine/placements.go index 193915e..bba0a23 100644 --- a/engine/placements.go +++ b/engine/placements.go @@ -2,7 +2,6 @@ package engine import ( "errors" - "fmt" "github.com/shellcell/snailmail/internal/state" ) @@ -63,7 +62,7 @@ func mutatePlacement(request PlacementMutationRequest, promote bool) (PlacementM } repository, exists := manifest.Repositories[request.Repository] if !exists { - return PlacementMutationResult{}, fmt.Errorf("repository %q is not configured", request.Repository) + return PlacementMutationResult{}, unknownRepository(manifest, request.Repository) } lock, err := state.LoadLock(root, repository) if err != nil { diff --git a/engine/prune.go b/engine/prune.go index 1ebf18b..e55cafc 100644 --- a/engine/prune.go +++ b/engine/prune.go @@ -41,7 +41,7 @@ func Prune(request PruneRequest) (PruneResult, error) { } repository, exists := manifest.Repositories[request.Repository] if !exists { - return PruneResult{}, fmt.Errorf("repository %q is not configured", request.Repository) + return PruneResult{}, unknownRepository(manifest, request.Repository) } lock, err := state.LoadLock(root, repository) if err != nil { diff --git a/engine/rollback.go b/engine/rollback.go index 6b164ad..25c5d78 100644 --- a/engine/rollback.go +++ b/engine/rollback.go @@ -68,7 +68,7 @@ func RollbackRepository(ctx context.Context, request RollbackRepositoryRequest) } repository, exists := manifest.Repositories[request.Repository] if !exists { - return RollbackRepositoryResult{}, fmt.Errorf("repository %q is not configured", request.Repository) + return RollbackRepositoryResult{}, unknownRepository(manifest, request.Repository) } hosts := request.Hosts hostIdentity, err := repositoryHostIdentity(repository) diff --git a/engine/site.go b/engine/site.go index 4e1b547..bdfe806 100644 --- a/engine/site.go +++ b/engine/site.go @@ -29,11 +29,11 @@ type SiteIndexRequest struct { type SiteIndexResult struct { // Path is where the page was written. - Path string + Path string `json:"path"` // Repositories and Packages are what it describes, reported so a caller can // tell an overview of an empty workspace from one that failed to see it. - Repositories int - Packages int + Repositories int `json:"repositories"` + Packages int `json:"packages"` } // SiteIndex writes the page that sits above the repositories. diff --git a/engine/workspace.go b/engine/workspace.go index 18632e9..55c1140 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -336,7 +336,7 @@ func AddArtifacts(request AddArtifactsRequest) (AddArtifactsResult, error) { } repository, exists := manifest.Repositories[request.Repository] if !exists { - return AddArtifactsResult{}, fmt.Errorf("repository %q is not configured", request.Repository) + return AddArtifactsResult{}, unknownRepository(manifest, request.Repository) } selectedFormat, err := formats.For(repository.Format) if err != nil { diff --git a/formats/format.go b/formats/format.go index 725edd1..6c4fe26 100644 --- a/formats/format.go +++ b/formats/format.go @@ -19,6 +19,7 @@ import ( "io" "path" "sort" + "strings" "time" "github.com/shellcell/snailmail/formats/apk" @@ -202,7 +203,8 @@ var registry = map[string]Format{ func For(name string) (Format, error) { format, known := registry[name] if !known { - return nil, fmt.Errorf("unsupported repository format %q", name) + return nil, fmt.Errorf("unsupported repository format %q; supported: %s", + name, strings.Join(Names(), ", ")) } return format, nil } diff --git a/internal/state/store.go b/internal/state/store.go index 1533816..f8c34dc 100644 --- a/internal/state/store.go +++ b/internal/state/store.go @@ -89,8 +89,8 @@ func Setup(root string, options SetupOptions) error { if err := ValidateRepositoryName(options.Name); err != nil { return err } - if !formats.Supported(options.Format) { - return fmt.Errorf("unsupported repository format %q", options.Format) + if _, err := formats.For(options.Format); err != nil { + return err } hostType := options.HostType if hostType == "" { @@ -609,7 +609,8 @@ func validateRepositoryHost(name string, repository Repository) error { } } default: - return fmt.Errorf("repository %q has unsupported host type %q", name, repository.Host.Type) + return fmt.Errorf("repository %q has unsupported host type %q; supported: %s", + name, repository.Host.Type, strings.Join(host.KnownHostTypes(), ", ")) } return nil } diff --git a/internal/wire/hosts.go b/internal/wire/hosts.go index 4d6c7bf..f1b34f0 100644 --- a/internal/wire/hosts.go +++ b/internal/wire/hosts.go @@ -3,6 +3,7 @@ package wire import ( "context" "fmt" + "strings" "sync" commandcredential "github.com/shellcell/snailmail/adapters/credential/command" @@ -48,7 +49,7 @@ func (resolver *HostResolver) Resolve(ctx context.Context, repository host.Repos case "github-pages": return githubpages.New(), nil default: - return nil, fmt.Errorf("unsupported host type %q", repository.Type) + return nil, fmt.Errorf("unsupported host type %q; supported: %s", repository.Type, strings.Join(host.KnownHostTypes(), ", ")) } } From e5c675c75ad400a716738e6509b4b344db0233ac Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 16:00:26 +0200 Subject: [PATCH 20/26] Give the output one vocabulary --- cmd/snailmail/main.go | 223 +++++++++++++++++++++++------------------- 1 file changed, 125 insertions(+), 98 deletions(-) diff --git a/cmd/snailmail/main.go b/cmd/snailmail/main.go index 4a32f2f..566f3d5 100644 --- a/cmd/snailmail/main.go +++ b/cmd/snailmail/main.go @@ -211,21 +211,21 @@ func runInit(args []string, stdout, stderr io.Writer) error { // Said rather than done silently: this wrote to the operator's directory beyond // the files they asked for. if created.CreatedGitRepository { - fmt.Fprintln(stdout, "📦 created a Git repository here, because snailmail reviews its state as a diff") + reportDone(stdout, "created a Git repository here, because snailmail reviews its state as a diff") } if created.Committed { - fmt.Fprintln(stdout, "📦 committed the new workspace, so it is ready to use") + reportDone(stdout, "committed the new workspace, so it is ready to use") } // The workspace is written and correct; git simply cannot say who made the // commit. Naming both commands, because someone who has never configured git is // exactly the person who will not know the second one either. if created.CommitPending { - fmt.Fprintln(stderr, "⚠️ the workspace was created but not committed: Git has no configured identity") + reportWarning(stderr, "the workspace was created but not committed: Git has no configured identity") fmt.Fprintln(stderr, " git config --global user.name \"Your Name\"") fmt.Fprintln(stderr, " git config --global user.email \"you@example.com\"") fmt.Fprintln(stderr, " git -C . add -A && git -C . commit -m \"snailmail workspace\"") } - fmt.Fprintf(stdout, "✉️ initialized workspace %s\n", *name) + reportDetail(stdout, "initialized workspace %s", *name) suggestNext(stdout, flags, "snailmail setup raw --name tools --output public/tools", "configure a repository") return nil @@ -314,8 +314,8 @@ func runSetup(args []string, stdout, stderr io.Writer) error { return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 configured %s repository %s\n", format, *name) - fmt.Fprintf(stdout, "✉️ desired state will publish to %s\n", target) + reportDone(stdout, "configured %s repository %s", format, *name) + reportDetail(stdout, "desired state will publish to %s", target) suggestNext(stdout, flags, "snailmail add "+*name+" ./path/to/artifact", "record an artifact") return nil @@ -349,8 +349,8 @@ func runKeys(ctx context.Context, args []string, stdout, stderr io.Writer) error return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 generated signing key %s\n", result.Name) - fmt.Fprintf(stdout, "✉️ fingerprint %s; expires %s\n", result.Fingerprint, result.ExpiresAt) + reportDone(stdout, "generated signing key %s", result.Name) + reportDetail(stdout, "fingerprint %s; expires %s", result.Fingerprint, result.ExpiresAt) fmt.Fprintf(stdout, " key reference %s\n", result.Reference) return nil case "attach": @@ -371,8 +371,8 @@ func runKeys(ctx context.Context, args []string, stdout, stderr io.Writer) error return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 %s is now signed by %s\n", result.Repository, result.Key) - fmt.Fprintf(stdout, "✉️ fingerprint %s\n", result.Fingerprint) + reportDone(stdout, "%s is now signed by %s", result.Repository, result.Key) + reportDetail(stdout, "fingerprint %s", result.Fingerprint) fmt.Fprintf(stdout, " clients install %s\n", result.Keyring) return nil case "publish": @@ -396,8 +396,8 @@ func runKeys(ctx context.Context, args []string, stdout, stderr io.Writer) error return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 published public forms for %s\n", result.Name) - fmt.Fprintf(stdout, "✉️ fingerprint %s\n", result.Fingerprint) + reportDone(stdout, "published public forms for %s", result.Name) + reportDetail(stdout, "fingerprint %s", result.Fingerprint) return nil case "rotate": if len(args) < 2 { @@ -438,8 +438,8 @@ func runKeys(ctx context.Context, args []string, stdout, stderr io.Writer) error return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 signing rotation for %s is %s\n", result.Repository, result.Phase) - fmt.Fprintf(stdout, "✉️ active %s; trusted %s\n", result.ActiveKey, strings.Join(result.TrustedKeys, ", ")) + reportDone(stdout, "signing rotation for %s is %s", result.Repository, result.Phase) + reportDetail(stdout, "active %s; trusted %s", result.ActiveKey, strings.Join(result.TrustedKeys, ", ")) if result.EarliestAdvance != "" { fmt.Fprintf(stdout, " earliest next transition %s\n", result.EarliestAdvance) } @@ -471,7 +471,7 @@ func runKeys(ctx context.Context, args []string, stdout, stderr io.Writer) error } printBrand(stdout) if len(result.Findings) == 0 { - fmt.Fprintln(stdout, "📦 signing keys and repository compatibility are valid") + reportDone(stdout, "signing keys and repository compatibility are valid") } for _, rotation := range result.Rotations { state := "awaiting deployment" @@ -480,11 +480,11 @@ func runKeys(ctx context.Context, args []string, stdout, stderr io.Writer) error } else if rotation.Deployed { state = "waiting until " + rotation.EarliestAdvance } - fmt.Fprintf(stdout, "📦 rotation %s is %s: %s\n", rotation.Repository, rotation.Phase, state) + reportDone(stdout, "rotation %s is %s: %s", rotation.Repository, rotation.Phase, state) } hasErrors := false for _, finding := range result.Findings { - fmt.Fprintf(stdout, "✉️ %s %s: %s\n", finding.Severity, finding.Subject, finding.Message) + reportDetail(stdout, "%s %s: %s", finding.Severity, finding.Subject, finding.Message) hasErrors = hasErrors || finding.Severity == "error" } if hasErrors { @@ -518,13 +518,14 @@ func runAdd(ctx context.Context, args []string, stdout, stderr io.Writer) error return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 locked %d %s in %s", result.Added, plural(result.Added, "artifact", "artifacts"), result.Repository) + locked := fmt.Sprintf("locked %d %s in %s", + result.Added, plural(result.Added, "artifact", "artifacts"), result.Repository) if result.Skipped != 0 { - fmt.Fprintf(stdout, " (%d already present)", result.Skipped) + locked += fmt.Sprintf(" (%d already present)", result.Skipped) } - fmt.Fprintln(stdout) + reportDone(stdout, "%s", locked) for _, packageVersion := range result.Packages { - fmt.Fprintf(stdout, "✉️ %s\n", packageVersion) + reportDetail(stdout, "%s", packageVersion) } // Commit first, and say why: plan refuses an uncommitted workspace, which is the // first refusal most people meet and reads as a bug until the reason is clear. @@ -555,9 +556,9 @@ func runPromote(args []string, stdout, stderr io.Writer) error { } printBrand(stdout) if result.Changed == 0 { - fmt.Fprintf(stdout, "📦 placement %s@%s is already present in %s/%s\n", result.Package, result.Version, result.Repository, result.Track) + reportDone(stdout, "placement %s@%s is already present in %s/%s", result.Package, result.Version, result.Repository, result.Track) } else { - fmt.Fprintf(stdout, "📦 placed %s@%s in %s/%s\n", result.Package, result.Version, result.Repository, result.Track) + reportDone(stdout, "placed %s@%s in %s/%s", result.Package, result.Version, result.Repository, result.Track) } return nil } @@ -584,11 +585,11 @@ func runYank(args []string, stdout, stderr io.Writer) error { } printBrand(stdout) if result.All { - fmt.Fprintf(stdout, "📦 removed %d %s for %s@%s from %s\n", result.Changed, plural(result.Changed, "placement", "placements"), result.Package, result.Version, result.Repository) + reportDone(stdout, "removed %d %s for %s@%s from %s", result.Changed, plural(result.Changed, "placement", "placements"), result.Package, result.Version, result.Repository) } else if result.Changed == 0 { - fmt.Fprintf(stdout, "📦 placement %s@%s is already absent from %s/%s\n", result.Package, result.Version, result.Repository, result.Track) + reportDone(stdout, "placement %s@%s is already absent from %s/%s", result.Package, result.Version, result.Repository, result.Track) } else { - fmt.Fprintf(stdout, "📦 removed %s@%s from %s/%s\n", result.Package, result.Version, result.Repository, result.Track) + reportDone(stdout, "removed %s@%s from %s/%s", result.Package, result.Version, result.Repository, result.Track) } return nil } @@ -615,11 +616,11 @@ func runPrune(args []string, stdout, stderr io.Writer) error { } printBrand(stdout) if result.Removed == 0 { - fmt.Fprintf(stdout, "📦 %s already retains at most %d %s per placement view\n", result.Repository, result.Keep, plural(result.Keep, "version", "versions")) + reportDone(stdout, "%s already retains at most %d %s per placement view", result.Repository, result.Keep, plural(result.Keep, "version", "versions")) } else { - fmt.Fprintf(stdout, "📦 removed %d old %s from %s placements\n", result.Removed, plural(result.Removed, "placement", "placements"), result.Repository) + reportDone(stdout, "removed %d old %s from %s placements", result.Removed, plural(result.Removed, "placement", "placements"), result.Repository) } - fmt.Fprintln(stdout, "✉️ package versions and blobs were retained") + reportDetail(stdout, "package versions and blobs were retained") return nil } @@ -650,15 +651,15 @@ func runCheck(ctx context.Context, args []string, stdout, stderr io.Writer) erro return nil } printBrand(stdout) - fmt.Fprintf(stdout, "📦 checked %d %s, %d package versions, and %d locked %s\n", + reportDone(stdout, "checked %d %s, %d package versions, and %d locked %s", result.Repositories, plural(result.Repositories, "repository", "repositories"), result.PackageVersions, result.Artifacts, plural(result.Artifacts, "artifact", "artifacts")) for _, finding := range result.Findings { - fmt.Fprintf(stdout, "✉️ [%s] %s: %s\n", finding.State, finding.Subject, finding.Message) + reportDetail(stdout, "[%s] %s: %s", finding.State, finding.Subject, finding.Message) } if *origins { - fmt.Fprintf(stdout, "✉️ checked %d recorded origins and skipped %d beyond the limit; artifacts without origins remain unavailable for source comparison\n", result.OriginsChecked, result.OriginsSkipped) + reportDetail(stdout, "checked %d recorded origins and skipped %d beyond the limit; artifacts without origins remain unavailable for source comparison", result.OriginsChecked, result.OriginsSkipped) } else { - fmt.Fprintln(stdout, "✉️ adopted-origin checks disabled; use --origins to re-fetch recorded pins") + reportDetail(stdout, "adopted-origin checks disabled; use --origins to re-fetch recorded pins") } if len(result.Findings) != 0 { return fmt.Errorf("check found %d unavailable or changed %s", len(result.Findings), plural(len(result.Findings), "artifact", "artifacts")) @@ -687,7 +688,7 @@ func runSite(ctx context.Context, args []string, stdout, stderr io.Writer) error return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 wrote %s: %d %s across %d %s\n", + reportDone(stdout, "wrote %s: %d %s across %d %s", result.Path, result.Packages, plural(result.Packages, "package", "packages"), result.Repositories, plural(result.Repositories, "repository", "repositories")) return nil @@ -741,18 +742,18 @@ func runRollout(ctx context.Context, args []string, stdout, stderr io.Writer) er return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 workspace %s at %s\n", result.Workspace, result.GitRevision) + reportDone(stdout, "workspace %s at %s", result.Workspace, result.GitRevision) for _, release := range result.Releases { note := fmt.Sprintf(", in %d published %s", release.Publications, plural(release.Publications, "tree", "trees")) if !release.Served { note += ", no longer served" } - fmt.Fprintf(stdout, "✉️ %s: %s@%s first published %s%s\n", + reportDetail(stdout, "%s: %s@%s first published %s%s", release.Repository, release.Package, release.Version, release.PublishedAt, note) } if len(result.Releases) == 0 { - fmt.Fprintln(stdout, "✉️ nothing has been published from this workspace") + reportDetail(stdout, "nothing has been published from this workspace") } return nil } @@ -770,14 +771,14 @@ func runStatus(ctx context.Context, args []string, stdout, stderr io.Writer) err return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 workspace %s at %s\n", result.Workspace, shortDigest(result.GitRevision)) + reportDone(stdout, "workspace %s at %s", result.Workspace, shortDigest(result.GitRevision)) // Said first, because it changes what everything below means. Everything here // is read at the revision above, so uncommitted work is not in it — and a // reader comparing this against their working tree needs to know that before // they read the numbers, not after. if len(result.Uncommitted) != 0 { - fmt.Fprintf(stdout, "⚠️ %d uncommitted %s, not included below:\n", + reportWarning(stdout, "%d uncommitted %s, not included below:", len(result.Uncommitted), plural(len(result.Uncommitted), "change", "changes")) for _, name := range result.Uncommitted { fmt.Fprintf(stdout, " %s\n", name) @@ -786,7 +787,7 @@ func runStatus(ctx context.Context, args []string, stdout, stderr io.Writer) err } for _, repository := range result.Repositories { - fmt.Fprintf(stdout, "✉️ %s (%s): %s\n", repository.Name, repository.Format, describePackages(repository)) + reportDetail(stdout, "%s (%s): %s", repository.Name, repository.Format, describePackages(repository)) if detail := describeRepositoryState(repository); detail != "" { fmt.Fprintf(stdout, " %s\n", detail) } @@ -796,14 +797,14 @@ func runStatus(ctx context.Context, args []string, stdout, stderr io.Writer) err // the workspace rather than per repository, with the largest named, because // that is the pair an operator can act on. if result.LockBytes != 0 { - fmt.Fprintf(stdout, "📦 %s of lock across %d %s%s\n", + reportDone(stdout, "%s of lock across %d %s%s", humanBytes(result.LockBytes), len(result.Repositories), plural(len(result.Repositories), "repository", "repositories"), largestLockSuffix(result)) } // What this ran on, rather than a list of what it did not do. The old line // named five things that were not observed and left the reader to work out // whether that mattered. - fmt.Fprintln(stdout, "✉️ read from committed workspace state; no host was contacted") + reportDetail(stdout, "read from committed workspace state; no host was contacted") return nil } @@ -882,9 +883,9 @@ func runDoctorWithFetcher(ctx context.Context, args []string, stdout, stderr io. return err } else if !done { printBrand(stdout) - fmt.Fprintf(stdout, "📦 inspected %s repository index with %d entries and %d referenced artifacts\n", result.Format, result.Entries, result.ArtifactsChecked) + reportDone(stdout, "inspected %s repository index with %d entries and %d referenced artifacts", result.Format, result.Entries, result.ArtifactsChecked) for _, finding := range result.Findings { - fmt.Fprintf(stdout, "✉️ [%s] %s %s: %s\n", finding.Severity, finding.Code, finding.Subject, finding.Message) + reportDetail(stdout, "[%s] %s %s: %s", finding.Severity, finding.Code, finding.Subject, finding.Message) } } errorsFound := 0 @@ -936,8 +937,8 @@ func runAdoptWithFetcher(ctx context.Context, args []string, stdout, stderr io.W } else if result.Changed { action = "recorded" } - fmt.Fprintf(stdout, "📦 %s %s@%s from pinned selected bytes\n", action, result.Package, result.Version) - fmt.Fprintf(stdout, "✉️ sha256:%s %s\n", result.SHA256, result.OriginURL) + reportDone(stdout, "%s %s@%s from pinned selected bytes", action, result.Package, result.Version) + reportDetail(stdout, "sha256:%s %s", result.SHA256, result.OriginURL) return nil } @@ -965,8 +966,8 @@ func runBlobStore(ctx context.Context, args []string, stdout, stderr io.Writer) return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 configured %s blob storage\n", storeType) - fmt.Fprintln(stdout, "✉️ existing locked artifacts are durable") + reportDone(stdout, "configured %s blob storage", storeType) + reportDetail(stdout, "existing locked artifacts are durable") return nil } @@ -1001,8 +1002,8 @@ func runPlan(ctx context.Context, args []string, stdout, stderr io.Writer) error return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 planned %d repository %s\n", result.Changes, plural(result.Changes, "change", "changes")) - fmt.Fprintf(stdout, "✉️ %s\n", result.Output) + reportDone(stdout, "planned %d repository %s", result.Changes, plural(result.Changes, "change", "changes")) + reportDetail(stdout, "%s", result.Output) fmt.Fprintf(stdout, " plan sha256:%s\n", result.PlanID) if result.Changes != 0 { // apply defaults to the same file plan defaults to, so the common case needs @@ -1015,7 +1016,7 @@ func runPlan(ctx context.Context, args []string, stdout, stderr io.Writer) error suggestNext(stdout, flags, next, "publish it") } for _, acquisition := range result.Acquisitions { - fmt.Fprintf(stdout, "✉️ adopted %s/%s@%s %s sha256:%s\n", + reportDetail(stdout, "adopted %s/%s@%s %s sha256:%s", acquisition.Repository, acquisition.Package, acquisition.Version, acquisition.OriginURL, acquisition.SHA256) } return nil @@ -1077,11 +1078,12 @@ func runApply(ctx context.Context, args []string, stdout, stderr io.Writer) erro if err != nil { if result.Applied != 0 || result.Current != 0 { printBrand(stderr) - fmt.Fprintf(stderr, "📦 applied %d repository %s before failure", result.Applied, plural(result.Applied, "change", "changes")) + partial := fmt.Sprintf("applied %d repository %s before failure", + result.Applied, plural(result.Applied, "change", "changes")) if result.Current != 0 { - fmt.Fprintf(stderr, " (%d already current)", result.Current) + partial += fmt.Sprintf(" (%d already current)", result.Current) } - fmt.Fprintln(stderr) + reportDone(stderr, "%s", partial) } return err } @@ -1095,15 +1097,16 @@ func runApply(ctx context.Context, args []string, stdout, stderr io.Writer) erro // question and a reader skimming would otherwise take it for a publication. verb = "would apply" } - fmt.Fprintf(stdout, "📦 %s %d repository %s", verb, result.Applied, plural(result.Applied, "change", "changes")) + applied := fmt.Sprintf("%s %d repository %s", + verb, result.Applied, plural(result.Applied, "change", "changes")) if result.Current != 0 { - fmt.Fprintf(stdout, " (%d already current)", result.Current) + applied += fmt.Sprintf(" (%d already current)", result.Current) } - fmt.Fprintln(stdout) + reportDone(stdout, "%s", applied) if result.DryRun { - fmt.Fprintln(stdout, "✉️ dry run: nothing was staged, recorded or published") + reportDetail(stdout, "dry run: nothing was staged, recorded or published") } - fmt.Fprintf(stdout, "✉️ plan sha256:%s\n", result.PlanID) + reportDetail(stdout, "plan sha256:%s", result.PlanID) // The README promises that each command names the one that usually comes // next, and this was the one that did not — at the moment a person most wants // to know whether it worked and where to look. @@ -1140,8 +1143,8 @@ func runApprove(args []string, stdout, stderr io.Writer) error { return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 approved repository %s\n", *repository) - fmt.Fprintf(stdout, "✉️ %s\n", result.Output) + reportDone(stdout, "approved repository %s", *repository) + reportDetail(stdout, "%s", result.Output) fmt.Fprintf(stdout, " plan sha256:%s\n", result.PlanID) fmt.Fprintf(stdout, " approver %s\n", result.Approver) return nil @@ -1170,8 +1173,8 @@ func runApprovalKey(args []string, stdout, stderr io.Writer) error { return err } printBrand(stdout) - fmt.Fprintln(stdout, "📦 generated Ed25519 approval key") - fmt.Fprintf(stdout, "✉️ %s\n", *output) + reportDone(stdout, "generated Ed25519 approval key") + reportDetail(stdout, "%s", *output) fmt.Fprintf(stdout, " public key %s\n", publicKey) return nil } @@ -1195,8 +1198,8 @@ func runRender(args []string, stdout, stderr io.Writer) error { return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 rendered %d repository %s\n", result.Repositories, plural(result.Repositories, "status", "statuses")) - fmt.Fprintf(stdout, "✉️ %s\n", result.Output) + reportDone(stdout, "rendered %d repository %s", result.Repositories, plural(result.Repositories, "status", "statuses")) + reportDetail(stdout, "%s", result.Output) return nil } @@ -1240,8 +1243,8 @@ func runBuildPyPI(ctx context.Context, args []string, stdout, stderr io.Writer) return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 packed %d %s across %d %s\n", result.DistributionCount, plural(result.DistributionCount, "distribution", "distributions"), result.ProjectCount, plural(result.ProjectCount, "project", "projects")) - fmt.Fprintf(stdout, "✉️ wrote %s to %s\n", result.Format, result.Output) + reportDone(stdout, "packed %d %s across %d %s", result.DistributionCount, plural(result.DistributionCount, "distribution", "distributions"), result.ProjectCount, plural(result.ProjectCount, "project", "projects")) + reportDetail(stdout, "wrote %s to %s", result.Format, result.Output) fmt.Fprintf(stdout, " tree sha256:%s\n", result.TreeSHA256) return nil } @@ -1276,8 +1279,8 @@ func runBuildDeb(ctx context.Context, args []string, stdout, stderr io.Writer) e return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 indexed %d %s across %d %s\n", result.DistributionCount, plural(result.DistributionCount, "package file", "package files"), result.PackageCount, plural(result.PackageCount, "package", "packages")) - fmt.Fprintf(stdout, "✉️ wrote %s to %s\n", result.Format, result.Output) + reportDone(stdout, "indexed %d %s across %d %s", result.DistributionCount, plural(result.DistributionCount, "package file", "package files"), result.PackageCount, plural(result.PackageCount, "package", "packages")) + reportDetail(stdout, "wrote %s to %s", result.Format, result.Output) fmt.Fprintf(stdout, " tree sha256:%s\n", result.TreeSHA256) return nil } @@ -1302,8 +1305,8 @@ func runBuildHelm(ctx context.Context, args []string, stdout, stderr io.Writer) return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 indexed %d %s across %d %s\n", result.DistributionCount, plural(result.DistributionCount, "chart archive", "chart archives"), result.PackageCount, plural(result.PackageCount, "chart", "charts")) - fmt.Fprintf(stdout, "✉️ wrote %s to %s\n", result.Format, result.Output) + reportDone(stdout, "indexed %d %s across %d %s", result.DistributionCount, plural(result.DistributionCount, "chart archive", "chart archives"), result.PackageCount, plural(result.PackageCount, "chart", "charts")) + reportDetail(stdout, "wrote %s to %s", result.Format, result.Output) fmt.Fprintf(stdout, " tree sha256:%s\n", result.TreeSHA256) return nil } @@ -1350,11 +1353,11 @@ func runVerifyPyPI(ctx context.Context, args []string, stdout, stderr io.Writer) return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 verified %d repository %s\n", result.FileCount, plural(result.FileCount, "file", "files")) + reportDone(stdout, "verified %d repository %s", result.FileCount, plural(result.FileCount, "file", "files")) if *structuralOnly { - fmt.Fprintln(stdout, "✉️ structural verification passed") + reportDetail(stdout, "structural verification passed") } else { - fmt.Fprintf(stdout, "✉️ pip installed %d %s from the staged repository\n", result.InstalledCases, plural(result.InstalledCases, "package version", "package versions")) + reportDetail(stdout, "pip installed %d %s from the staged repository", result.InstalledCases, plural(result.InstalledCases, "package version", "package versions")) } fmt.Fprintf(stdout, " tree sha256:%s\n", result.TreeSHA256) return nil @@ -1384,11 +1387,11 @@ func runVerifyDeb(ctx context.Context, args []string, stdout, stderr io.Writer) return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 verified %d repository %s\n", result.FileCount, plural(result.FileCount, "file", "files")) + reportDone(stdout, "verified %d repository %s", result.FileCount, plural(result.FileCount, "file", "files")) if *structuralOnly { - fmt.Fprintln(stdout, "✉️ structural verification passed") + reportDetail(stdout, "structural verification passed") } else { - fmt.Fprintf(stdout, "✉️ apt installed %d %s from the staged repository\n", result.InstalledCases, plural(result.InstalledCases, "package version", "package versions")) + reportDetail(stdout, "apt installed %d %s from the staged repository", result.InstalledCases, plural(result.InstalledCases, "package version", "package versions")) } fmt.Fprintf(stdout, " tree sha256:%s\n", result.TreeSHA256) return nil @@ -1411,11 +1414,11 @@ func runVerifyHelm(ctx context.Context, args []string, stdout, stderr io.Writer) return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 verified %d repository %s\n", result.FileCount, plural(result.FileCount, "file", "files")) + reportDone(stdout, "verified %d repository %s", result.FileCount, plural(result.FileCount, "file", "files")) if *structuralOnly { - fmt.Fprintln(stdout, "✉️ structural verification passed") + reportDetail(stdout, "structural verification passed") } else { - fmt.Fprintf(stdout, "✉️ Helm pulled, linted, and rendered %d %s\n", result.InstalledCases, plural(result.InstalledCases, "chart version", "chart versions")) + reportDetail(stdout, "Helm pulled, linted, and rendered %d %s", result.InstalledCases, plural(result.InstalledCases, "chart version", "chart versions")) } fmt.Fprintf(stdout, " tree sha256:%s\n", result.TreeSHA256) return nil @@ -1438,8 +1441,8 @@ func runVerifyRaw(args []string, stdout, stderr io.Writer) error { return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 verified %d repository %s\n", result.FileCount, plural(result.FileCount, "file", "files")) - fmt.Fprintf(stdout, "✉️ listing and checksums cover %d package %s\n", + reportDone(stdout, "verified %d repository %s", result.FileCount, plural(result.FileCount, "file", "files")) + reportDetail(stdout, "listing and checksums cover %d package %s", result.InstalledCases, plural(result.InstalledCases, "version", "versions")) fmt.Fprintf(stdout, " tree sha256:%s\n", result.TreeSHA256) return nil @@ -1486,11 +1489,11 @@ func runVerifyRPM(ctx context.Context, args []string, stdout, stderr io.Writer) return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 verified %d repository %s\n", result.FileCount, plural(result.FileCount, "file", "files")) + reportDone(stdout, "verified %d repository %s", result.FileCount, plural(result.FileCount, "file", "files")) if *structuralOnly { - fmt.Fprintln(stdout, "✉️ structural verification passed") + reportDetail(stdout, "structural verification passed") } else { - fmt.Fprintf(stdout, "✉️ dnf installed %d %s\n", result.InstalledCases, plural(result.InstalledCases, "package", "packages")) + reportDetail(stdout, "dnf installed %d %s", result.InstalledCases, plural(result.InstalledCases, "package", "packages")) } fmt.Fprintf(stdout, " tree sha256:%s\n", result.TreeSHA256) return nil @@ -1517,11 +1520,11 @@ func runVerifyAPK(ctx context.Context, args []string, stdout, stderr io.Writer) return err } printBrand(stdout) - fmt.Fprintf(stdout, "📦 verified %d repository %s\n", result.FileCount, plural(result.FileCount, "file", "files")) + reportDone(stdout, "verified %d repository %s", result.FileCount, plural(result.FileCount, "file", "files")) if *structuralOnly { - fmt.Fprintln(stdout, "✉️ structural verification passed") + reportDetail(stdout, "structural verification passed") } else { - fmt.Fprintf(stdout, "✉️ apk installed %d %s\n", result.InstalledCases, plural(result.InstalledCases, "package", "packages")) + reportDetail(stdout, "apk installed %d %s", result.InstalledCases, plural(result.InstalledCases, "package", "packages")) } fmt.Fprintf(stdout, " tree sha256:%s\n", result.TreeSHA256) return nil @@ -1568,8 +1571,8 @@ func runServe(ctx context.Context, args []string, stdout, stderr io.Writer) erro }() printBrand(stdout) - fmt.Fprintf(stdout, "📦 checked %s repository with %d %s\n", info.Format, info.FileCount, plural(info.FileCount, "file", "files")) - fmt.Fprintf(stdout, "✉️ serving %s at http://%s\n", absolute, listener.Addr()) + reportDone(stdout, "checked %s repository with %d %s", info.Format, info.FileCount, plural(info.FileCount, "file", "files")) + reportDetail(stdout, "serving %s at http://%s", absolute, listener.Addr()) err = server.Serve(listener) close(stopped) if errors.Is(err, http.ErrServerClosed) { @@ -1687,6 +1690,30 @@ func optionalTime(value string) (time.Time, error) { return parsed, nil } +// Output has three prefixes and one rule. +// +// 📦 is something that happened — a file written, a change applied, a +// repository configured. ✉️ is a detail about it: a path, a digest, a +// description of what was read. ⚠️ is something the reader has to act on. 🐌 is +// the brand line and appears once. +// +// They went through these three functions rather than being written inline at +// eighty-nine call sites so the rule is somewhere rather than nowhere, and so +// the alignment is decided once: ✉️ is an emoji-presentation sequence that most +// terminals render two cells wide and some render one, which is why it carries +// a different amount of padding from the others. +func reportDone(output io.Writer, format string, arguments ...any) { + fmt.Fprintf(output, "📦 "+format+"\n", arguments...) +} + +func reportDetail(output io.Writer, format string, arguments ...any) { + fmt.Fprintf(output, "✉️ "+format+"\n", arguments...) +} + +func reportWarning(output io.Writer, format string, arguments ...any) { + fmt.Fprintf(output, "⚠️ "+format+"\n", arguments...) +} + func printBrand(output io.Writer) { fmt.Fprintln(output, "🐌 snailmail") } @@ -1901,10 +1928,10 @@ func runCollect(ctx context.Context, args []string, stdout, stderr io.Writer) er printBrand(stdout) for _, reported := range result.Repositories { if !reported.Collectable { - fmt.Fprintf(stdout, "✉️ %s: %s\n", reported.Name, reported.Note) + reportDetail(stdout, "%s: %s", reported.Name, reported.Note) continue } - fmt.Fprintf(stdout, "✉️ %s: %d %s (%s), %d %s kept\n", + reportDetail(stdout, "%s: %d %s (%s), %d %s kept", reported.Name, reported.Removed, plural(reported.Removed, "object", "objects"), humanBytes(reported.RemovedBytes), reported.KeptRevisions, plural(reported.KeptRevisions, "revision", "revisions")) @@ -1913,10 +1940,10 @@ func runCollect(ctx context.Context, args []string, stdout, stderr io.Writer) er if result.Applied { verb = "removed" } - fmt.Fprintf(stdout, "📦 %s %d %s, %s\n", verb, result.Removed, + reportDone(stdout, "%s %d %s, %s", verb, result.Removed, plural(result.Removed, "object", "objects"), humanBytes(result.RemovedBytes)) if !result.Applied && result.Removed != 0 { - fmt.Fprintln(stdout, "✉️ nothing was removed; pass --yes to collect") + reportDetail(stdout, "nothing was removed; pass --yes to collect") } return nil } @@ -1978,18 +2005,18 @@ func runImport(ctx context.Context, args []string, stdout, stderr io.Writer) err if result.DryRun { verb = "would import" } - fmt.Fprintf(stdout, "📦 %s %d of %d %s from %s\n", verb, len(result.Imported), result.Listed, + reportDone(stdout, "%s %d of %d %s from %s", verb, len(result.Imported), result.Listed, plural(result.Listed, "artifact", "artifacts"), result.IndexURL) for _, imported := range result.Imported { - fmt.Fprintf(stdout, "✉️ %s@%s sha256:%s\n", imported.Package, imported.Version, imported.SHA256) + reportDetail(stdout, "%s@%s sha256:%s", imported.Package, imported.Version, imported.SHA256) } // Skipped artifacts are named rather than counted. An import that took nine of // ten and said only "9 imported" would be discovered later, by a client. for _, skipped := range result.Skipped { - fmt.Fprintf(stdout, "⚠️ skipped %s: %s\n", skipped.Filename, skipped.Reason) + reportWarning(stdout, "skipped %s: %s", skipped.Filename, skipped.Reason) } if result.DryRun && len(result.Imported) != 0 { - fmt.Fprintln(stdout, "✉️ nothing was recorded; drop --dry-run to import") + reportDetail(stdout, "nothing was recorded; drop --dry-run to import") } else if len(result.Imported) != 0 { suggestNext(stdout, flags, "git commit -am \"import "+positional[0]+"\" && snailmail plan", "review what was imported as a diff, then publish it") From 95520419d1b0ac5c4846d88d8d51059005139ebe Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 16:02:48 +0200 Subject: [PATCH 21/26] Repin the build image past the patched standard library --- Dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index fd3f489..66be054 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1@sha256:87999aa3d42bdc6bea60565083ee17e86d1f3339802f543c0d03998580f9cb89 -FROM golang:1.25-alpine@sha256:56961d79ea8129efddcc0b8643fd8a5416b4e6228cfd477e3fd61deb2672c587 AS build +FROM golang:1.25-alpine@sha256:1ae0735f00daffa3aaf1363a5184c0d2dc55c78e3db4ec70241cdac97bf84b59 AS build WORKDIR /src COPY go.mod go.sum ./ RUN go mod download @@ -9,7 +9,14 @@ RUN CGO_ENABLED=0 go test -c -o /out/openpgp.test ./signer/openpgp # Scanned here rather than on the runner: `go run tool@version` builds the tool # with the ambient toolchain, so a runner older than go.mod's requirement cannot -# load these packages at all. The build image is digest-pinned and current. +# load these packages at all. +# +# The build image is digest-pinned, which means it has to be repinned when the +# standard library is patched — a pin freezes the toolchain along with +# everything else, so this job failing is the pin doing its job rather than a +# surprise. The digest above is what `docker pull golang:1.25-alpine` resolves +# to; take the new one from there and check `go version` moved past whatever +# govulncheck named. FROM build AS vulncheck RUN go run golang.org/x/vuln/cmd/govulncheck@v1.1.4 ./... From d111b7da25501ae28ba5170d353f4b686e123b37 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 16:27:19 +0200 Subject: [PATCH 22/26] Decide what a valid endpoint is in one place --- adapters/blob/s3/aws.go | 23 ++------- adapters/blob/s3/aws_test.go | 17 +++---- adapters/host/s3/s3.go | 30 ++---------- internal/app/deb_endpoint.go | 3 +- internal/app/verify.go | 7 +-- internal/endpoint/endpoint.go | 77 ++++++++++++++++++++++++++++++ internal/endpoint/endpoint_test.go | 66 +++++++++++++++++++++++++ internal/state/store.go | 51 ++++---------------- 8 files changed, 174 insertions(+), 100 deletions(-) create mode 100644 internal/endpoint/endpoint.go create mode 100644 internal/endpoint/endpoint_test.go diff --git a/adapters/blob/s3/aws.go b/adapters/blob/s3/aws.go index 31f951d..5db7cb0 100644 --- a/adapters/blob/s3/aws.go +++ b/adapters/blob/s3/aws.go @@ -9,12 +9,12 @@ import ( "errors" "fmt" "io" - "net/url" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/shellcell/snailmail/blob" + endpointcheck "github.com/shellcell/snailmail/internal/endpoint" "github.com/shellcell/snailmail/internal/awss3" "github.com/shellcell/snailmail/internal/hexdigest" @@ -29,8 +29,10 @@ func NewAWS(ctx context.Context, configuration blob.Configuration) (*Store, erro if configuration.Type != "s3" || configuration.Bucket == "" { return nil, errors.New("S3 blob bucket is required") } - if err := validateAWSEndpoint(configuration.Endpoint); err != nil { - return nil, err + if configuration.Endpoint != "" { + if err := endpointcheck.RequireSecure(configuration.Endpoint); err != nil { + return nil, fmt.Errorf("S3 blob endpoint: %w", err) + } } client, err := awss3.NewClient(ctx, awss3.Config{ Bucket: configuration.Bucket, Region: configuration.Region, @@ -42,21 +44,6 @@ func NewAWS(ctx context.Context, configuration blob.Configuration) (*Store, erro return New(&AWSClient{client: client, bucket: configuration.Bucket}, configuration) } -func validateAWSEndpoint(value string) error { - if value == "" { - return nil - } - parsed, err := url.Parse(value) - if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" || (parsed.Path != "" && parsed.Path != "/") { - return errors.New("invalid S3 blob endpoint") - } - loopback := parsed.Hostname() == "localhost" || parsed.Hostname() == "127.0.0.1" || parsed.Hostname() == "::1" - if parsed.Scheme != "https" && !loopback { - return errors.New("S3 blob endpoint must use HTTPS") - } - return nil -} - func (client *AWSClient) Head(ctx context.Context, key string) (ObjectInfo, error) { result, err := client.client.HeadObject(ctx, &s3.HeadObjectInput{Bucket: &client.bucket, Key: &key, ChecksumMode: types.ChecksumModeEnabled}) if err != nil { diff --git a/adapters/blob/s3/aws_test.go b/adapters/blob/s3/aws_test.go index e6dbcdb..12fa588 100644 --- a/adapters/blob/s3/aws_test.go +++ b/adapters/blob/s3/aws_test.go @@ -42,13 +42,14 @@ func TestNormalizeAWSErrorTreatsABarePreconditionStatusAsPrecondition(t *testing } } -func TestValidateAWSEndpointRequiresHTTPSOutsideLoopback(t *testing.T) { - if err := validateAWSEndpoint("http://objects.example"); err == nil { - t.Fatal("plaintext remote endpoint was accepted") - } - for _, endpoint := range []string{"https://objects.example", "http://127.0.0.1:9000", "http://localhost:9000"} { - if err := validateAWSEndpoint(endpoint); err != nil { - t.Fatalf("endpoint %q rejected: %v", endpoint, err) - } +// The endpoint rule lives in internal/endpoint and is tested there. This is the +// blob store asking for it: a plaintext endpoint that is not this machine sends +// artifacts and the digests authenticating them somewhere anyone on the path can +// rewrite them. +func TestBlobStoreRequiresHTTPSOutsideLoopback(t *testing.T) { + if _, err := NewAWS(t.Context(), blob.Configuration{ + Type: "s3", Bucket: "packages", Endpoint: "http://objects.example", + }); err == nil { + t.Fatal("a plaintext remote endpoint was accepted") } } diff --git a/adapters/host/s3/s3.go b/adapters/host/s3/s3.go index b1eb0c5..c49efdb 100644 --- a/adapters/host/s3/s3.go +++ b/adapters/host/s3/s3.go @@ -24,6 +24,7 @@ import ( "github.com/shellcell/snailmail/formats/pypi" "github.com/shellcell/snailmail/host" "github.com/shellcell/snailmail/internal/buildgraph" + endpointcheck "github.com/shellcell/snailmail/internal/endpoint" "github.com/shellcell/snailmail/internal/listing" "github.com/shellcell/snailmail/internal/hexdigest" @@ -1135,20 +1136,12 @@ func validateRepository(repository host.Repository) error { if prefix != repository.S3.Prefix || strings.ContainsRune(prefix, '\\') || hasControl(prefix) || (prefix != "" && (path.Clean(prefix) != prefix || strings.HasPrefix(prefix, "../"))) { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("S3 prefix is invalid")} } - if err := validateHTTPURL(repository.CanonicalEndpoint); err != nil { + if err := endpointcheck.RequireSecure(repository.CanonicalEndpoint); err != nil { return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: fmt.Errorf("canonical endpoint: %w", err)} } - parsed, _ := url.Parse(repository.CanonicalEndpoint) - if parsed.Scheme != "https" && !(parsed.Scheme == "http" && isLoopbackHost(parsed.Hostname())) { - return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("client endpoint must use HTTPS")} - } if repository.S3.Endpoint != "" { - if err := validateHTTPURL(repository.S3.Endpoint); err != nil { - return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: fmt.Errorf("S3 endpoint: %w", err)} - } - parsed, _ := url.Parse(repository.S3.Endpoint) - if parsed.Scheme != "https" && !(parsed.Scheme == "http" && isLoopbackHost(parsed.Hostname())) { - return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: errors.New("S3 API endpoint must use HTTPS")} + if err := endpointcheck.RequireSecure(repository.S3.Endpoint); err != nil { + return &host.Error{Kind: host.ErrorInvalidConfiguration, Operation: "configure S3 host", Err: fmt.Errorf("S3 API endpoint: %w", err)} } } return nil @@ -1164,21 +1157,6 @@ func (adapter *Adapter) validateRepository(repository host.Repository) error { return nil } -func validateHTTPURL(value string) error { - if hasControl(value) { - return errors.New("must not contain control characters") - } - parsed, err := url.Parse(value) - if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { - return errors.New("must be an HTTP(S) URL without credentials, query, or fragment") - } - return nil -} - -func isLoopbackHost(value string) bool { - return value == "localhost" || value == "127.0.0.1" || value == "::1" -} - func validateFile(file host.File) error { if file.Path == "" || file.Path == "." || len(file.Path) > 1024 || path.IsAbs(file.Path) || path.Clean(file.Path) != file.Path || strings.HasPrefix(file.Path, "../") || strings.ContainsRune(file.Path, '\\') || file.Size < 0 || !hexdigest.ValidSHA256(file.SHA256) { diff --git a/internal/app/deb_endpoint.go b/internal/app/deb_endpoint.go index 74941d1..1b7d9a4 100644 --- a/internal/app/deb_endpoint.go +++ b/internal/app/deb_endpoint.go @@ -20,6 +20,7 @@ import ( "github.com/shellcell/snailmail/host" "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/domain" + endpointcheck "github.com/shellcell/snailmail/internal/endpoint" ) // VerifyDebClientEndpointAccess installs from a Debian repository the host is @@ -93,7 +94,7 @@ func VerifyDebClientEndpointAccess(ctx context.Context, root string, access host // A loopback endpoint is the host's loopback, not the container's, so it is // only reachable with host networking. A public endpoint uses the runner's // default network and stays isolated from the host. - hostNetwork := isLoopbackHost(endpoint.Hostname()) + hostNetwork := endpointcheck.IsLoopback(endpoint.Hostname()) verificationCases = scope.selection(verificationCases, debCompare) if err := verifyCases(ctx, verificationCases, func(caseCtx context.Context, verification domain.VerificationCase) error { return verifyDebEndpointCase(caseCtx, runner, image, access.Endpoint, trust, hostNetwork, workspaceBytes, manifest, verification) diff --git a/internal/app/verify.go b/internal/app/verify.go index 799e759..86bada4 100644 --- a/internal/app/verify.go +++ b/internal/app/verify.go @@ -31,6 +31,7 @@ import ( "github.com/shellcell/snailmail/host" "github.com/shellcell/snailmail/internal/buildgraph" "github.com/shellcell/snailmail/internal/domain" + endpointcheck "github.com/shellcell/snailmail/internal/endpoint" ) const ( @@ -287,7 +288,7 @@ func validateClientEndpoint(endpoint string) (*url.URL, error) { parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { return nil, errors.New("verification endpoint must be an HTTP(S) URL") } - if parsed.Scheme != "https" && !isLoopbackHost(parsed.Hostname()) { + if parsed.Scheme != "https" && !endpointcheck.IsLoopback(parsed.Hostname()) { return nil, errors.New("verification endpoint must use HTTPS") } return parsed, nil @@ -409,10 +410,6 @@ func validNetrcValue(value string) bool { return true } -func isLoopbackHost(value string) bool { - return value == "localhost" || value == "127.0.0.1" || value == "::1" -} - func redactCredential(value, username, password string) string { // With no credential every derived form is a constant — base64(":") in // particular — so redacting them would rewrite unrelated client output. diff --git a/internal/endpoint/endpoint.go b/internal/endpoint/endpoint.go new file mode 100644 index 0000000..5c1a8b6 --- /dev/null +++ b/internal/endpoint/endpoint.go @@ -0,0 +1,77 @@ +// Package endpoint validates the URLs snailmail is configured to talk to. +// +// There were three copies of this, and they disagreed. One rejected any control +// character, one rejected only NUL, CR and LF. One rejected a URL with a path, +// two allowed it. Each was reached by a different route into the same +// configuration — the manifest validator, the S3 host adapter, the blob store — +// so which rule applied depended on which check happened to run first. +// +// The shape rule is the strict one, because a control character in a URL was +// never valid anywhere. The path rule is the permissive one, because an S3 +// endpoint behind a gateway legitimately has a path prefix and the copy that +// refused it would have turned a working configuration into a refused one. +package endpoint + +import ( + "errors" + "net/url" + "strings" + "unicode" +) + +// Validate checks the shape of a URL: that it is HTTP or HTTPS, names a host, +// and carries nothing that has no business in a configured endpoint. +// +// Credentials, a query and a fragment are all refused. An endpoint is a base +// that paths are appended to, and each of those either travels somewhere it +// should not — a URL in a published listing, a log line — or silently stops +// meaning what it says once something is appended. +func Validate(value string) error { + if hasControl(value) { + return errors.New("must not contain control characters") + } + parsed, err := url.Parse(value) + if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || + parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return errors.New("must be an HTTP(S) URL without credentials, query, or fragment") + } + return nil +} + +// RequireSecure checks the shape and requires HTTPS. +// +// Loopback is the exception, and only loopback: a test server or a local MinIO +// has no certificate anybody would accept, and refusing plaintext there would +// make the tool untestable without inventing one. Everywhere else, a plaintext +// endpoint means artifacts and the digests that authenticate them travel where +// anyone on the path can rewrite them. +func RequireSecure(value string) error { + if err := Validate(value); err != nil { + return err + } + parsed, err := url.Parse(value) + if err != nil { + return err + } + if parsed.Scheme == "https" { + return nil + } + if IsLoopback(parsed.Hostname()) { + return nil + } + return errors.New("must use HTTPS") +} + +// IsLoopback reports whether a hostname names this machine. +func IsLoopback(hostname string) bool { + return hostname == "localhost" || hostname == "127.0.0.1" || hostname == "::1" +} + +// hasControl reports whether a string carries any control character. +// +// The strict form. One copy of this checked only NUL, CR and LF, which lets a +// tab or an escape through into a URL that is later written into a shell +// snippet on a published page. +func hasControl(value string) bool { + return strings.IndexFunc(value, unicode.IsControl) >= 0 +} diff --git a/internal/endpoint/endpoint_test.go b/internal/endpoint/endpoint_test.go new file mode 100644 index 0000000..040684a --- /dev/null +++ b/internal/endpoint/endpoint_test.go @@ -0,0 +1,66 @@ +package endpoint + +import "testing" + +func TestValidate(t *testing.T) { + for _, testcase := range []struct { + value string + wantError bool + }{ + {"https://packages.example", false}, + {"https://packages.example/repo", false}, + // A path prefix is what an S3 endpoint behind a gateway looks like. One of + // the three copies of this rule refused it and the other two did not, so + // whether it worked depended on which check ran first. + {"https://gateway.example/s3", false}, + {"http://localhost:9000", false}, + + {"", true}, + {"packages.example", true}, + {"ftp://packages.example", true}, + {"https://", true}, + // Credentials, a query and a fragment all either travel somewhere they + // should not or stop meaning what they say once a path is appended. + {"https://user:pass@packages.example", true}, + {"https://packages.example?token=1", true}, + {"https://packages.example#fragment", true}, + // The strict control-character rule. One copy checked only NUL, CR and LF, + // which lets a tab through into a URL that is later written into a shell + // snippet on a published page. + {"https://packages.example/\trepo", true}, + {"https://packages.example/\x00", true}, + {"https://packages.example/\x1b[0m", true}, + } { + err := Validate(testcase.value) + if testcase.wantError && err == nil { + t.Errorf("Validate(%q) was accepted", testcase.value) + } + if !testcase.wantError && err != nil { + t.Errorf("Validate(%q) was refused: %v", testcase.value, err) + } + } +} + +// Plaintext is refused everywhere except this machine, where a test server or a +// local MinIO has no certificate anyone would accept. +func TestRequireSecure(t *testing.T) { + for _, secure := range []string{ + "https://packages.example", + "http://localhost:9000", "http://127.0.0.1:9000", "http://[::1]:9000", + } { + if err := RequireSecure(secure); err != nil { + t.Errorf("RequireSecure(%q) was refused: %v", secure, err) + } + } + for _, insecure := range []string{ + "http://packages.example", + "http://192.168.1.10:9000", + // Not loopback: a name that merely contains one. + "http://localhost.example.com", + "http://notlocalhost", + } { + if err := RequireSecure(insecure); err == nil { + t.Errorf("RequireSecure(%q) was accepted", insecure) + } + } +} diff --git a/internal/state/store.go b/internal/state/store.go index f8c34dc..d359d41 100644 --- a/internal/state/store.go +++ b/internal/state/store.go @@ -25,6 +25,7 @@ import ( "github.com/shellcell/snailmail/blob" "github.com/shellcell/snailmail/formats" "github.com/shellcell/snailmail/host" + endpointcheck "github.com/shellcell/snailmail/internal/endpoint" "github.com/shellcell/snailmail/source" "github.com/shellcell/snailmail/forge" @@ -495,7 +496,7 @@ func validateRepositoryHost(name string, repository Repository) error { // and clients get no address to point at. It is documentation only: // nothing publishes to it, so it is validated and otherwise unused. if repository.Host.CanonicalEndpoint != "" { - if err := validateHTTPURL(repository.Host.CanonicalEndpoint); err != nil { + if err := endpointcheck.Validate(repository.Host.CanonicalEndpoint); err != nil { return fmt.Errorf("repository %q base URL: %w", name, err) } } @@ -524,7 +525,7 @@ func validateRepositoryHost(name string, repository Repository) error { return fmt.Errorf("repository %q: rsync hosting serves whatever the web server serves, so it supports public repositories only", name) } if repository.Host.CanonicalEndpoint != "" { - if err := validateHTTPURL(repository.Host.CanonicalEndpoint); err != nil { + if err := endpointcheck.Validate(repository.Host.CanonicalEndpoint); err != nil { return fmt.Errorf("repository %q base URL: %w", name, err) } } @@ -545,20 +546,12 @@ func validateRepositoryHost(name string, repository Repository) error { if prefix != repository.Host.Prefix || (prefix != "" && (path.Clean(prefix) != prefix || strings.HasPrefix(prefix, "../"))) { return fmt.Errorf("repository %q has invalid S3 prefix %q", name, repository.Host.Prefix) } - if err := validateHTTPURL(repository.Host.CanonicalEndpoint); err != nil { - return fmt.Errorf("repository %q canonical endpoint: %w", name, err) - } - parsed, _ := url.Parse(repository.Host.CanonicalEndpoint) - if parsed.Scheme != "https" && !(parsed.Scheme == "http" && isLoopbackHost(parsed.Hostname())) { - return fmt.Errorf("repository %q client endpoint must use HTTPS", name) + if err := endpointcheck.RequireSecure(repository.Host.CanonicalEndpoint); err != nil { + return fmt.Errorf("repository %q client endpoint: %w", name, err) } if repository.Host.Endpoint != "" { - if err := validateHTTPURL(repository.Host.Endpoint); err != nil { - return fmt.Errorf("repository %q S3 endpoint: %w", name, err) - } - parsed, _ := url.Parse(repository.Host.Endpoint) - if parsed.Scheme != "https" && !(parsed.Scheme == "http" && isLoopbackHost(parsed.Hostname())) { - return fmt.Errorf("repository %q S3 API endpoint must use HTTPS", name) + if err := endpointcheck.RequireSecure(repository.Host.Endpoint); err != nil { + return fmt.Errorf("repository %q S3 API endpoint: %w", name, err) } } case "github-pages": @@ -600,13 +593,9 @@ func validateRepositoryHost(name string, repository Repository) error { } for _, configured := range endpoints { label, endpoint := configured.label, configured.endpoint - if err := validateHTTPURL(endpoint); err != nil { + if err := endpointcheck.RequireSecure(endpoint); err != nil { return fmt.Errorf("repository %q %s endpoint: %w", name, label, err) } - parsed, _ := url.Parse(endpoint) - if parsed.Scheme != "https" && !(parsed.Scheme == "http" && isLoopbackHost(parsed.Hostname())) { - return fmt.Errorf("repository %q %s endpoint must use HTTPS", name, label) - } } default: return fmt.Errorf("repository %q has unsupported host type %q; supported: %s", @@ -629,21 +618,6 @@ func requireHostServesFormat(name string, repository Repository) error { name, repository.Host.Type, repository.Format, strings.Join(supported, ", ")) } -func validateHTTPURL(value string) error { - if strings.ContainsAny(value, "\x00\r\n") { - return errors.New("must not contain control characters") - } - parsed, err := url.Parse(value) - if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" || parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { - return errors.New("must be an HTTP(S) URL without credentials, query, or fragment") - } - return nil -} - -func isLoopbackHost(value string) bool { - return value == "localhost" || value == "127.0.0.1" || value == "::1" -} - // validateForgeIdentity checks the provider, its repository reference and its // host together, because they are only meaningful as a set: a reference is valid // for a provider rather than in general, and a provider that exists only @@ -968,16 +942,9 @@ func ValidateBlobStore(configuration BlobStoreConfig) error { return errors.New("S3 blob store prefix is invalid") } if configuration.Endpoint != "" { - if err := validateHTTPURL(configuration.Endpoint); err != nil { + if err := endpointcheck.RequireSecure(configuration.Endpoint); err != nil { return fmt.Errorf("S3 blob endpoint: %w", err) } - parsed, _ := url.Parse(configuration.Endpoint) - if parsed.Scheme != "https" && !(parsed.Scheme == "http" && isLoopbackHost(parsed.Hostname())) { - return errors.New("S3 blob endpoint must use HTTPS") - } - if parsed.Path != "" && parsed.Path != "/" { - return errors.New("S3 blob endpoint must not contain a path") - } } default: return fmt.Errorf("unsupported blob store type %q", configuration.Type) From 2c44d611efa16e44a185a1f5bbd77f19157da214 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 16:27:19 +0200 Subject: [PATCH 23/26] Put the comments back on the methods they describe --- formats/format.go | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/formats/format.go b/formats/format.go index 6c4fe26..4497118 100644 --- a/formats/format.go +++ b/formats/format.go @@ -270,11 +270,9 @@ func (pypiFormat) Inspect(filename string, reader io.ReaderAt, size int64, suppl func (pypiFormat) ArtifactCoordinate(artifact Artifact) string { return artifact.Filename } func (pypiFormat) SupportsDistros() bool { return false } -// PyPI dropped GPG signatures in 2023, so there is no repository signature to -// carry. -// PyPI dropped repository signing in 2023. func (pypiFormat) RequiresLegacyDigests() bool { return false } +// PyPI dropped repository signatures in 2023, so there is no scheme to carry. func (pypiFormat) SigningAlgorithm() string { return "" } func (pypiFormat) ImplementsSigning() bool { return false } @@ -311,11 +309,10 @@ func (debFormat) Inspect(filename string, reader io.ReaderAt, size int64, suppli func (debFormat) ArtifactCoordinate(artifact Artifact) string { return artifact.Architecture } func (debFormat) SupportsDistros() bool { return true } -// apt verifies OpenPGP over the Release document. -// apt reads MD5sum and SHA1 from a Packages file, so a Debian repository is -// the one place these still have to be computed. +// The one format that needs them: see the interface doc. func (debFormat) RequiresLegacyDigests() bool { return true } +// apt verifies OpenPGP over the Release document. func (debFormat) SigningAlgorithm() string { return signer.AlgorithmOpenPGPRSA4096 } func (debFormat) ImplementsSigning() bool { return true } @@ -428,9 +425,6 @@ func (rawFormat) Inspect(filename string, reader io.ReaderAt, size int64, suppli func (rawFormat) ArtifactCoordinate(artifact Artifact) string { return artifact.Filename } func (rawFormat) SupportsDistros() bool { return false } -// Detached signatures over the listing are the documented raw scheme; they are -// not implemented yet. -// Loose files are not an ecosystem and define no signing. func (rawFormat) RequiresLegacyDigests() bool { return false } func (rawFormat) SigningAlgorithm() string { return "" } @@ -484,12 +478,11 @@ func (rpmFormat) ArtifactCoordinate(artifact Artifact) string { return artifact. // coordinate inside it, so releases do not carry one. func (rpmFormat) SupportsDistros() bool { return false } +func (rpmFormat) RequiresLegacyDigests() bool { return false } + // Detached OpenPGP over repomd.xml is what repo_gpgcheck verifies, and it is // produced. Per-package signing is not: that signature lives in the package // header and is made by whoever built the package, not by the repository. -// repo_gpgcheck verifies OpenPGP over repomd.xml. -func (rpmFormat) RequiresLegacyDigests() bool { return false } - func (rpmFormat) SigningAlgorithm() string { return signer.AlgorithmOpenPGPRSA4096 } func (rpmFormat) ImplementsSigning() bool { return true } @@ -542,11 +535,11 @@ func (apkFormat) ArtifactCoordinate(artifact Artifact) string { return artifact. // directory is part of that URL rather than a coordinate inside the index. func (apkFormat) SupportsDistros() bool { return false } -// apk signs an index by prepending a signature stream to it, with the signing -// key's filename identifying which key to check. That is not produced yet. -// apk verifies a bare RSA signature against a key held by filename. func (apkFormat) RequiresLegacyDigests() bool { return false } +// apk signs an index by prepending a signature stream to it, with the signing +// key's filename identifying which key to check — a bare RSA signature rather +// than anything with OpenPGP structure around it. func (apkFormat) SigningAlgorithm() string { return signer.AlgorithmAPKRSA4096 } func (apkFormat) ImplementsSigning() bool { return true } From 590e6b8c274cf0b8437955e651c26315e945923a Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 16:27:45 +0200 Subject: [PATCH 24/26] Run staticcheck, and fix what it found --- .github/workflows/ci.yml | 1 + Makefile | 16 +++++++++++++++- adapters/host/rsync/rsync_test.go | 9 --------- engine/workspace.go | 6 ++---- formats/deb/inspect.go | 4 ++-- formats/helm/inspect.go | 2 +- gate/gate.go | 4 ---- internal/knowledge/signing_test.go | 6 +++++- internal/state/git_layout_cache.go | 10 ---------- internal/state/lockshard.go | 12 ------------ internal/testutil/wheel.go | 2 +- staticcheck.conf | 10 ++++++++++ 12 files changed, 37 insertions(+), 45 deletions(-) create mode 100644 staticcheck.conf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eba4592..99fd43d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,6 +51,7 @@ jobs: test -e /proc/sys/fs/binfmt_misc/qemu-aarch64 - run: make fmt - run: make vet + - run: make lint - run: make test-race # The S3 adapters are build-tagged out for smaller binaries; that # configuration has to keep compiling and passing. diff --git a/Makefile b/Makefile index 0dab0e9..d1a9bf4 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,7 @@ help: ## Show the available targets # ---------------------------------------------------------------- checks .PHONY: check -check: fmt vet test ## Everything CI checks, in the order that fails fastest +check: fmt vet lint test ## Everything CI checks, in the order that fails fastest .PHONY: fmt fmt: ## Fail if anything is unformatted @@ -63,6 +63,20 @@ vet: ## Vet the default build and the one with S3 compiled out go vet ./... go vet -tags nos3 ./... +.PHONY: lint +# staticcheck finds what vet does not: unused code, impossible conditions, +# results ignored where they matter. An error taxonomy sat here fully populated +# and entirely unread for long enough to be worth a tool that says so. +# +# Pinned like every other tool here, so a lint that passes today passes +# tomorrow and an upgrade is a reviewable change rather than a surprise. The pin +# has to be recent enough to read the toolchain go.mod asks for: an older +# staticcheck cannot decode a newer Go export format and fails on the standard +# library rather than on this code. +STATICCHECK ?= go run honnef.co/go/tools/cmd/staticcheck@2026.2.1 +lint: ## Run staticcheck + $(STATICCHECK) ./... + .PHONY: test test: ## Run the suite go test -count=1 ./... diff --git a/adapters/host/rsync/rsync_test.go b/adapters/host/rsync/rsync_test.go index b66c282..715c011 100644 --- a/adapters/host/rsync/rsync_test.go +++ b/adapters/host/rsync/rsync_test.go @@ -57,15 +57,6 @@ func (runner *localRunner) Send(_ context.Context, localDirectory, remotePath st return os.CopyFS(remotePath, os.DirFS(localDirectory)) } -func (runner *localRunner) ran(name string) bool { - for _, argv := range runner.commands { - if argv[0] == name { - return true - } - } - return false -} - // verifiedTree builds a repository the adapter will accept, since Stage verifies // before sending anything to the far side. func verifiedTree(t *testing.T, version string) (string, string) { diff --git a/engine/workspace.go b/engine/workspace.go index 55c1140..b60f6c9 100644 --- a/engine/workspace.go +++ b/engine/workspace.go @@ -2132,9 +2132,7 @@ func (preparation *applyPreparation) commitPublicationLedgers(prepared []applyRe if err != nil { return "", errors.New("committed publication ledgers do not match the plan") } - if len(ledgerRepositories) == 0 { - ledgerRevision = plan.Payload.GitRevision - } else { + if len(ledgerRepositories) != 0 { ledgerRevision, err = state.PlanLedgerRevision(root, plan.PlanID) if err != nil { return "", err @@ -2228,7 +2226,7 @@ func (preparation *applyPreparation) publishAndRecord(prepared []applyRepository return ApplyWorkspaceResult{}, err } result := ApplyWorkspaceResult{PlanID: plan.PlanID} - deployments := make([]state.DeploymentRecord, 0, len(prepared)) + var deployments []state.DeploymentRecord execution := &applyExecution{ ctx: ctx, root: root, request: request, plan: plan, applyGitRevision: applyGitRevision, authorize: preparation.authorize, result: result, diff --git a/formats/deb/inspect.go b/formats/deb/inspect.go index 344669c..84f57ed 100644 --- a/formats/deb/inspect.go +++ b/formats/deb/inspect.go @@ -210,8 +210,8 @@ func validateDataArchive(name string, raw io.Reader, maximumExpanded int64) (int return 0, fmt.Errorf("data archive contains unsafe path %q", header.Name) } switch header.Typeflag { - case tar.TypeReg, tar.TypeRegA, tar.TypeDir, tar.TypeSymlink, tar.TypeLink: - if header.Typeflag == tar.TypeReg || header.Typeflag == tar.TypeRegA { + case tar.TypeReg, tar.TypeDir, tar.TypeSymlink, tar.TypeLink: + if header.Typeflag == tar.TypeReg { if header.Size < 0 || header.Size > maximumExpanded-installedSize { return 0, errors.New("data archive expanded size exceeds limit") } diff --git a/formats/helm/inspect.go b/formats/helm/inspect.go index 1b90af1..f5cbe92 100644 --- a/formats/helm/inspect.go +++ b/formats/helm/inspect.go @@ -198,7 +198,7 @@ func readChartYAML(filename string, reader io.ReaderAt, size, maximumExpanded in } switch header.Typeflag { case tar.TypeDir: - case tar.TypeReg, tar.TypeRegA: + case tar.TypeReg: if header.Size < 0 || header.Size > maximumExpanded-expandedSize { return nil, "", 0, fmt.Errorf("inspect %q: expanded chart exceeds limit", filename) } diff --git a/gate/gate.go b/gate/gate.go index 140b0c8..f53428e 100644 --- a/gate/gate.go +++ b/gate/gate.go @@ -264,7 +264,3 @@ func writePrivateFile(filename string, content []byte) error { } return os.Rename(temporaryName, filename) } - -func decodeJSON(content []byte, destination any) error { - return jsonstrict.DecodeAllowUnknown(content, destination, 1<<20) -} diff --git a/internal/knowledge/signing_test.go b/internal/knowledge/signing_test.go index 2759a64..29bf062 100644 --- a/internal/knowledge/signing_test.go +++ b/internal/knowledge/signing_test.go @@ -6,7 +6,11 @@ func TestSigningCompatibilityTable(t *testing.T) { if !Compatible("deb", "openpgp-rsa4096") || !Compatible("rpm", "openpgp-rsa4096") || Compatible("rpm", "openpgp-ed25519") || Compatible("nix", "openpgp-rsa4096") || Compatible("pypi", "openpgp-rsa4096") { t.Fatal("signing compatibility table does not enforce format constraints") } - if len(SigningDigest()) != 64 || SigningDigest() != SigningDigest() { + // Twice, into two variables: the digest is pinned into every plan, so it has + // to be the same answer each time it is asked. Comparing the call with itself + // said the same thing but read like a typo. + first, second := SigningDigest(), SigningDigest() + if len(first) != 64 || first != second { t.Fatal("signing knowledge digest is not stable") } } diff --git a/internal/state/git_layout_cache.go b/internal/state/git_layout_cache.go index 137952c..7a9023b 100644 --- a/internal/state/git_layout_cache.go +++ b/internal/state/git_layout_cache.go @@ -51,13 +51,3 @@ func cachedGitLayout(root, question string, ask func() (string, error)) (string, gitLayout.errors[key] = err return answer, err } - -// forgetGitLayout drops what has been remembered. Tests create many -// repositories at the same paths, where a real workspace is one repository for -// the life of a command. -func forgetGitLayout() { - gitLayout.Lock() - defer gitLayout.Unlock() - gitLayout.answers = nil - gitLayout.errors = nil -} diff --git a/internal/state/lockshard.go b/internal/state/lockshard.go index 43c3876..70e830f 100644 --- a/internal/state/lockshard.go +++ b/internal/state/lockshard.go @@ -428,15 +428,3 @@ func looksSharded(content []byte) (lockRoot, bool) { } return root, root.SchemaVersion == LockShardSchema } - -func bytesEqual(left, right []byte) bool { - if len(left) != len(right) { - return false - } - for index := range left { - if left[index] != right[index] { - return false - } - } - return true -} diff --git a/internal/testutil/wheel.go b/internal/testutil/wheel.go index ec5a610..85a6907 100644 --- a/internal/testutil/wheel.go +++ b/internal/testutil/wheel.go @@ -49,7 +49,7 @@ func WheelWithDependencies(name, version, requiresPython string, requirements [] archive := zip.NewWriter(&buffer) for _, file := range files { header := &zip.FileHeader{Name: file.name, Method: zip.Store} - header.SetModTime(time.Date(1980, time.January, 1, 0, 0, 0, 0, time.UTC)) + header.Modified = time.Date(1980, time.January, 1, 0, 0, 0, 0, time.UTC) header.SetMode(0o644) entry, err := archive.CreateHeader(header) if err != nil { diff --git a/staticcheck.conf b/staticcheck.conf new file mode 100644 index 0000000..b2d2a3a --- /dev/null +++ b/staticcheck.conf @@ -0,0 +1,10 @@ +# ST1005 asks that error strings start lowercase. Go's own convention exempts a +# string beginning with a proper noun or an acronym, and staticcheck cannot tell +# one from a sentence — so it flagged 128 strings here, essentially all of them +# beginning with GitHub, Git, S3, PyPI, Debian, Alpine or Helm. Renaming those +# to satisfy the check would make the messages worse. +# +# Everything else stays on, including the SA checks that found two dead stores, +# four unused functions and a comparison of a call with itself the first time +# this ran. +checks = ["inherit", "-ST1005"] From b1f0abdba1ef5d21d7f079cc7194433bcabd5097 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 16:27:45 +0200 Subject: [PATCH 25/26] Bump dependencies and correct what the docs claim --- PLAN.md | 6 ++-- README.md | 7 ++-- engine/ci.go | 4 +-- engine/ci_gitlab.go | 6 ++-- go.mod | 44 +++++++++++------------ go.sum | 88 ++++++++++++++++++++++----------------------- 6 files changed, 79 insertions(+), 76 deletions(-) diff --git a/PLAN.md b/PLAN.md index e87ef12..6eb915b 100644 --- a/PLAN.md +++ b/PLAN.md @@ -1450,8 +1450,10 @@ The format-and-host coupling that this phase owed is closed for its first cases: GitHub Pages serves signed Debian, rpm, apk, raw and Helm alongside PyPI. The matrix itself is declared in `host/support.go` rather than inferred, so the gaps that remain are readable rather than discovered — and each undeclared pair -records why. Remaining: S3 beyond PyPI, additional key backends, `import`, and -the TUI. The next formats are nix cache, cargo, go, and maven, in that order. +records why. Remaining: additional key backends and the TUI. The next formats +are nix cache, cargo, go, and maven, in that order. `import` is implemented, and +so is S3 beyond PyPI — see the paragraph below, which was written when it landed +and outlived this sentence. S3 beyond PyPI turned out not to be a declaration. The adapter no longer hardcodes `simple/index.html`: `host.Repository` carries `CommitPaths`, filled diff --git a/README.md b/README.md index c561194..bff64b1 100644 --- a/README.md +++ b/README.md @@ -94,9 +94,10 @@ Replacing an existing managed release needs an atomic directory-entry exchange, so that path is implemented on Linux and macOS; creating a first release is portable. There is no Windows build yet. -Additional key backends, more formats, `import`, and an interactive setup remain -Phase 3 work. [ARCHITECTURE.md](ARCHITECTURE.md) is the implementation contract -and [PLAN.md](PLAN.md) the broader design. +Additional key backends, more formats, and an interactive setup remain Phase 3 +work; `import` is done and has [its own section](#importing-an-existing-repository). +[ARCHITECTURE.md](ARCHITECTURE.md) is the implementation contract and +[PLAN.md](PLAN.md) the broader design. ## Promotions and yanks diff --git a/engine/ci.go b/engine/ci.go index f24e4b1..7dddc2f 100644 --- a/engine/ci.go +++ b/engine/ci.go @@ -177,7 +177,7 @@ jobs: echo "no release to adopt; republishing current desired state" exit 0 fi - # TODO: write the script that adopts your producers' release assets, + # FILL IN: the script that adopts your producers' release assets, # or call "snailmail adopt --sha256 ... --public-origin REPO URL" per # artifact. Which repository an asset belongs in, and whether a # producer publishes digests for all of them, is project-specific. @@ -305,7 +305,7 @@ func ciSiteStep(directories []string) string { # %s into a directory, so something else has to serve it. This # assembles one site and pushes it to a branch Pages is configured from. # - # TODO: set the CNAME below, or delete it if the site is served from a + # FILL IN: set the CNAME below, or delete it if the site is served from a # github.io address. - name: Assemble the site run: | diff --git a/engine/ci_gitlab.go b/engine/ci_gitlab.go index 8af01db..cef9cb3 100644 --- a/engine/ci_gitlab.go +++ b/engine/ci_gitlab.go @@ -126,7 +126,7 @@ publish: if [ -z "${TOOL_PROJECT:-}" ] || [ -z "${TOOL_TAG:-}" ]; then echo "no release to adopt; republishing current desired state" else - # TODO: write the script that adopts your producers' release assets, or + # FILL IN: the script that adopts your producers' release assets, or # call "snailmail adopt --sha256 ... --public-origin PROJECT URL" per # artifact. Which repository an asset belongs in, and whether a producer # publishes digests for all of them, is project-specific. @@ -152,7 +152,7 @@ publish: out.WriteString(` # Pushed with a project access token, because the job token cannot write to # a protected default branch. # - # TODO: create a project access token with write_repository, store it as the + # FILL IN: create a project access token with write_repository, store it as the # masked CI/CD variable SNAILMAIL_PUSH_TOKEN, and allow it to push to the # default branch. - | @@ -188,7 +188,7 @@ func gitlabKeyScript(keys []string) string { # never in an environment a child process inherits. The passphrase is a file # for the same reason. # - # TODO: for each key below, add a file-type variable holding the armoured + # FILL IN: for each key below, add a file-type variable holding the armoured # private key, and a variable naming the reference snailmail knows it by. - | set -euo pipefail diff --git a/go.mod b/go.mod index 4c893a4..6e97ec8 100644 --- a/go.mod +++ b/go.mod @@ -4,37 +4,37 @@ go 1.25.0 require ( github.com/Masterminds/semver/v3 v3.5.0 - github.com/klauspost/compress v1.19.1 + github.com/klauspost/compress v1.19.2 github.com/ulikunitz/xz v0.5.16 - golang.org/x/net v0.57.0 + golang.org/x/net v0.58.0 golang.org/x/sys v0.47.0 gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/ProtonMail/go-crypto v1.4.1 - github.com/aws/aws-sdk-go-v2 v1.43.0 - github.com/aws/aws-sdk-go-v2/config v1.32.31 - github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 - github.com/aws/smithy-go v1.27.5 + github.com/aws/aws-sdk-go-v2 v1.43.7 + github.com/aws/aws-sdk-go-v2/config v1.32.38 + github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3 + github.com/aws/smithy-go v1.27.9 github.com/pelletier/go-toml/v2 v2.4.3 ) require ( - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.19.30 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 // indirect - github.com/cloudflare/circl v1.6.4 // indirect - golang.org/x/crypto v0.54.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.37 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.38 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.38 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.38 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.39 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.7 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.45.7 // indirect + github.com/cloudflare/circl v1.6.5 // indirect + golang.org/x/crypto v0.55.0 // indirect ) diff --git a/go.sum b/go.sum index d21e05e..252d55e 100644 --- a/go.sum +++ b/go.sum @@ -2,54 +2,54 @@ github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAw github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/ProtonMail/go-crypto v1.4.1 h1:9RfcZHqEQUvP8RzecWEUafnZVtEvrBVL9BiF67IQOfM= github.com/ProtonMail/go-crypto v1.4.1/go.mod h1:e1OaTyu5SYVrO9gKOEhTc+5UcXtTUa+P3uLudwcgPqo= -github.com/aws/aws-sdk-go-v2 v1.43.0 h1:fharf/WhbRAVZ1du0QL7roNFxZ6T/sWr+4Ni617bwSI= -github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E= -github.com/aws/aws-sdk-go-v2/config v1.32.31 h1:n4nY9O3QKoHIkL85EX+V8RcMFtOhlpTFhGArg915PXk= -github.com/aws/aws-sdk-go-v2/config v1.32.31/go.mod h1:PN0NYDCCoOpGGsZ2+elDUidmHfQBPyYzN2GCgl8HEBs= -github.com/aws/aws-sdk-go-v2/credentials v1.19.30 h1:TTCvvzFU6gXa4iJecNG/0F/B0oYTiazoRECr2XyLHrY= -github.com/aws/aws-sdk-go-v2/credentials v1.19.30/go.mod h1:jKxAp2AEncnliinzpgOSZDFv6+VjvWhjw/AtbfsWT9U= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 h1:kfVL5wAunCJycL6MOQ6aNh6PlAYEymflcjuKmrWUA0o= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31/go.mod h1:nWfRNDAppujCQgOUd43lKT4yeLv9z3nJ3bw1G3BgQKo= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 h1:Z8F3hfCY33IGpJjFAnv0wvtv1FIKj1GHmRDEYqy64tw= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 h1:hyOxUyXdh3AyjE93gBgsfziJag9ACwcs+ZpDBLzi8mw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 h1:0MrUL35H/Y4kdFfItoR5jCgtDQ4Z/8LudAoIHRfA4hE= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 h1:mdPwDQPqxlw9Sc62Nt15yjEcARaDbPXkjRYtXsUripo= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24/go.mod h1:ls5ytnwLTcQaUu32fMYXFI3MjpKuTwL840PAm9iqyEg= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 h1:w2SIhW92DZPFrSL4ksVCr8IYff5OZwIcxg8+95tzvAI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 h1:jWXtZdCnhXa9sGFixRaU2AxT4DIVse9HS4E2f+/KwV0= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32/go.mod h1:9JS1UpfVvyD/ZPX8GsKb/Pq8scEM+7GP5fqh9SwH7po= -github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 h1:7QZWVJZWzHivHWIa+5TELLaBBkbuoj0GPwQtMlJ0sqk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0/go.mod h1:fcvq5L7dK+5cQFicEJwpI6e6Wn8NY2i6yT5wRLYVc7s= -github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 h1:OHH5iTQvVGmfHjX/5Q+vFuA/Rf2x6/95aJ/75QCQSm4= -github.com/aws/aws-sdk-go-v2/service/signin v1.5.0/go.mod h1:mCF3AK9PpL49oOrhniUXWAfhVBVQ/XbytoE5eccZUIs= -github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 h1:CaJyYhxBE0M/HJX/YvSaSmQlsI91VHB0lKU8LtLxL3A= -github.com/aws/aws-sdk-go-v2/service/sso v1.33.0/go.mod h1:+e6BMRMPjBQoCw/WovYR9GLy2IU0z4Q77smOB1DraSg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 h1:tC323YV77QdafeBr6LUhLDTsboyuyHLNRwAyCP44kGU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0/go.mod h1:SfLK1sgviHmbI+MozR9iDwDjL4cdCVZtahsjoR+z7wg= -github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 h1:Pd6PNlp4t8PTXxqzstICl52Wsy78vpjFZ7PRUj44mJc= -github.com/aws/aws-sdk-go-v2/service/sts v1.45.0/go.mod h1:rmQ0TnHzuLPmabgjPcsywhsSOmaBDgzR4zvDxSPsGdg= -github.com/aws/smithy-go v1.27.5 h1:d1ro7KpYOYwP6m73YFa+Kc/A130VsAdX68SpsJwARMM= -github.com/aws/smithy-go v1.27.5/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= -github.com/cloudflare/circl v1.6.4 h1:pOXuDTCEYyzydgUpQ0CQz3LsinKjiSk6nNP5Lt5K64U= -github.com/cloudflare/circl v1.6.4/go.mod h1:YxarevkLlbaHuWsxG6vmYNWBEsSp4pnp7j+4VljMavY= -github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= -github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/aws/aws-sdk-go-v2 v1.43.7 h1:msCzvkeYJA9ehbV8mRRmkZLo/zJg/+yDVLNtflg83hQ= +github.com/aws/aws-sdk-go-v2 v1.43.7/go.mod h1:tXpPM+v0D1lndmga+HqqLDIzUFJlEeR21aspVklHF00= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18 h1:LAfOuhAH331fmOjTQpAaOlH+Ftn7RzSDJ2VFwjdMMy4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.18/go.mod h1:4e5xhuXHx1e4U9EthvbPP1r/DIMp5c2823OL8karzcM= +github.com/aws/aws-sdk-go-v2/config v1.32.38 h1:n4yPHBjtQ3BrIIUyk0/LAqf/BL2iv0Tw6XZcMRzM0ps= +github.com/aws/aws-sdk-go-v2/config v1.32.38/go.mod h1:dencYsOS1R7rBy8zehCvwBYzdxxL4Q/nRK7In03wjN8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.37 h1:FJ8Iz4/xISMB/rwLlgfWujfGDFWr0oneQgtA6KPcYLY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.37/go.mod h1:Q6pWOgVUp49x4g5QVi29wHofUoICnZ+Zq4jHbRN/7ec= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.38 h1:Nqo2jU1wz5rnBM9XQyXfVD1RP8txkbP3EDx8hR/hbCE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.38/go.mod h1:PzJFHhjR2vWFKHe8HmY5Lxhvwyxnr5MERtk0nDxWNbk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.38 h1:MBMg0zJ6i4TkAJ0dVFLKKn2cOkY6FkicmUDM67BRr6g= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.38/go.mod h1:9MWuJbyiUyj6eA7W1/zm1zuePDPSB3g+xcgRQeMWsXc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.38 h1:lHm4jPf3k1Lz5ZWc+Vcn3MKVwym+26kWCba9FkJ4f0Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.38/go.mod h1:Rn+P2XR+FbyZzjmWKjg/KUZNxmGfr5oZwh5jQiE+CzI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.39 h1:vo4xvMRs/F6h1E52qsgLqCQgWIQXgIJUauG6rlZEh4U= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.39/go.mod h1:jB03R1ij/A+OE2e1dz6vgj076gd7vlYcfstAzj3HcnU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17 h1:OvYZOB3qA6zvfdRFiRFRzVSiElMYrz3GdntkXZxlp1o= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.17/go.mod h1:JgR/2Ew50ACfIWau1oeMRX59tMtC0kM+PYQGEaT04cY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31 h1:uZOinZb+h7lZw8IYzP1z1IuEnueB76/EFkcf/fEW4Ag= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.31/go.mod h1:NRtwAM/p5VRt03TlEUs0pH3TeWamWdf4YyJpSrzPYLc= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38 h1:H/5TI1jqaHsNoDQ60UwvPvJBg4GURkinXI3Qga29t2w= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.38/go.mod h1:PTVFf+XH++7NJOky+RLBYQx0QA5NcaeEYFQ2fsi0nwo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39 h1:HLPAVrlLDaN2boN0xJx7MgaQDNEO3Q+c9L6kl/8m47Q= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.39/go.mod h1:Pg/dVfsNkm1hsIDK/gMvCKtmyNfNTV12mrgHqVE/6Oo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3 h1:IKoCZqfWfZzSBi16QFQ+QcbQ3LRQ7QgB1S5tDAyPBQQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.107.3/go.mod h1:RBpRcXiM4s2pOInVs32GsBonnje+fiAj4mcrStRmlCA= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.7 h1:YcczQ6zNH/ojIzD/ikDrO+RfW06wmdMp18d4NH5hXY4= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.7/go.mod h1:nl9RVnb9ulgAYzOkjLq1NyFxmWcnH2maCUEuOdESy98= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.7 h1:P+bMNiA93gyuYT3Oh+4dWtvrnGcu2bd9Uy5hRJM8BNo= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.7/go.mod h1:zy+397isDFLvleg9H18Zq2MGzMso7uKyJyzR7DWSgFk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.7 h1:WWkehGZ4nWtOKLMy0yi8+RqzzVqAGe60hGaxwF06JAw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.7/go.mod h1:T8AI4SbQYm9ybcVmki2T3n7Qg1g3kfWoeQlNwNYOyO8= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.7 h1:yU/9y2r7s9kSUPbHXbpQTa4LA8kt+CMgpu1OBrhx8p4= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.7/go.mod h1:0lQTDEBArMevQXpxu443LVGjKxxEeSsSnrw9n8YiTMg= +github.com/aws/smithy-go v1.27.9 h1:flT/ACSU1ksz3V+8wj8kN8DOB9tsc/ggWPTJXIieRpw= +github.com/aws/smithy-go v1.27.9/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cloudflare/circl v1.6.5 h1:O64F26HEqNhznd/hrC5KZXVKYuKM2rx4deZDTc4ihQA= +github.com/cloudflare/circl v1.6.5/go.mod h1:h5LNyxAc5nTue9DS5jT+48en2PSDYt3zdGnz5OstK6c= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/ulikunitz/xz v0.5.16 h1:ld6NyySjx5lowVKwJvMRLnW5nxKX/xnpSiFYZ/Lxur0= github.com/ulikunitz/xz v0.5.16/go.mod h1:H9Rt/W6/Qj27PGauhQc6nfCDy7vHpzsOThBSaYDoEhw= -golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= -golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= -golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= -golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From c8bca68d9807ccf2f5f4e1fdee959bdd95f76c43 Mon Sep 17 00:00:00 2001 From: Polina Simonenko Date: Sat, 22 Aug 2026 16:33:01 +0200 Subject: [PATCH 26/26] Split the reference out of the front page --- README.md | 838 +--------------------------------------------- docs/adopting.md | 130 +++++++ docs/curating.md | 160 +++++++++ docs/deploying.md | 83 +++++ docs/hosts.md | 174 ++++++++++ docs/operating.md | 196 +++++++++++ docs/signing.md | 128 +++++++ 7 files changed, 887 insertions(+), 822 deletions(-) create mode 100644 docs/adopting.md create mode 100644 docs/curating.md create mode 100644 docs/deploying.md create mode 100644 docs/hosts.md create mode 100644 docs/operating.md create mode 100644 docs/signing.md diff --git a/README.md b/README.md index bff64b1..e3a7791 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ ssh directory or GitHub Pages commits a whole tree at once and serves all six. The `rsync` host serves every format but publishes no preview site, so it works under the `auto` gate and is refused under `pr` and `approval`, which exist to -review one. [Publishing over ssh](#publishing-over-ssh) has the detail. +review one. [Publishing over ssh](docs/hosts.md#publishing-over-ssh) has the detail. One caveat for object storage: the browsable `index.html` is generated fresh for every revision, so it is kept with the release rather than at the repository root. @@ -95,828 +95,22 @@ so that path is implemented on Linux and macOS; creating a first release is portable. There is no Windows build yet. Additional key backends, more formats, and an interactive setup remain Phase 3 -work; `import` is done and has [its own section](#importing-an-existing-repository). -[ARCHITECTURE.md](ARCHITECTURE.md) is the implementation contract and -[PLAN.md](PLAN.md) the broader design. - -## Promotions and yanks - -Promotions and yanks edit only placement records. Package versions, blob -bindings, and publication history remain available for later re-promotion: - -```sh -go run ./cmd/snailmail promote --track testing python snail-demo 1.2.3 -go run ./cmd/snailmail yank --track stable python snail-demo 1.2.3 -# Or remove every placement for the exact version: -go run ./cmd/snailmail yank --all python snail-demo 1.2.3 - -git diff -- repos/python.lock.toml -git add repos/python.lock.toml -git commit -m "update Python package placements" -go run ./cmd/snailmail plan -go run ./cmd/snailmail apply -``` - -The repository and package version are exact; promotion does not copy package -versions between repositories. Each repository renders only its configured -`--track` (default `stable`), and Debian additionally renders only placements for -its configured suite. Other placements remain recorded but are not exposed in -that view. Removing the final visible placement publishes a valid empty index -while retaining immutable package and blob records in Git. Debian defaults a -new placement's distro to the configured suite; `--distro DISTRO` selects another -distro coordinate explicitly. - -Retention pruning removes only older placements, independently per package, -track, and distro, using native PEP 440, Debian, or SemVer precedence: - -```sh -go run ./cmd/snailmail prune python --keep 5 -git add repos/python.lock.toml -git commit -m "prune old Python placements" -go run ./cmd/snailmail plan -go run ./cmd/snailmail apply -``` - -Versions tied at the retention boundary are kept together. Prune does not remove -package-version records, CAS objects, remote blobs, or publication history; -physical blob GC remains a separate future operation with tombstones and a grace -period. - -## Auditing what is published - -`snailmail check` is a read-only integrity audit of every retained package -version, including yanked and pruned versions. It verifies local CAS bytes or -fetches the configured S3 authority into temporary storage, reparses native -package facts, and revalidates historical publication bindings. Upstream release -discovery remains unavailable until releases are modeled. `check --origins` -re-fetches explicitly adopted URLs and compares their pinned bytes; default -checks remain offline from external sources. Origin checks process at most four -sorted records per run; `--origin-offset` selects subsequent batches. - -## Importing an existing repository - -Most people arrive with a repository already published somewhere. `import` reads -its index and records every artifact it names, rather than adopting each by hand: - -```sh -snailmail import --project six --public-origin --dry-run python https://pypi.org/ -snailmail import --project six --public-origin python https://pypi.org/ -``` - -Each artifact goes through the same path as `adopt`: fetched, checked against the -digest its index published, and recorded with its origin URL so it can be refetched -later. - -What a recorded digest is worth depends on where it came from — an index someone -signed is not the same as one served over TLS alone — so the lock records that -beside it, and `status --json` reports the counts: - -``` - python -> {"index-stated": 2} -``` - -`index-stated` means the index published the digest and the fetched bytes matched, -which is the strongest a simple index supports. Artifacts recorded by `adopt` read -as `operator`, including in locks written before this field existed. PLAN.md §3.8 -has the full set and what each level is worth. Anything the index names but does not publish a SHA-256 for is skipped and -reported — a locked artifact is pinned to a digest someone stated in advance, and -one computed from the bytes a download happened to return would prove only that the -download was self-consistent. - -One artifact failing does not abandon the rest, so a repository with a broken file -imports the other 47 and names the one that failed. - -PyPI and Helm today. A Helm repository has no per-project page, so importing one -imports the repository: - -```sh -snailmail import --public-origin --dry-run charts https://grafana.github.io/helm-charts -``` - -Where an index lists several mirrors of a chart, the origin recorded is the URL -that actually served. Where it lists the same name and version twice, both entries -are skipped and named — two entries claiming one identity cannot both be it, and -picking one would record bytes nobody chose. - -Debian too, which reads a suite rather than a project: - -```sh -snailmail import --public-origin --dry-run --suite bookworm apt https://deb.debian.org/debian -``` - -A Debian import walks the chain rather than trusting the leaf: `Release` states the -digest of `Packages`, and a `Packages` whose bytes disagree is refused outright — -nothing in an index that failed its own root can be trusted. That is why Debian -artifacts record `index-chain` where PyPI and Helm record `index-stated`. The -`Release` signature itself is not verified yet, so the root of trust is still the -transport, and the lock says exactly that. - -And yum, which needs no extra flags because a repository root is one repository: - -```sh -snailmail import --public-origin --dry-run rocky https://dl.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os -``` - -yum walks the same chain as Debian: `repomd.xml` states the digest of -`primary.xml.gz`, and a primary whose bytes disagree is refused. So rpm artifacts -also record `index-chain`. If `repomd.xml` states only a sha1 or md5 for its -primary, the import stops rather than quietly recording a weaker provenance than -was asked for — signing `repomd.xml` is what would raise this to `signed-index`. - -Multilib is preserved: `i686` and `x86_64` builds of one name-version are two -artifacts, recorded as two blobs under the same package version rather than one -overwriting the other. - -And Alpine, which is the honest exception: - -```sh -snailmail import --public-origin --dry-run alpine https://dl-cdn.alpinelinux.org/alpine/v3.19/main/x86_64 -``` - -An `APKINDEX` entry carries `C:Q1…`, which decodes to a SHA-1 of the package's -*control section* — not of the file. Checked against Alpine's own archive: the -index states `6026787b…` for `7zip-23.01-r0.apk`, whose actual SHA-1 is -`76a96042…`. They differ because they are digests of different things. So there is -nothing in an Alpine index to pin an artifact to, and an imported apk records -`computed`: a digest of the bytes snailmail downloaded, and nothing more. - -That is allowed but never hidden. If your workspace will not accept -unauthenticated bytes, say so once: - -```sh -snailmail import --public-origin --min-provenance index-stated alpine https://… -``` - -Every artifact then reports why it was refused, rather than being pinned to -something weaker than you asked for. The floor works for any format — a Debian -import establishes `index-chain`, so it passes an `index-stated` floor. - -## Adopting an artifact from a URL - -`snailmail adopt --sha256 HEX --public-origin REPOSITORY URL` records one -explicitly selected artifact in an existing owned repository. The lowercase -SHA-256 pin is mandatory; `--public-origin` confirms that the complete requested -URL is non-secret and may be committed and printed. Lock schema 2 retains that -URL, plans display every visible adopted acquisition, and `--dry-run` validates -without changing CAS or lock state. An adopted artifact is streamed to disk rather -than held in memory, so its size costs disk and time rather than resident memory; -`SNAILMAIL_MAX_ARTIFACT_BYTES` raises the 2 GiB ceiling that remains. Adoption -requires the local blob store and does not claim authorship, build provenance, -source signatures, or historical snailmail publication. - -## Inspecting a workspace - -`snailmail status` also reports size: each repository's lock in bytes, the count -and total size of the distinct artifacts it binds, and the workspace's lock total -with the largest repository named. The lock is parsed whole on every plan and -every apply, so its size is the number that predicts where a workspace stops -being comfortable — and which repository to split when it does. - -`snailmail status` reports committed workspace evidence without contacting -hosts or providers. Human output summarizes visible and retained versions, -visible publication-binding completeness, and whether a managed deployment -receipt is recorded; `--json` emits the same deterministic schema for -automation. A receipt is evidence of a prior successful apply, not proof that a -host currently serves those bytes. - -`snailmail rollout` answers when each version reached a client, derived from the -publication ledger rather than stored anywhere. The ledger already records one -append-only entry per publication, so the date is read back rather than kept a -second time. A version is listed with the date it was first published, the -number of published trees that have carried it, and whether the repository still -serves it; `--withdrawn` includes versions that were published and later yanked -or pruned, because publication is immutable even when the offer is not. - -`snailmail ci github` and `snailmail ci gitlab` emit a pipeline that publishes -this workspace, to stdout rather than to a file: it carries decisions snailmail -cannot make — which registry to pull verification images through, which secret -names a project uses — and a file the tool owned would be rewritten over an -operator's edits. What it does derive is what the workspace already says: which -signing keys need materialising, whether a foreign architecture needs emulation, -and which repositories publish into a directory something else must serve. Both -providers derive the same facts; only the rendering differs, and it differs more -than wording. A GitLab runner has no Docker daemon, so client verification needs -one as a service; its jobs share no filesystem, so what apply builds is declared -as an artifact; and Pages there serves a job artifact rather than a moved ref, so -there is no orphan commit. - -## Two runners at once - -The workspace lock is a file inside the checkout, so it serialises operations on -one machine and not two. What protects a repository when two CI runners publish -at the same time is the conditional commit: each states the revision it observed, -and the second to arrive is refused because that is no longer what is live. - -So the failure mode is **one run fails, not one run waits**. It fails as a stale -plan, which is a retry signal — replanning against what is actually live and -applying again succeeds. A pipeline that publishes from more than one place at once -wants a concurrency group so this is rare rather than routine; the generated -workflows set one. - -Two things worth knowing. Staging is *not* exclusive, deliberately: two runners can -both stage, because a stage writes only under its own identifier and touches nothing -a client reads. And a local host cannot be shared at all — its output path is -relative to its workspace — so this only arises for object storage and Pages, where -two runners can name the same bucket and prefix or the same repository and branch. - -## How large a workspace can get - -A repository lock is parsed whole on every plan, apply, status and check, so its -size is what decides how long those take and how much memory they need. Measured -at roughly 385 bytes per package-version, with parsing costing about one and a half -times the file in heap: - -| package-versions | lock file | parse | heap | -|---|---|---|---| -| 2,000 | 0.8 MB | 5 ms | 1 MB | -| 20,000 | 7.7 MB | 44 ms | 12 MB | -| 100,000 | 38.5 MB | 226 ms | 59 MB | - -snailmail refuses a lock over **128 MiB** — about 330,000 package-versions — and -says so, naming the repository, the size and what to do: - -``` -repository "repos/apt.lock.toml" has a 200 MiB lock, over the 128 MiB limit, and -parsing it needs roughly 300 MiB of memory on every plan and apply: prune retained -versions, split the repository, or set SNAILMAIL_MAX_LOCK_BYTES to proceed anyway -``` - -That is the honest state of things: one lock per repository is the current design, -and a workspace past this size wants the lock sharded per package rather than a -larger limit. The limit exists so that a workspace which has outgrown the design -finds out in a sentence instead of getting slower until something is killed. -`snailmail status` reports every lock's size, so the number can be watched before -it becomes a refusal. - -Artifacts themselves are not held in memory — they stream from content-addressed -storage — so the ceiling is index and lock size, not repository size. Generated -indexes are held whole, which for Debian and yum is the binding constraint at -around a million package-versions. - -Past 2,000 package-versions a repository's lock is written as one file per -package, under a root that indexes them and a Merkle digest over the set: - -``` -repos/apt.lock.toml the root: schema, placement, shard index, Merkle root -repos/apt.lock.d/3f/… one file per package -``` - -Nothing about a lock's meaning changes — it still has a single identity that a -plan pins, and a shard edited on its own is refused by name rather than accepted -quietly. What changes is what a publication costs. Adding one version rewrites one -small file instead of the whole lock, so a reviewer sees the package that changed -rather than a multi-megabyte diff. Measured at 10,000 packages: adding a version -takes 108 ms and reading the lock 102 ms, against 3.4 s to write when every shard -is compared and 950 ms to read them one at a time. - -The transition happens once, automatically, when a repository grows past the -threshold, and a lock never goes back to one file — returning would itself be a -diff rewriting everything. The first sharded write of a large repository creates -every file and takes seconds; subsequent ones do not. - -## One workspace, or one per team - -Use one workspace for the whole organisation unless you have a reason not to. - -It sounds like fifty teams would contend on one review queue and one lock, and -they do not. Each repository has its own lock file, so two teams publishing to two -repositories touch two different files and git merges them; `snailmail.toml` is the -only shared file, and it changes when a repository is configured rather than when -one is published to. Point CODEOWNERS at `repos/.lock.toml` and each team -reviews its own publications. The workspace lock covers a single checkout, so CI -runners with their own checkouts never wait on each other — what orders two -concurrent publications is the host refusing the second, per repository. - -Split when one of these is true: - -- **A team needs its own blob store.** A plan is bound to one, so a workspace has - exactly one. -- **A team cannot share read access** to the state repository, because a workspace - is a git repository and access to it is all-or-nothing. - -Separate workspaces can still publish into the same bucket under different -prefixes. What you lose is the shared view: `snailmail site` describes one -workspace, so splitting means several index pages and no single answer to where a -package is published. - -## Publishing over ssh - -The `rsync` host publishes to a directory on a machine you reach over ssh — the -shape most people already have, where a web server serves a filesystem path. - -```sh -snailmail setup rpm --name yum --host rsync \ - --target deploy@packages.example \ - --output /srv/www/yum \ - --base-url https://packages.example/yum -``` - -`--output` is the absolute path on the far side, where for a local host it is -workspace-relative. `--target` is the ssh destination; port, key and jump host -belong in `ssh_config`. - -**This is the only host that serves every format**, including a signed yum -repository. A revision goes live by renaming one symlink, so the whole tree -switches at once and the number of files that must change together does not -matter. An object store commits one object and no more, which is why it cannot -serve Debian, Alpine, or signed yum at all. - -Two POSIX primitives carry the guarantees, neither of them from the transport: - -- **`rename(2)` makes a publication atomic.** The published path is a symlink into - `.snailmail/releases//`, and a revision goes live when a new symlink is - renamed over the old one. A client follows one whole revision or the other, - never a half-copied tree. The tree is copied to a staging path and renamed into - its release first, so the symlink never points at a partial release. -- **`mkdir(2)` makes it conditional.** There is no compare-and-swap on a symlink, - so the expected revision is checked and the swap performed while holding a lock - directory whose creation fails if it already exists. Two runners publishing at - once means one fails, not one waits — the same answer every other host gives. - -`snailmail collect` works here too. Every revision leaves a tree under -`.snailmail/releases/`, and the swap that makes a new one live does not remove the -one it replaced. Collection keeps the live revision — read from the far side, not -trusted from the workspace — and whatever retention names. Unlike object storage -there is no restore target held back, because rollback is not offered, so an older -revision survives only if you ask for it. - -Requirements and limits, stated because they are not checked from this side: - -- The published path and its release directory must be on **one filesystem**, - since `rename` is not atomic across mount points. -- ssh options — port, key, jump host — belong in `ssh_config`. A second place to - configure ssh is a second place for it to be wrong. -- A path that already exists and was not published by snailmail is **refused** - rather than replaced, so a publication cannot unpublish somebody else's files. -- **Rollback is not offered.** Pointing the symlink at an earlier release is easy; - establishing that the release is still intact is not, because nothing on the far - side verifies it. The adapter declines rather than claiming a rollback it cannot - check. -- **No preview site**, so the `pr` and `approval` gates are refused. Nothing is - copied to the far side before the commit, so there is no URL a reviewer could - install from. Under the `auto` gate a real client still installs from the exact - staged bytes before they are published; what goes unchecked is that the far side - serves them correctly, which is what a preview buys. - -## Browsing a bucket-hosted repository - -A repository published to object storage serves `index.html` at its root, so -`https://packages.example/apt/` opens in a browser and shows what is published, -how to install it, and each artifact's digest. - -That page is a **convenience copy and is not covered by the publication's -guarantees**. Nothing verifies it, `observe` does not read it, and a rollback does -not restore it — it is refreshed by the next publication. The verified copy lives -inside the release directory, because it is regenerated for every revision and -writing it canonically would leave the previous revision unverifiable after a -rollback. Clients are unaffected either way: they read `simple/`, `index.yaml`, -`repodata/` or `SHA256SUMS`, and those are the complete, verified answer. - -The page shows the 500 most recently published artifacts and says so when there -are more. A rendered row costs about 610 bytes, so a Debian suite of 63,440 -artifacts would otherwise be a 38 MB page rebuilt and re-uploaded on every -publication. Set it with `Window` if you want a different size; the footer always -reports the repository's true total. - -## Undoing a publication - -When a publication succeeded and turned out to be wrong: - -```sh -snailmail rollback apt # report what would be restored -snailmail rollback apt --yes # restore the previous publication -``` +work; `import` is done and has [its own page](docs/adopting.md#importing-an-existing-repository). -**One step, deliberately.** A published revision carries a reference to the root it -replaced, and collection protects that target for exactly this reason — so going -back one publication is something the host can still verify. Going back further is -not offered: nothing keeps a chain of restore references, and the older releases -may have been collected. A rollback whose target cannot be checked is not a -rollback. +## Documentation -**Not every host can do it.** A local directory and an ssh host both decline, -because neither can establish that the release it would point back at is intact. -There the answer is to revert the lock in git and publish forward, and `rollback` -says so rather than failing obscurely. +The front page is the shape of the thing. Everything else is one page per +question, because a reference manual and an introduction want different orders +and this file was trying to be both. -**Your lock still describes what you rolled back from.** That is deliberate — the -rollback changed what is served, not what your workspace wants — but it means the -next `apply` will republish what you just undid unless you revert or amend the lock -first. The command says this every time it runs. +| | | +|---|---| +| [Where repositories are published](docs/hosts.md) | ssh, object storage, GitHub Pages, and what each can serve | +| [Deciding what a repository publishes](docs/curating.md) | promote, yank, prune, collect, rollback, and raw artifacts | +| [Signing and review](docs/signing.md) | keys, rotation, and the gates that hold a publication for a person | +| [Taking on a repository that already exists](docs/adopting.md) | import, adopt, and inspecting somebody else's | +| [Running it](docs/operating.md) | status, check, exit codes, concurrency, and how large a workspace can get | +| [Shipping snailmail itself](docs/deploying.md) | the container image, generated CI, and binary size | -## Collecting superseded releases - -An object store keeps every revision it has ever published: a publication writes a -whole tree under `.snailmail/releases//`, and nothing removes the previous -one. A project publishing daily accumulates a copy a day. - -```sh -snailmail collect # report what would be removed -snailmail collect --keep 3 --yes # remove it, retaining three recent revisions -``` - -Reporting is the default; `--yes` deletes. Three things always survive whatever -`--keep` says: the live revision, whatever its rollback depends on, and any -revision the ledger records within `--keep`. The first two are established by -reading the host rather than the workspace, so a checkout whose ledger is behind -cannot delete what is being served. - -Note that with only two publications nothing is collectable — both are protected, -the live one and the one it rolls back to. Local directories and GitHub Pages have -nothing to collect and say so: the first leaves nothing behind, and the second -leaves unreachable objects to git. - -Collection is the one operation that needs `s3:ListBucket`, so it may run under a -different credential from publishing. - -Two things accumulate, and they are collected together. Release directories are -one; the other only exists for helm and unsigned rpm, which write their artifacts -at the paths clients fetch them from rather than inside a release directory — -because `index.yaml` and `repomd.xml` name those paths, and rewriting a signed rpm -would invalidate it. A chart dropped from the workspace a year ago is still a -billable object that no release collection has ever touched. `collect` now removes -those too, keeping every file that any surviving revision publishes. - -Two revisions are always protected: the live one and the one its restore rolls -back to. So the first collectable revision is the live one's grandparent, which is -worth knowing before reading a `--dry-run` and concluding it reclaims less than -expected. If a surviving revision's release descriptor cannot be read, the whole -collection is refused rather than run against an unknown reference set — deleting -too little costs storage, deleting too much costs the repository. - -How much a repository keeps is part of its configuration, not a flag someone -remembers: - -```sh -snailmail setup deb --name apt --keep 20 ... -``` - -`collect` uses that, and `--keep N` overrides it for one run. The order matters: -collection is the only operation here that deletes published bytes, so the policy -belongs in the reviewed manifest where changing it is a diff, exactly like changing -a gate or a signing key. A repository that declares nothing keeps the default of -10, so nothing configured before this behaves differently. - -## Inspecting someone else's repository - -`snailmail doctor URL` needs no workspace and inspects a public HTTPS PyPI, -Debian, or Helm repository. It parses bounded native indexes, follows at most the -configured artifact limit, and checks referenced availability, size, SHA-256, -archive validity, and package identity. Use `--project` for PyPI artifacts and -`--suite` for a Debian base URL. Debian Release signatures and Helm provenance -are explicitly reported as unverified in this initial slice. Runs inspect at -most four artifacts, cap each expanded archive at 64 MiB, and stop after two -minutes. - -## Raw repositories - -`raw` repositories publish artifacts that carry no ecosystem metadata: release -tarballs, static binaries, installers. Every other format reads a package name -and version out of the bytes; raw cannot, so identity comes from the filename by -convention, or from you when the convention does not apply. - -```sh -go run ./cmd/snailmail setup raw --name tools --output public/tools - -# _[_][_]. is read without flags. -go run ./cmd/snailmail add tools ./dist/ttysvg_0.1.2_linux_amd64.tar.gz - -# Anything else needs identity supplied, including a name with an underscore, -# which cannot be told apart from an extra field. -go run ./cmd/snailmail add --name ttysvg --version 0.2.0 tools ./dist/build-final.bin - -go run ./cmd/snailmail plan && go run ./cmd/snailmail apply -go run ./cmd/snailmail verify raw --repo public/tools -``` - -Artifacts are published at `//` beside a generated -`SHA256SUMS` and `index.html`. Identity lives in the path on purpose: because -it may have come from a flag rather than from the bytes, the published tree has -to record it somewhere a later reader can check without knowing what was typed. -`SHA256SUMS` is the interchange format — `sha256sum -c` verifies a raw -repository with no snailmail present. Raw has no signing: it is not an -ecosystem, so no client knows to check a signature. - -## Signing keys - -Signed Debian repositories use an encrypted private key outside the workspace -and commit only canonical public forms. Set a passphrase through the environment -(never an argument), generate the key, and reference it during setup: - -```sh -export SNAILMAIL_KEY_PASSPHRASE='use-a-secret-manager-value' -go run ./cmd/snailmail keys new archive-signing --expires-in 17520h -go run ./cmd/snailmail setup deb \ - --name debian \ - --output public/debian \ - --suite stable \ - --architectures amd64 \ - --signing-key archive-signing -go run ./cmd/snailmail keys audit -git add snailmail.toml keys/ repos/debian.lock.toml -git commit -m "configure signed Debian repository" -go run ./cmd/snailmail plan -go run ./cmd/snailmail apply -``` - -The file backend stores encrypted private keys under -`$XDG_DATA_HOME/snailmail/private-keys` (or -`~/.local/share/snailmail/private-keys`) with mode `0600`. Passphrases must -contain at least 24 bytes and should come from a secret manager. In a container, -prefer `SNAILMAIL_KEY_PASSPHRASE_FILE` pointing at a mounted file: a container's -environment stays readable through the runtime's inspection API for as long as -the container exists, whereas a file can be read once and removed. Setting both -variables is rejected rather than resolved silently. `keys publish` -recreates missing public forms after validating the private identity. Planning -compiles content-addressed `InRelease` and `Release.gpg` signing nodes over the -exact Debian `Release` bytes, signs each node twice to prove deterministic -backend behavior, verifies both signatures, and embeds only public node -responses in the reviewed plan. Apply does not load the private key. Signed -client verification uses apt's `signed-by`; migrated unsigned Debian -repositories remain readable but `keys audit` reports them as errors. New -unsigned Debian setup requires the explicit `--allow-unsigned` compatibility -opt-out. - -Every format the knowledge bundle records as signable is signed, each in the -form its own clients read: Debian publishes `InRelease` and `Release.gpg`, yum an -armored key beside a detached `repomd.xml.asc`, Alpine one signature per -architecture index under the key filename its index names, and Helm a `.prov` -per chart beside the archive it covers. The published key differs with the -client — apt installs a binary keyring, dnf imports an armored export, apk holds -a bare RSA key by filename — so `keys attach` refuses a key whose algorithm the -format's clients cannot check. Every signature is verified before it is -published, because a repository carrying one that does not check out is worse -than an unsigned one: a client reports it as tampering. - -## Rotating a signing key - -Debian rotation keeps one active `InRelease` signer and a stable binary keyring. -Introduction publishes both identities while the old key continues signing; -activation switches to the successor only after the introducing deployment -receipt has aged through the minimum refresh window; retirement removes old -trust only after a second deployed overlap window: - -```sh -go run ./cmd/snailmail keys rotate debian \ - --successor archive-signing-2027 \ - --minimum-refresh 720h -git add snailmail.toml keys/ -git commit -m "introduce successor archive key" -go run ./cmd/snailmail plan && go run ./cmd/snailmail apply - -# After `keys audit` reports the introducing state ready: -go run ./cmd/snailmail keys rotate debian --advance --yes -git add snailmail.toml && git commit -m "activate successor archive key" -go run ./cmd/snailmail plan && go run ./cmd/snailmail apply - -# After the activated overlap is ready: -go run ./cmd/snailmail keys rotate debian --advance --yes -git add snailmail.toml && git commit -m "retire old archive key" -go run ./cmd/snailmail plan && go run ./cmd/snailmail apply -``` - -The minimum window is seven days. Its clock starts from the canonical -post-verification deployment receipt, not from the manifest edit or plan time. -Repository-hosted keyrings do not update clients automatically: refresh the -local `/usr/share/keyrings` file through configuration management or a keyring -package before activation. In CI, replace `SNAILMAIL_SIGNING_KEY_REF` and its -encrypted private-key secret with the successor before planning the activated -state; apply still receives no private key. - -## What plan requires - -`plan` requires the manifest, configured locks, publication ledgers, and -deployment receipts to be committed in a complete, non-shallow Git repository. `apply` -executes the reviewed plan without replanning, verifies staged repository -bytes, commits its exact publication ledger records with a compare-and-swap, -and publishes through the selected host's conditional release switch. After -canonical verification succeeds, apply commits a deployment receipt separately -from the pre-effect publication ledger. S3 keeps -the verified tree under an immutable digest prefix and conditionally updates a -small root index that points clients at that tree. - -## Publishing to object storage - -Object storage serves PyPI, Helm and unsigned yum repositories. Setup uses the -standard AWS credential chain; no -credential values are written to the manifest or plan: - -```sh -go run ./cmd/snailmail setup pypi \ - --name python \ - --host s3 \ - --bucket example-packages \ - --prefix python \ - --region us-east-1 \ - --base-url https://packages.example.com/python -git add snailmail.toml repos/python.lock.toml docs/install-python.md -git commit -m "configure hosted Python repository" -go run ./cmd/snailmail plan -go run ./cmd/snailmail apply -``` - -### What the credential has to be allowed to do - -Publishing needs, scoped to the configured prefix: - -- `s3:GetObject` — reading back what it wrote, to verify it -- `s3:PutObject` — writing artifacts, indexes and the root object -- `s3:DeleteObject` — removing an abandoned stage, and removing the root object - when a restore has to leave a repository with none - -It does **not** need `s3:ListBucket`. Every operation on the publishing path -addresses an object by a key snailmail already knows, which is what lets a -publication be verified without trusting a listing. - -`s3:ListBucket` is needed only to discover state a publication has superseded — -collecting old releases. That is a separate operation and may run under a separate -credential, so a publishing role stays as narrow as the list above. - -The bucket or gateway must serve the configured prefix, including -`.snailmail/stages/` during pre-publication verification. Use `--endpoint` and -`--use-path-style` for compatible object stores. All non-loopback S3 API and -package client endpoints must use HTTPS. Configure a -bucket lifecycle rule to expire abandoned `.snailmail/stages/` objects after a -grace period; immutable `.snailmail/releases/`, `.snailmail/manifests/`, and -`.snailmail/restores/` objects must not use that short-lived rule. - -A private object-storage repository uses a Basic-auth gateway and a short-lived -credential broker. The broker is a compiled executable selected at runtime by -`SNAILMAIL_CREDENTIAL_BROKER`; set `--visibility private --read-auth basic ---credential-broker default` during setup. Snailmail sends the reviewed -workspace, host, plan, change, tree, and object-prefix scope as JSON on stdin. -The helper returns `username`, `password`, and RFC3339 `expires_at` JSON, with a -maximum lifetime of 15 minutes. The helper and gateway are trusted to enforce -the supplied scope. The helper receives only selected profile, web-identity, -TLS, and `SNAILMAIL_BROKER_*` environment variables, not the complete snailmail -environment. Credentials stay out of Git, plans, URLs, and argv; pip receives -them through an isolated temporary netrc that is destroyed after verification. - -Shared S3 blob storage keeps Git locks provider-neutral while treating the -local CAS as a verified disposable cache: - -```sh -go run ./cmd/snailmail blob-store s3 \ - --bucket example-artifacts \ - --prefix snailmail/cas \ - --region us-east-1 -``` - -## Publishing to GitHub Pages - -Public GitHub Pages requires distinct pre-provisioned production and preview -sites that deploy the root of the configured branch. Authenticate `gh`, then: - -```sh -go run ./cmd/snailmail setup pypi \ - --name python \ - --host github-pages \ - --github-repo example/packages \ - --github-preview-repo example/packages-preview \ - --base-url https://example.github.io/packages \ - --preview-url https://example.github.io/packages-preview -``` - -Pages publication uses exact orphan commits, immutable stage and restore refs, -force-with-lease compare-and-swap, `.nojekyll`, and bounded propagation polling. -Private Pages repositories are rejected. - -## Review and approval gates - -For a PR gate, initialize the workspace with its reviewed state repository and -configure `--gate pr`. Apply verifies that the exact Git revision was merged by -a PR and remains reachable from that repository's default branch. - -```sh -go run ./cmd/snailmail init --name example --forge-repo example/state - -# On a forge other than GitHub, name it. The reference shape alone cannot say -# which service to ask whether a revision was reviewed, and omitting this means -# github.com. -go run ./cmd/snailmail init --name example \ - --forge gitlab --forge-repo acme/platform/state -go run ./cmd/snailmail init --name example \ - --forge gitea --forge-repo acme/state --forge-host git.acme.example -``` - -Approval gates use Ed25519 evidence. Keep the generated private key outside the -workspace; commit only the printed public key in repository configuration: - -```sh -go run ./cmd/snailmail approval-key generate --out ../snailmail-approval-key.json -go run ./cmd/snailmail setup pypi --name python --output public/python \ - --gate approval --approval-keys BASE64_PUBLIC_KEY -go run ./cmd/snailmail plan -go run ./cmd/snailmail approve --repository python \ - --key ../snailmail-approval-key.json --yes -go run ./cmd/snailmail apply -``` - -`approve` signs the exact plan ID, repository, key identity, and expiry. Apply -reauthorizes before every stage, commit, and compensating restore. - -## Status pages - -Render the read-only public matrix and machine-readable status from committed -locks, ledgers, deployment receipts, and an optional current plan: - -```sh -go run ./cmd/snailmail dashboard -``` - -## Build layout and binary size - -`plan` and `apply` build and stage under `.snailmail/stage` in the workspace -rather than in `TMPDIR`, so artifacts hard-link from the local CAS instead of -being copied and large trees are never held in a `tmpfs`. Set an absolute -`SNAILMAIL_STAGE_DIR` to stage elsewhere. - -The AWS SDK is the largest single component of the binary. Builds that only -target local directories or GitHub Pages can exclude it: - -```sh -go build -trimpath -ldflags="-s -w" -tags nos3 ./cmd/snailmail -``` - -That drops the stripped binary from about 20.6 MB to 12.8 MB. S3 hosts and S3 -blob stores then report that the build has no S3 support; every other format, -host, and command is unaffected. The default build keeps S3. - -## Exit codes - -Every command exits `0` on success. A failure exits with what to do about it, -so a CI job can retry a flaky network without also retrying a malformed bucket -name: - -| Code | Meaning | What to do | -|---|---|---| -| `1` | Failed | Read the message. | -| `2` | The configuration is wrong | Fix the workspace or the host configuration; retrying fails again. | -| `3` | The host or network failed | Retry. | -| `4` | The plan no longer matches the world | Run `plan` again, review the diff, then `apply`. | -| `5` | A publication may or may not have taken effect | **Do not retry blindly.** Check the host, then `snailmail status`. | - -Codes `3`, `4` and `5` also print a line saying the same thing, for whoever is -reading the log rather than branching on the number. - -Only failures reported by a host carry this detail; anything else exits `1`. -Codes are stable — a job that branches on them should not break under an -upgrade. - -## Container image and CI examples - -`Dockerfile` builds the runtime image. `examples/github-actions.yml` is a pinned -workflow template to copy into your own workspace repository, with read-only PR -testing and a protected default-branch apply job; this repository is not itself -a snailmail workspace yet, so the template is linted here rather than run. Configure `AWS_ROLE_ARN`/`AWS_REGION` repository variables for OIDC-backed -S3 access. Cross-repository Pages writes need `SNAILMAIL_GITHUB_TOKEN`. Approval -jobs need `SNAILMAIL_APPROVAL_REPOSITORIES` and the -`SNAILMAIL_APPROVAL_PRIVATE_KEY` secret. Private S3 requires an organization -image containing its compiled broker plus the corresponding runtime variables. -Signed Debian planning additionally uses `SNAILMAIL_SIGNING_KEY_REF` as the -non-secret `/` repository variable and the -`SNAILMAIL_SIGNING_PRIVATE_KEY` and `SNAILMAIL_KEY_PASSPHRASE` secrets. The -workflow materializes that encrypted key only in runner temporary storage and -does not pass it to apply. - -```sh -go run ./cmd/snailmail build pypi --input ./dist --output ./repository -go run ./cmd/snailmail verify pypi --repo ./repository -go run ./cmd/snailmail build deb --input ./dist --output ./apt-repository -go run ./cmd/snailmail verify deb --repo ./apt-repository -go run ./cmd/snailmail build helm --input ./dist --output ./helm-repository -go run ./cmd/snailmail verify helm --repo ./helm-repository -go run ./cmd/snailmail serve --repo ./repository -``` - -The short version: - -- **Static-first.** Index generation is a pure function returning a file tree, - so GitHub Pages, S3, or a USB stick are all valid hosting. The server is - optional and never load-bearing. -- **Git is the state.** A repository's contents are a committed lockfile; the - served index is a build artifact. Rollback is `git revert`. -- **Declarative.** One manifest says what should be published where; `plan` and - `apply` reconcile against it. -- **Scriptable.** Every command that reports a result accepts `--json`, so CI - is the CLI in a container. -- **Gated per repository.** `auto`, provider-bound merged-PR evidence, and - plan-bound Ed25519 approval evidence share the same apply graph. -- **Verified.** Apply verifies staged bytes structurally and, unless explicitly - disabled, with the ecosystem client before switching the local target. -- **Local, S3, and Pages.** `snailmail setup` records deterministic local - targets, public or private object storage, or public GitHub Pages with a - separate preview site. -- **Plan-resolved signing.** Debian signing keys remain outside Git; exact - verified `InRelease` and `Release.gpg` responses are reviewed in the plan and - replayed without signer access during apply. - -``` - apt dnf apk aur aur-bin brew nixpkgs - ttysvg 0.1.2 0.1.2 ✗ 0.1.2 0.1.2 0.1.2 0.1.2 0.0.7 ⚠ - exex 0.3.2 0.3.2 ✗ 0.3.2 0.3.2 0.3.2 0.3.2 — - cnvrt 0.0.3 0.0.3 ✗ 0.0.3 0.0.3 0.0.3 0.0.3 — - snailrace 0.0.5 0.0.5 ✗ 0.0.5 0.0.5 PR #12 0.0.5 — - - ✗ verify failing ⚠ lagging PR # gate pending -``` +[ARCHITECTURE.md](ARCHITECTURE.md) is the implementation contract and +[PLAN.md](PLAN.md) the broader design. diff --git a/docs/adopting.md b/docs/adopting.md new file mode 100644 index 0000000..6a1095f --- /dev/null +++ b/docs/adopting.md @@ -0,0 +1,130 @@ +# Taking on a repository that already exists + +Most people arrive with packages already published somewhere. These commands +record what is there, byte for byte, rather than asking anyone to start again. + +## Importing an existing repository + +Most people arrive with a repository already published somewhere. `import` reads +its index and records every artifact it names, rather than adopting each by hand: + +```sh +snailmail import --project six --public-origin --dry-run python https://pypi.org/ +snailmail import --project six --public-origin python https://pypi.org/ +``` + +Each artifact goes through the same path as `adopt`: fetched, checked against the +digest its index published, and recorded with its origin URL so it can be refetched +later. + +What a recorded digest is worth depends on where it came from — an index someone +signed is not the same as one served over TLS alone — so the lock records that +beside it, and `status --json` reports the counts: + +``` + python -> {"index-stated": 2} +``` + +`index-stated` means the index published the digest and the fetched bytes matched, +which is the strongest a simple index supports. Artifacts recorded by `adopt` read +as `operator`, including in locks written before this field existed. PLAN.md §3.8 +has the full set and what each level is worth. Anything the index names but does not publish a SHA-256 for is skipped and +reported — a locked artifact is pinned to a digest someone stated in advance, and +one computed from the bytes a download happened to return would prove only that the +download was self-consistent. + +One artifact failing does not abandon the rest, so a repository with a broken file +imports the other 47 and names the one that failed. + +PyPI and Helm today. A Helm repository has no per-project page, so importing one +imports the repository: + +```sh +snailmail import --public-origin --dry-run charts https://grafana.github.io/helm-charts +``` + +Where an index lists several mirrors of a chart, the origin recorded is the URL +that actually served. Where it lists the same name and version twice, both entries +are skipped and named — two entries claiming one identity cannot both be it, and +picking one would record bytes nobody chose. + +Debian too, which reads a suite rather than a project: + +```sh +snailmail import --public-origin --dry-run --suite bookworm apt https://deb.debian.org/debian +``` + +A Debian import walks the chain rather than trusting the leaf: `Release` states the +digest of `Packages`, and a `Packages` whose bytes disagree is refused outright — +nothing in an index that failed its own root can be trusted. That is why Debian +artifacts record `index-chain` where PyPI and Helm record `index-stated`. The +`Release` signature itself is not verified yet, so the root of trust is still the +transport, and the lock says exactly that. + +And yum, which needs no extra flags because a repository root is one repository: + +```sh +snailmail import --public-origin --dry-run rocky https://dl.rockylinux.org/pub/rocky/9/BaseOS/x86_64/os +``` + +yum walks the same chain as Debian: `repomd.xml` states the digest of +`primary.xml.gz`, and a primary whose bytes disagree is refused. So rpm artifacts +also record `index-chain`. If `repomd.xml` states only a sha1 or md5 for its +primary, the import stops rather than quietly recording a weaker provenance than +was asked for — signing `repomd.xml` is what would raise this to `signed-index`. + +Multilib is preserved: `i686` and `x86_64` builds of one name-version are two +artifacts, recorded as two blobs under the same package version rather than one +overwriting the other. + +And Alpine, which is the honest exception: + +```sh +snailmail import --public-origin --dry-run alpine https://dl-cdn.alpinelinux.org/alpine/v3.19/main/x86_64 +``` + +An `APKINDEX` entry carries `C:Q1…`, which decodes to a SHA-1 of the package's +*control section* — not of the file. Checked against Alpine's own archive: the +index states `6026787b…` for `7zip-23.01-r0.apk`, whose actual SHA-1 is +`76a96042…`. They differ because they are digests of different things. So there is +nothing in an Alpine index to pin an artifact to, and an imported apk records +`computed`: a digest of the bytes snailmail downloaded, and nothing more. + +That is allowed but never hidden. If your workspace will not accept +unauthenticated bytes, say so once: + +```sh +snailmail import --public-origin --min-provenance index-stated alpine https://… +``` + +Every artifact then reports why it was refused, rather than being pinned to +something weaker than you asked for. The floor works for any format — a Debian +import establishes `index-chain`, so it passes an `index-stated` floor. + +## Adopting an artifact from a URL + +`snailmail adopt --sha256 HEX --public-origin REPOSITORY URL` records one +explicitly selected artifact in an existing owned repository. The lowercase +SHA-256 pin is mandatory; `--public-origin` confirms that the complete requested +URL is non-secret and may be committed and printed. Lock schema 2 retains that +URL, plans display every visible adopted acquisition, and `--dry-run` validates +without changing CAS or lock state. An adopted artifact is streamed to disk rather +than held in memory, so its size costs disk and time rather than resident memory; +`SNAILMAIL_MAX_ARTIFACT_BYTES` raises the 2 GiB ceiling that remains. Adoption +requires the local blob store and does not claim authorship, build provenance, +source signatures, or historical snailmail publication. + +## Inspecting someone else's repository + +`snailmail doctor URL` needs no workspace and inspects a public HTTPS PyPI, +Debian, or Helm repository. It parses bounded native indexes, follows at most the +configured artifact limit, and checks referenced availability, size, SHA-256, +archive validity, and package identity. Use `--project` for PyPI artifacts and +`--suite` for a Debian base URL. Debian Release signatures and Helm provenance +are explicitly reported as unverified in this initial slice. Runs inspect at +most four artifacts, cap each expanded archive at 64 MiB, and stop after two +minutes. + +--- + +Back to [snailmail](../README.md). diff --git a/docs/curating.md b/docs/curating.md new file mode 100644 index 0000000..329bc2d --- /dev/null +++ b/docs/curating.md @@ -0,0 +1,160 @@ +# Deciding what a repository publishes + +A repository publishes what its lock says is placed on the configured track. +Promoting, yanking and pruning edit those placements; collecting and rolling back +act on what has already been published. + +## Promotions and yanks + +Promotions and yanks edit only placement records. Package versions, blob +bindings, and publication history remain available for later re-promotion: + +```sh +go run ./cmd/snailmail promote --track testing python snail-demo 1.2.3 +go run ./cmd/snailmail yank --track stable python snail-demo 1.2.3 +# Or remove every placement for the exact version: +go run ./cmd/snailmail yank --all python snail-demo 1.2.3 + +git diff -- repos/python.lock.toml +git add repos/python.lock.toml +git commit -m "update Python package placements" +go run ./cmd/snailmail plan +go run ./cmd/snailmail apply +``` + +The repository and package version are exact; promotion does not copy package +versions between repositories. Each repository renders only its configured +`--track` (default `stable`), and Debian additionally renders only placements for +its configured suite. Other placements remain recorded but are not exposed in +that view. Removing the final visible placement publishes a valid empty index +while retaining immutable package and blob records in Git. Debian defaults a +new placement's distro to the configured suite; `--distro DISTRO` selects another +distro coordinate explicitly. + +Retention pruning removes only older placements, independently per package, +track, and distro, using native PEP 440, Debian, or SemVer precedence: + +```sh +go run ./cmd/snailmail prune python --keep 5 +git add repos/python.lock.toml +git commit -m "prune old Python placements" +go run ./cmd/snailmail plan +go run ./cmd/snailmail apply +``` + +Versions tied at the retention boundary are kept together. Prune does not remove +package-version records, CAS objects, remote blobs, or publication history; +physical blob GC remains a separate future operation with tombstones and a grace +period. + +## Raw repositories + +`raw` repositories publish artifacts that carry no ecosystem metadata: release +tarballs, static binaries, installers. Every other format reads a package name +and version out of the bytes; raw cannot, so identity comes from the filename by +convention, or from you when the convention does not apply. + +```sh +go run ./cmd/snailmail setup raw --name tools --output public/tools + +# _[_][_]. is read without flags. +go run ./cmd/snailmail add tools ./dist/ttysvg_0.1.2_linux_amd64.tar.gz + +# Anything else needs identity supplied, including a name with an underscore, +# which cannot be told apart from an extra field. +go run ./cmd/snailmail add --name ttysvg --version 0.2.0 tools ./dist/build-final.bin + +go run ./cmd/snailmail plan && go run ./cmd/snailmail apply +go run ./cmd/snailmail verify raw --repo public/tools +``` + +Artifacts are published at `//` beside a generated +`SHA256SUMS` and `index.html`. Identity lives in the path on purpose: because +it may have come from a flag rather than from the bytes, the published tree has +to record it somewhere a later reader can check without knowing what was typed. +`SHA256SUMS` is the interchange format — `sha256sum -c` verifies a raw +repository with no snailmail present. Raw has no signing: it is not an +ecosystem, so no client knows to check a signature. + +## Collecting superseded releases + +An object store keeps every revision it has ever published: a publication writes a +whole tree under `.snailmail/releases//`, and nothing removes the previous +one. A project publishing daily accumulates a copy a day. + +```sh +snailmail collect # report what would be removed +snailmail collect --keep 3 --yes # remove it, retaining three recent revisions +``` + +Reporting is the default; `--yes` deletes. Three things always survive whatever +`--keep` says: the live revision, whatever its rollback depends on, and any +revision the ledger records within `--keep`. The first two are established by +reading the host rather than the workspace, so a checkout whose ledger is behind +cannot delete what is being served. + +Note that with only two publications nothing is collectable — both are protected, +the live one and the one it rolls back to. Local directories and GitHub Pages have +nothing to collect and say so: the first leaves nothing behind, and the second +leaves unreachable objects to git. + +Collection is the one operation that needs `s3:ListBucket`, so it may run under a +different credential from publishing. + +Two things accumulate, and they are collected together. Release directories are +one; the other only exists for helm and unsigned rpm, which write their artifacts +at the paths clients fetch them from rather than inside a release directory — +because `index.yaml` and `repomd.xml` name those paths, and rewriting a signed rpm +would invalidate it. A chart dropped from the workspace a year ago is still a +billable object that no release collection has ever touched. `collect` now removes +those too, keeping every file that any surviving revision publishes. + +Two revisions are always protected: the live one and the one its restore rolls +back to. So the first collectable revision is the live one's grandparent, which is +worth knowing before reading a `--dry-run` and concluding it reclaims less than +expected. If a surviving revision's release descriptor cannot be read, the whole +collection is refused rather than run against an unknown reference set — deleting +too little costs storage, deleting too much costs the repository. + +How much a repository keeps is part of its configuration, not a flag someone +remembers: + +```sh +snailmail setup deb --name apt --keep 20 ... +``` + +`collect` uses that, and `--keep N` overrides it for one run. The order matters: +collection is the only operation here that deletes published bytes, so the policy +belongs in the reviewed manifest where changing it is a diff, exactly like changing +a gate or a signing key. A repository that declares nothing keeps the default of +10, so nothing configured before this behaves differently. + +## Undoing a publication + +When a publication succeeded and turned out to be wrong: + +```sh +snailmail rollback apt # report what would be restored +snailmail rollback apt --yes # restore the previous publication +``` + +**One step, deliberately.** A published revision carries a reference to the root it +replaced, and collection protects that target for exactly this reason — so going +back one publication is something the host can still verify. Going back further is +not offered: nothing keeps a chain of restore references, and the older releases +may have been collected. A rollback whose target cannot be checked is not a +rollback. + +**Not every host can do it.** A local directory and an ssh host both decline, +because neither can establish that the release it would point back at is intact. +There the answer is to revert the lock in git and publish forward, and `rollback` +says so rather than failing obscurely. + +**Your lock still describes what you rolled back from.** That is deliberate — the +rollback changed what is served, not what your workspace wants — but it means the +next `apply` will republish what you just undid unless you revert or amend the lock +first. The command says this every time it runs. + +--- + +Back to [snailmail](../README.md). diff --git a/docs/deploying.md b/docs/deploying.md new file mode 100644 index 0000000..9b72004 --- /dev/null +++ b/docs/deploying.md @@ -0,0 +1,83 @@ +# Shipping and running snailmail itself + +The container image, the CI pipelines it generates, and what the binary costs. + +## Container image and CI examples + +`Dockerfile` builds the runtime image. `examples/github-actions.yml` is a pinned +workflow template to copy into your own workspace repository, with read-only PR +testing and a protected default-branch apply job; this repository is not itself +a snailmail workspace yet, so the template is linted here rather than run. Configure `AWS_ROLE_ARN`/`AWS_REGION` repository variables for OIDC-backed +S3 access. Cross-repository Pages writes need `SNAILMAIL_GITHUB_TOKEN`. Approval +jobs need `SNAILMAIL_APPROVAL_REPOSITORIES` and the +`SNAILMAIL_APPROVAL_PRIVATE_KEY` secret. Private S3 requires an organization +image containing its compiled broker plus the corresponding runtime variables. +Signed Debian planning additionally uses `SNAILMAIL_SIGNING_KEY_REF` as the +non-secret `/` repository variable and the +`SNAILMAIL_SIGNING_PRIVATE_KEY` and `SNAILMAIL_KEY_PASSPHRASE` secrets. The +workflow materializes that encrypted key only in runner temporary storage and +does not pass it to apply. + +```sh +go run ./cmd/snailmail build pypi --input ./dist --output ./repository +go run ./cmd/snailmail verify pypi --repo ./repository +go run ./cmd/snailmail build deb --input ./dist --output ./apt-repository +go run ./cmd/snailmail verify deb --repo ./apt-repository +go run ./cmd/snailmail build helm --input ./dist --output ./helm-repository +go run ./cmd/snailmail verify helm --repo ./helm-repository +go run ./cmd/snailmail serve --repo ./repository +``` + +The short version: + +- **Static-first.** Index generation is a pure function returning a file tree, + so GitHub Pages, S3, or a USB stick are all valid hosting. The server is + optional and never load-bearing. +- **Git is the state.** A repository's contents are a committed lockfile; the + served index is a build artifact. Rollback is `git revert`. +- **Declarative.** One manifest says what should be published where; `plan` and + `apply` reconcile against it. +- **Scriptable.** Every command that reports a result accepts `--json`, so CI + is the CLI in a container. +- **Gated per repository.** `auto`, provider-bound merged-PR evidence, and + plan-bound Ed25519 approval evidence share the same apply graph. +- **Verified.** Apply verifies staged bytes structurally and, unless explicitly + disabled, with the ecosystem client before switching the local target. +- **Local, S3, and Pages.** `snailmail setup` records deterministic local + targets, public or private object storage, or public GitHub Pages with a + separate preview site. +- **Plan-resolved signing.** Debian signing keys remain outside Git; exact + verified `InRelease` and `Release.gpg` responses are reviewed in the plan and + replayed without signer access during apply. + +``` + apt dnf apk aur aur-bin brew nixpkgs + ttysvg 0.1.2 0.1.2 ✗ 0.1.2 0.1.2 0.1.2 0.1.2 0.0.7 ⚠ + exex 0.3.2 0.3.2 ✗ 0.3.2 0.3.2 0.3.2 0.3.2 — + cnvrt 0.0.3 0.0.3 ✗ 0.0.3 0.0.3 0.0.3 0.0.3 — + snailrace 0.0.5 0.0.5 ✗ 0.0.5 0.0.5 PR #12 0.0.5 — + + ✗ verify failing ⚠ lagging PR # gate pending +``` + +## Build layout and binary size + +`plan` and `apply` build and stage under `.snailmail/stage` in the workspace +rather than in `TMPDIR`, so artifacts hard-link from the local CAS instead of +being copied and large trees are never held in a `tmpfs`. Set an absolute +`SNAILMAIL_STAGE_DIR` to stage elsewhere. + +The AWS SDK is the largest single component of the binary. Builds that only +target local directories or GitHub Pages can exclude it: + +```sh +go build -trimpath -ldflags="-s -w" -tags nos3 ./cmd/snailmail +``` + +That drops the stripped binary from about 20.6 MB to 12.8 MB. S3 hosts and S3 +blob stores then report that the build has no S3 support; every other format, +host, and command is unaffected. The default build keeps S3. + +--- + +Back to [snailmail](../README.md). diff --git a/docs/hosts.md b/docs/hosts.md new file mode 100644 index 0000000..5145dfe --- /dev/null +++ b/docs/hosts.md @@ -0,0 +1,174 @@ +# Where repositories are published + +Every host publishes the same built tree; what differs is how a revision becomes +live, and that decides which formats each can serve. The table in the +[README](../README.md#what-runs-where) is the summary; this is the detail. + +## Publishing over ssh + +The `rsync` host publishes to a directory on a machine you reach over ssh — the +shape most people already have, where a web server serves a filesystem path. + +```sh +snailmail setup rpm --name yum --host rsync \ + --target deploy@packages.example \ + --output /srv/www/yum \ + --base-url https://packages.example/yum +``` + +`--output` is the absolute path on the far side, where for a local host it is +workspace-relative. `--target` is the ssh destination; port, key and jump host +belong in `ssh_config`. + +**This is the only host that serves every format**, including a signed yum +repository. A revision goes live by renaming one symlink, so the whole tree +switches at once and the number of files that must change together does not +matter. An object store commits one object and no more, which is why it cannot +serve Debian, Alpine, or signed yum at all. + +Two POSIX primitives carry the guarantees, neither of them from the transport: + +- **`rename(2)` makes a publication atomic.** The published path is a symlink into + `.snailmail/releases//`, and a revision goes live when a new symlink is + renamed over the old one. A client follows one whole revision or the other, + never a half-copied tree. The tree is copied to a staging path and renamed into + its release first, so the symlink never points at a partial release. +- **`mkdir(2)` makes it conditional.** There is no compare-and-swap on a symlink, + so the expected revision is checked and the swap performed while holding a lock + directory whose creation fails if it already exists. Two runners publishing at + once means one fails, not one waits — the same answer every other host gives. + +`snailmail collect` works here too. Every revision leaves a tree under +`.snailmail/releases/`, and the swap that makes a new one live does not remove the +one it replaced. Collection keeps the live revision — read from the far side, not +trusted from the workspace — and whatever retention names. Unlike object storage +there is no restore target held back, because rollback is not offered, so an older +revision survives only if you ask for it. + +Requirements and limits, stated because they are not checked from this side: + +- The published path and its release directory must be on **one filesystem**, + since `rename` is not atomic across mount points. +- ssh options — port, key, jump host — belong in `ssh_config`. A second place to + configure ssh is a second place for it to be wrong. +- A path that already exists and was not published by snailmail is **refused** + rather than replaced, so a publication cannot unpublish somebody else's files. +- **Rollback is not offered.** Pointing the symlink at an earlier release is easy; + establishing that the release is still intact is not, because nothing on the far + side verifies it. The adapter declines rather than claiming a rollback it cannot + check. +- **No preview site**, so the `pr` and `approval` gates are refused. Nothing is + copied to the far side before the commit, so there is no URL a reviewer could + install from. Under the `auto` gate a real client still installs from the exact + staged bytes before they are published; what goes unchecked is that the far side + serves them correctly, which is what a preview buys. + +## Publishing to object storage + +Object storage serves PyPI, Helm and unsigned yum repositories. Setup uses the +standard AWS credential chain; no +credential values are written to the manifest or plan: + +```sh +go run ./cmd/snailmail setup pypi \ + --name python \ + --host s3 \ + --bucket example-packages \ + --prefix python \ + --region us-east-1 \ + --base-url https://packages.example.com/python +git add snailmail.toml repos/python.lock.toml docs/install-python.md +git commit -m "configure hosted Python repository" +go run ./cmd/snailmail plan +go run ./cmd/snailmail apply +``` + +### What the credential has to be allowed to do + +Publishing needs, scoped to the configured prefix: + +- `s3:GetObject` — reading back what it wrote, to verify it +- `s3:PutObject` — writing artifacts, indexes and the root object +- `s3:DeleteObject` — removing an abandoned stage, and removing the root object + when a restore has to leave a repository with none + +It does **not** need `s3:ListBucket`. Every operation on the publishing path +addresses an object by a key snailmail already knows, which is what lets a +publication be verified without trusting a listing. + +`s3:ListBucket` is needed only to discover state a publication has superseded — +collecting old releases. That is a separate operation and may run under a separate +credential, so a publishing role stays as narrow as the list above. + +The bucket or gateway must serve the configured prefix, including +`.snailmail/stages/` during pre-publication verification. Use `--endpoint` and +`--use-path-style` for compatible object stores. All non-loopback S3 API and +package client endpoints must use HTTPS. Configure a +bucket lifecycle rule to expire abandoned `.snailmail/stages/` objects after a +grace period; immutable `.snailmail/releases/`, `.snailmail/manifests/`, and +`.snailmail/restores/` objects must not use that short-lived rule. + +A private object-storage repository uses a Basic-auth gateway and a short-lived +credential broker. The broker is a compiled executable selected at runtime by +`SNAILMAIL_CREDENTIAL_BROKER`; set `--visibility private --read-auth basic +--credential-broker default` during setup. Snailmail sends the reviewed +workspace, host, plan, change, tree, and object-prefix scope as JSON on stdin. +The helper returns `username`, `password`, and RFC3339 `expires_at` JSON, with a +maximum lifetime of 15 minutes. The helper and gateway are trusted to enforce +the supplied scope. The helper receives only selected profile, web-identity, +TLS, and `SNAILMAIL_BROKER_*` environment variables, not the complete snailmail +environment. Credentials stay out of Git, plans, URLs, and argv; pip receives +them through an isolated temporary netrc that is destroyed after verification. + +Shared S3 blob storage keeps Git locks provider-neutral while treating the +local CAS as a verified disposable cache: + +```sh +go run ./cmd/snailmail blob-store s3 \ + --bucket example-artifacts \ + --prefix snailmail/cas \ + --region us-east-1 +``` + +## Publishing to GitHub Pages + +Public GitHub Pages requires distinct pre-provisioned production and preview +sites that deploy the root of the configured branch. Authenticate `gh`, then: + +```sh +go run ./cmd/snailmail setup pypi \ + --name python \ + --host github-pages \ + --github-repo example/packages \ + --github-preview-repo example/packages-preview \ + --base-url https://example.github.io/packages \ + --preview-url https://example.github.io/packages-preview +``` + +Pages publication uses exact orphan commits, immutable stage and restore refs, +force-with-lease compare-and-swap, `.nojekyll`, and bounded propagation polling. +Private Pages repositories are rejected. + +## Browsing a bucket-hosted repository + +A repository published to object storage serves `index.html` at its root, so +`https://packages.example/apt/` opens in a browser and shows what is published, +how to install it, and each artifact's digest. + +That page is a **convenience copy and is not covered by the publication's +guarantees**. Nothing verifies it, `observe` does not read it, and a rollback does +not restore it — it is refreshed by the next publication. The verified copy lives +inside the release directory, because it is regenerated for every revision and +writing it canonically would leave the previous revision unverifiable after a +rollback. Clients are unaffected either way: they read `simple/`, `index.yaml`, +`repodata/` or `SHA256SUMS`, and those are the complete, verified answer. + +The page shows the 500 most recently published artifacts and says so when there +are more. A rendered row costs about 610 bytes, so a Debian suite of 63,440 +artifacts would otherwise be a 38 MB page rebuilt and re-uploaded on every +publication. Set it with `Window` if you want a different size; the footer always +reports the repository's true total. + +--- + +Back to [snailmail](../README.md). diff --git a/docs/operating.md b/docs/operating.md new file mode 100644 index 0000000..166cb88 --- /dev/null +++ b/docs/operating.md @@ -0,0 +1,196 @@ +# Running it + +What to look at when something is wrong, what each command needs before it will +act, and where a workspace stops being comfortable. + +## Inspecting a workspace + +`snailmail status` also reports size: each repository's lock in bytes, the count +and total size of the distinct artifacts it binds, and the workspace's lock total +with the largest repository named. The lock is parsed whole on every plan and +every apply, so its size is the number that predicts where a workspace stops +being comfortable — and which repository to split when it does. + +`snailmail status` reports committed workspace evidence without contacting +hosts or providers. Human output summarizes visible and retained versions, +visible publication-binding completeness, and whether a managed deployment +receipt is recorded; `--json` emits the same deterministic schema for +automation. A receipt is evidence of a prior successful apply, not proof that a +host currently serves those bytes. + +`snailmail rollout` answers when each version reached a client, derived from the +publication ledger rather than stored anywhere. The ledger already records one +append-only entry per publication, so the date is read back rather than kept a +second time. A version is listed with the date it was first published, the +number of published trees that have carried it, and whether the repository still +serves it; `--withdrawn` includes versions that were published and later yanked +or pruned, because publication is immutable even when the offer is not. + +`snailmail ci github` and `snailmail ci gitlab` emit a pipeline that publishes +this workspace, to stdout rather than to a file: it carries decisions snailmail +cannot make — which registry to pull verification images through, which secret +names a project uses — and a file the tool owned would be rewritten over an +operator's edits. What it does derive is what the workspace already says: which +signing keys need materialising, whether a foreign architecture needs emulation, +and which repositories publish into a directory something else must serve. Both +providers derive the same facts; only the rendering differs, and it differs more +than wording. A GitLab runner has no Docker daemon, so client verification needs +one as a service; its jobs share no filesystem, so what apply builds is declared +as an artifact; and Pages there serves a job artifact rather than a moved ref, so +there is no orphan commit. + +## Auditing what is published + +`snailmail check` is a read-only integrity audit of every retained package +version, including yanked and pruned versions. It verifies local CAS bytes or +fetches the configured S3 authority into temporary storage, reparses native +package facts, and revalidates historical publication bindings. Upstream release +discovery remains unavailable until releases are modeled. `check --origins` +re-fetches explicitly adopted URLs and compares their pinned bytes; default +checks remain offline from external sources. Origin checks process at most four +sorted records per run; `--origin-offset` selects subsequent batches. + +## What plan requires + +`plan` requires the manifest, configured locks, publication ledgers, and +deployment receipts to be committed in a complete, non-shallow Git repository. `apply` +executes the reviewed plan without replanning, verifies staged repository +bytes, commits its exact publication ledger records with a compare-and-swap, +and publishes through the selected host's conditional release switch. After +canonical verification succeeds, apply commits a deployment receipt separately +from the pre-effect publication ledger. S3 keeps +the verified tree under an immutable digest prefix and conditionally updates a +small root index that points clients at that tree. + +## Exit codes + +Every command exits `0` on success. A failure exits with what to do about it, +so a CI job can retry a flaky network without also retrying a malformed bucket +name: + +| Code | Meaning | What to do | +|---|---|---| +| `1` | Failed | Read the message. | +| `2` | The configuration is wrong | Fix the workspace or the host configuration; retrying fails again. | +| `3` | The host or network failed | Retry. | +| `4` | The plan no longer matches the world | Run `plan` again, review the diff, then `apply`. | +| `5` | A publication may or may not have taken effect | **Do not retry blindly.** Check the host, then `snailmail status`. | + +Codes `3`, `4` and `5` also print a line saying the same thing, for whoever is +reading the log rather than branching on the number. + +Only failures reported by a host carry this detail; anything else exits `1`. +Codes are stable — a job that branches on them should not break under an +upgrade. + +## Status pages + +Render the read-only public matrix and machine-readable status from committed +locks, ledgers, deployment receipts, and an optional current plan: + +```sh +go run ./cmd/snailmail dashboard +``` + +## Two runners at once + +The workspace lock is a file inside the checkout, so it serialises operations on +one machine and not two. What protects a repository when two CI runners publish +at the same time is the conditional commit: each states the revision it observed, +and the second to arrive is refused because that is no longer what is live. + +So the failure mode is **one run fails, not one run waits**. It fails as a stale +plan, which is a retry signal — replanning against what is actually live and +applying again succeeds. A pipeline that publishes from more than one place at once +wants a concurrency group so this is rare rather than routine; the generated +workflows set one. + +Two things worth knowing. Staging is *not* exclusive, deliberately: two runners can +both stage, because a stage writes only under its own identifier and touches nothing +a client reads. And a local host cannot be shared at all — its output path is +relative to its workspace — so this only arises for object storage and Pages, where +two runners can name the same bucket and prefix or the same repository and branch. + +## One workspace, or one per team + +Use one workspace for the whole organisation unless you have a reason not to. + +It sounds like fifty teams would contend on one review queue and one lock, and +they do not. Each repository has its own lock file, so two teams publishing to two +repositories touch two different files and git merges them; `snailmail.toml` is the +only shared file, and it changes when a repository is configured rather than when +one is published to. Point CODEOWNERS at `repos/.lock.toml` and each team +reviews its own publications. The workspace lock covers a single checkout, so CI +runners with their own checkouts never wait on each other — what orders two +concurrent publications is the host refusing the second, per repository. + +Split when one of these is true: + +- **A team needs its own blob store.** A plan is bound to one, so a workspace has + exactly one. +- **A team cannot share read access** to the state repository, because a workspace + is a git repository and access to it is all-or-nothing. + +Separate workspaces can still publish into the same bucket under different +prefixes. What you lose is the shared view: `snailmail site` describes one +workspace, so splitting means several index pages and no single answer to where a +package is published. + +## How large a workspace can get + +A repository lock is parsed whole on every plan, apply, status and check, so its +size is what decides how long those take and how much memory they need. Measured +at roughly 385 bytes per package-version, with parsing costing about one and a half +times the file in heap: + +| package-versions | lock file | parse | heap | +|---|---|---|---| +| 2,000 | 0.8 MB | 5 ms | 1 MB | +| 20,000 | 7.7 MB | 44 ms | 12 MB | +| 100,000 | 38.5 MB | 226 ms | 59 MB | + +snailmail refuses a lock over **128 MiB** — about 330,000 package-versions — and +says so, naming the repository, the size and what to do: + +``` +repository "repos/apt.lock.toml" has a 200 MiB lock, over the 128 MiB limit, and +parsing it needs roughly 300 MiB of memory on every plan and apply: prune retained +versions, split the repository, or set SNAILMAIL_MAX_LOCK_BYTES to proceed anyway +``` + +That is the honest state of things: one lock per repository is the current design, +and a workspace past this size wants the lock sharded per package rather than a +larger limit. The limit exists so that a workspace which has outgrown the design +finds out in a sentence instead of getting slower until something is killed. +`snailmail status` reports every lock's size, so the number can be watched before +it becomes a refusal. + +Artifacts themselves are not held in memory — they stream from content-addressed +storage — so the ceiling is index and lock size, not repository size. Generated +indexes are held whole, which for Debian and yum is the binding constraint at +around a million package-versions. + +Past 2,000 package-versions a repository's lock is written as one file per +package, under a root that indexes them and a Merkle digest over the set: + +``` +repos/apt.lock.toml the root: schema, placement, shard index, Merkle root +repos/apt.lock.d/3f/… one file per package +``` + +Nothing about a lock's meaning changes — it still has a single identity that a +plan pins, and a shard edited on its own is refused by name rather than accepted +quietly. What changes is what a publication costs. Adding one version rewrites one +small file instead of the whole lock, so a reviewer sees the package that changed +rather than a multi-megabyte diff. Measured at 10,000 packages: adding a version +takes 108 ms and reading the lock 102 ms, against 3.4 s to write when every shard +is compared and 950 ms to read them one at a time. + +The transition happens once, automatically, when a repository grows past the +threshold, and a lock never goes back to one file — returning would itself be a +diff rewriting everything. The first sharded write of a large repository creates +every file and takes seconds; subsequent ones do not. + +--- + +Back to [snailmail](../README.md). diff --git a/docs/signing.md b/docs/signing.md new file mode 100644 index 0000000..9dee984 --- /dev/null +++ b/docs/signing.md @@ -0,0 +1,128 @@ +# Signing and review + +What makes a repository trustworthy to the client installing from it, and what +makes a change to it reviewable by a person before it takes effect. + +## Signing keys + +Signed Debian repositories use an encrypted private key outside the workspace +and commit only canonical public forms. Set a passphrase through the environment +(never an argument), generate the key, and reference it during setup: + +```sh +export SNAILMAIL_KEY_PASSPHRASE='use-a-secret-manager-value' +go run ./cmd/snailmail keys new archive-signing --expires-in 17520h +go run ./cmd/snailmail setup deb \ + --name debian \ + --output public/debian \ + --suite stable \ + --architectures amd64 \ + --signing-key archive-signing +go run ./cmd/snailmail keys audit +git add snailmail.toml keys/ repos/debian.lock.toml +git commit -m "configure signed Debian repository" +go run ./cmd/snailmail plan +go run ./cmd/snailmail apply +``` + +The file backend stores encrypted private keys under +`$XDG_DATA_HOME/snailmail/private-keys` (or +`~/.local/share/snailmail/private-keys`) with mode `0600`. Passphrases must +contain at least 24 bytes and should come from a secret manager. In a container, +prefer `SNAILMAIL_KEY_PASSPHRASE_FILE` pointing at a mounted file: a container's +environment stays readable through the runtime's inspection API for as long as +the container exists, whereas a file can be read once and removed. Setting both +variables is rejected rather than resolved silently. `keys publish` +recreates missing public forms after validating the private identity. Planning +compiles content-addressed `InRelease` and `Release.gpg` signing nodes over the +exact Debian `Release` bytes, signs each node twice to prove deterministic +backend behavior, verifies both signatures, and embeds only public node +responses in the reviewed plan. Apply does not load the private key. Signed +client verification uses apt's `signed-by`; migrated unsigned Debian +repositories remain readable but `keys audit` reports them as errors. New +unsigned Debian setup requires the explicit `--allow-unsigned` compatibility +opt-out. + +Every format the knowledge bundle records as signable is signed, each in the +form its own clients read: Debian publishes `InRelease` and `Release.gpg`, yum an +armored key beside a detached `repomd.xml.asc`, Alpine one signature per +architecture index under the key filename its index names, and Helm a `.prov` +per chart beside the archive it covers. The published key differs with the +client — apt installs a binary keyring, dnf imports an armored export, apk holds +a bare RSA key by filename — so `keys attach` refuses a key whose algorithm the +format's clients cannot check. Every signature is verified before it is +published, because a repository carrying one that does not check out is worse +than an unsigned one: a client reports it as tampering. + +## Rotating a signing key + +Debian rotation keeps one active `InRelease` signer and a stable binary keyring. +Introduction publishes both identities while the old key continues signing; +activation switches to the successor only after the introducing deployment +receipt has aged through the minimum refresh window; retirement removes old +trust only after a second deployed overlap window: + +```sh +go run ./cmd/snailmail keys rotate debian \ + --successor archive-signing-2027 \ + --minimum-refresh 720h +git add snailmail.toml keys/ +git commit -m "introduce successor archive key" +go run ./cmd/snailmail plan && go run ./cmd/snailmail apply + +# After `keys audit` reports the introducing state ready: +go run ./cmd/snailmail keys rotate debian --advance --yes +git add snailmail.toml && git commit -m "activate successor archive key" +go run ./cmd/snailmail plan && go run ./cmd/snailmail apply + +# After the activated overlap is ready: +go run ./cmd/snailmail keys rotate debian --advance --yes +git add snailmail.toml && git commit -m "retire old archive key" +go run ./cmd/snailmail plan && go run ./cmd/snailmail apply +``` + +The minimum window is seven days. Its clock starts from the canonical +post-verification deployment receipt, not from the manifest edit or plan time. +Repository-hosted keyrings do not update clients automatically: refresh the +local `/usr/share/keyrings` file through configuration management or a keyring +package before activation. In CI, replace `SNAILMAIL_SIGNING_KEY_REF` and its +encrypted private-key secret with the successor before planning the activated +state; apply still receives no private key. + +## Review and approval gates + +For a PR gate, initialize the workspace with its reviewed state repository and +configure `--gate pr`. Apply verifies that the exact Git revision was merged by +a PR and remains reachable from that repository's default branch. + +```sh +go run ./cmd/snailmail init --name example --forge-repo example/state + +# On a forge other than GitHub, name it. The reference shape alone cannot say +# which service to ask whether a revision was reviewed, and omitting this means +# github.com. +go run ./cmd/snailmail init --name example \ + --forge gitlab --forge-repo acme/platform/state +go run ./cmd/snailmail init --name example \ + --forge gitea --forge-repo acme/state --forge-host git.acme.example +``` + +Approval gates use Ed25519 evidence. Keep the generated private key outside the +workspace; commit only the printed public key in repository configuration: + +```sh +go run ./cmd/snailmail approval-key generate --out ../snailmail-approval-key.json +go run ./cmd/snailmail setup pypi --name python --output public/python \ + --gate approval --approval-keys BASE64_PUBLIC_KEY +go run ./cmd/snailmail plan +go run ./cmd/snailmail approve --repository python \ + --key ../snailmail-approval-key.json --yes +go run ./cmd/snailmail apply +``` + +`approve` signs the exact plan ID, repository, key identity, and expiry. Apply +reauthorizes before every stage, commit, and compensating restore. + +--- + +Back to [snailmail](../README.md).