From f3ebd6fdde02f79e4ef1b59d9e71d60d6b12454a Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:49:05 +0000 Subject: [PATCH 01/21] Add imagepush manager for outbound image push jobs Orchestrates pushing cached images to remote registries: resolves a hypeman image to its cached digest, runs push jobs on a bounded queue with digest+target deduplication, persists job status to disk with FIFO recovery across restarts, and exposes in-flight digests so the OCI cache GC can keep required blobs alive mid-push. --- lib/imagepush/imagepush.go | 82 ++++++ lib/imagepush/manager.go | 310 ++++++++++++++++++++ lib/imagepush/manager_test.go | 520 ++++++++++++++++++++++++++++++++++ lib/imagepush/queue.go | 106 +++++++ lib/imagepush/queue_test.go | 122 ++++++++ lib/imagepush/storage.go | 136 +++++++++ lib/imagepush/storage_test.go | 94 ++++++ lib/paths/paths.go | 17 ++ 8 files changed, 1387 insertions(+) create mode 100644 lib/imagepush/imagepush.go create mode 100644 lib/imagepush/manager.go create mode 100644 lib/imagepush/manager_test.go create mode 100644 lib/imagepush/queue.go create mode 100644 lib/imagepush/queue_test.go create mode 100644 lib/imagepush/storage.go create mode 100644 lib/imagepush/storage_test.go diff --git a/lib/imagepush/imagepush.go b/lib/imagepush/imagepush.go new file mode 100644 index 00000000..366a7b90 --- /dev/null +++ b/lib/imagepush/imagepush.go @@ -0,0 +1,82 @@ +// Package imagepush manages outbound image pushes: exporting images from +// hypeman's OCI cache to remote container registries. +// +// A push job resolves a hypeman image to its cached manifest digest, then +// streams the cached blobs to the target reference through lib/registrypush. +// Jobs run on a bounded queue, persist their status to disk, and expose +// in-flight digests so the OCI cache GC keeps the required blobs alive. +package imagepush + +import ( + "context" + "errors" + "time" + + "github.com/kernel/hypeman/lib/images" +) + +const ( + StatusQueued = "queued" + StatusPushing = "pushing" + StatusPushed = "pushed" + StatusFailed = "failed" +) + +var ( + ErrNotFound = errors.New("push not found") + ErrImageNotReady = errors.New("image not ready for push") + ErrInvalidTarget = errors.New("invalid push target") +) + +// PushRequest describes a request to push a hypeman image to a remote registry. +type PushRequest struct { + // Image is the hypeman image name (tag or digest form). + Image string + // Target is the full remote reference, e.g. "registry.example.com/app:v1". + Target string + // Insecure allows pushing to plain-HTTP registries. + Insecure bool +} + +// Push is the state of one push job. +type Push struct { + ID string + Image string + Digest string + Target string + Status string + QueuePosition *int + Error *string + Layers int + Bytes int64 + CreatedAt time.Time + CompletedAt *time.Time +} + +// ImageResolver resolves a hypeman image name to its stored state. +type ImageResolver interface { + GetImage(ctx context.Context, name string) (*images.Image, error) +} + +// StatusEvent represents a terminal status change for push notifications. +type StatusEvent struct { + Status string + Err error +} + +// Manager orchestrates push jobs. +type Manager interface { + // CreatePush validates the request, persists a queued job, and enqueues it. + // A request that matches an in-flight job (same digest and target) returns + // the existing job instead of creating a duplicate. + CreatePush(ctx context.Context, req PushRequest) (*Push, error) + GetPush(ctx context.Context, id string) (*Push, error) + // ListPushes returns all pushes, newest first. + ListPushes(ctx context.Context) ([]Push, error) + // WaitForPush blocks until the push reaches a terminal state (pushed or + // failed) or the context is cancelled. + WaitForPush(ctx context.Context, id string) error + // InProgressDigests returns the manifest digests of queued and pushing + // jobs so the OCI cache GC can keep their blobs alive mid-push. + InProgressDigests() []string +} diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go new file mode 100644 index 00000000..bc70e986 --- /dev/null +++ b/lib/imagepush/manager.go @@ -0,0 +1,310 @@ +package imagepush + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registrypush" + "github.com/nrednav/cuid2" +) + +type inflightPush struct { + id string + digest string +} + +type manager struct { + paths *paths.Paths + resolver ImageResolver + provider registrypush.Provider + queue *pushQueue + + mu sync.Mutex + inflight map[string]inflightPush // key = pushKey(digest, target) + + subscriberMu sync.RWMutex + subscribers map[string][]chan StatusEvent // keyed by push ID +} + +// NewManager creates a push manager. provider may be nil, in which case +// credentials resolve from the default Docker keychain. Interrupted pushes +// from a previous run are re-enqueued FIFO. +func NewManager(p *paths.Paths, resolver ImageResolver, provider registrypush.Provider, maxConcurrent int) (Manager, error) { + if resolver == nil { + return nil, fmt.Errorf("image resolver is required") + } + if provider == nil { + provider = ®istrypush.KeychainProvider{} + } + + m := &manager{ + paths: p, + resolver: resolver, + provider: provider, + queue: newPushQueue(maxConcurrent), + inflight: make(map[string]inflightPush), + subscribers: make(map[string][]chan StatusEvent), + } + + m.recoverInterruptedPushes() + return m, nil +} + +// pushKey identifies in-flight work by digest and target. +func pushKey(digest, target string) string { + return digest + "->" + target +} + +func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error) { + if req.Image == "" { + return nil, fmt.Errorf("%w: image is required", images.ErrInvalidName) + } + + img, err := m.resolver.GetImage(ctx, req.Image) + if err != nil { + return nil, err + } + if img.Status != images.StatusReady { + return nil, fmt.Errorf("%w: %s is %s", ErrImageNotReady, img.Name, img.Status) + } + + // Validate the target before persisting anything so typos fail fast. + var refOpts []name.Option + if req.Insecure { + refOpts = append(refOpts, name.Insecure) + } + dstRef, err := name.ParseReference(req.Target, refOpts...) + if err != nil { + return nil, fmt.Errorf("%w: %v", ErrInvalidTarget, err) + } + + key := pushKey(img.Digest, dstRef.String()) + + // Hold the lock across dedup check, metadata write, and registration so a + // concurrent request for the same digest+target cannot slip in between and + // leave an orphaned queued job behind. + m.mu.Lock() + if existing, ok := m.inflight[key]; ok { + id := existing.id + m.mu.Unlock() + return m.GetPush(ctx, id) + } + + meta := &pushMetadata{ + ID: cuid2.Generate(), + Status: StatusQueued, + Image: img.Name, + Digest: img.Digest, + Target: dstRef.String(), + Insecure: req.Insecure, + CreatedAt: time.Now(), + } + if err := writeMetadata(m.paths, meta); err != nil { + m.mu.Unlock() + return nil, fmt.Errorf("write initial metadata: %w", err) + } + m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest} + m.mu.Unlock() + + metaCopy := *meta + queuePos := m.queue.Enqueue(key, func() { + m.executePush(context.Background(), &metaCopy) + }) + + push := meta.toPush() + if queuePos > 0 { + push.QueuePosition = &queuePos + } + return push, nil +} + +func (m *manager) executePush(ctx context.Context, meta *pushMetadata) { + key := pushKey(meta.Digest, meta.Target) + defer func() { + m.mu.Lock() + delete(m.inflight, key) + m.mu.Unlock() + }() + + meta.Status = StatusPushing + writeMetadata(m.paths, meta) + + result, err := registrypush.PushFromCache(ctx, m.paths, meta.Digest, meta.Target, m.provider, registrypush.Options{ + Insecure: meta.Insecure, + }) + now := time.Now() + if err != nil { + errorMsg := err.Error() + meta.Status = StatusFailed + meta.Error = &errorMsg + meta.CompletedAt = &now + writeMetadata(m.paths, meta) + m.notify(meta.ID, StatusFailed, err) + return + } + + meta.Status = StatusPushed + meta.Layers = result.Layers + meta.Bytes = result.Bytes + meta.CompletedAt = &now + writeMetadata(m.paths, meta) + m.notify(meta.ID, StatusPushed, nil) +} + +func (m *manager) GetPush(ctx context.Context, id string) (*Push, error) { + meta, err := readMetadata(m.paths, id) + if err != nil { + return nil, err + } + + push := meta.toPush() + if meta.Status == StatusQueued { + push.QueuePosition = m.queue.GetPosition(pushKey(meta.Digest, meta.Target)) + } + return push, nil +} + +func (m *manager) ListPushes(ctx context.Context) ([]Push, error) { + metas, err := listAllPushes(m.paths) + if err != nil { + return nil, err + } + + pushes := make([]Push, 0, len(metas)) + for _, meta := range metas { + push := meta.toPush() + if meta.Status == StatusQueued { + push.QueuePosition = m.queue.GetPosition(pushKey(meta.Digest, meta.Target)) + } + pushes = append(pushes, *push) + } + return pushes, nil +} + +// WaitForPush blocks until the push reaches a terminal state (pushed or +// failed) or the context is cancelled. +func (m *manager) WaitForPush(ctx context.Context, id string) error { + push, err := m.GetPush(ctx, id) + if err != nil { + return err + } + + switch push.Status { + case StatusPushed: + return nil + case StatusFailed: + return pushError(push) + } + + ch := make(chan StatusEvent, 1) + m.subscribe(id, ch) + defer m.unsubscribe(id, ch) + + // Re-check after subscribing to close the race window. + push, err = m.GetPush(ctx, id) + if err != nil { + return err + } + switch push.Status { + case StatusPushed: + return nil + case StatusFailed: + return pushError(push) + } + + select { + case event := <-ch: + if event.Status == StatusPushed { + return nil + } + if event.Err != nil { + return fmt.Errorf("push failed: %w", event.Err) + } + return fmt.Errorf("push failed") + case <-ctx.Done(): + return ctx.Err() + } +} + +func pushError(push *Push) error { + if push.Error != nil { + return fmt.Errorf("push failed: %s", *push.Error) + } + return fmt.Errorf("push failed") +} + +func (m *manager) InProgressDigests() []string { + m.mu.Lock() + defer m.mu.Unlock() + + seen := make(map[string]struct{}, len(m.inflight)) + digests := make([]string, 0, len(m.inflight)) + for _, job := range m.inflight { + if _, ok := seen[job.digest]; ok { + continue + } + seen[job.digest] = struct{}{} + digests = append(digests, job.digest) + } + return digests +} + +func (m *manager) recoverInterruptedPushes() { + pending, err := listPendingPushes(m.paths) + if err != nil { + return // Best effort + } + + for _, meta := range pending { + key := pushKey(meta.Digest, meta.Target) + + m.mu.Lock() + m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest} + m.mu.Unlock() + + metaCopy := *meta + m.queue.Enqueue(key, func() { + m.executePush(context.Background(), &metaCopy) + }) + } +} + +func (m *manager) subscribe(id string, ch chan StatusEvent) { + m.subscriberMu.Lock() + defer m.subscriberMu.Unlock() + m.subscribers[id] = append(m.subscribers[id], ch) +} + +func (m *manager) unsubscribe(id string, ch chan StatusEvent) { + m.subscriberMu.Lock() + defer m.subscriberMu.Unlock() + + subs := m.subscribers[id] + for i, sub := range subs { + if sub == ch { + m.subscribers[id] = append(subs[:i], subs[i+1:]...) + break + } + } + if len(m.subscribers[id]) == 0 { + delete(m.subscribers, id) + } +} + +func (m *manager) notify(id, status string, err error) { + m.subscriberMu.RLock() + defer m.subscriberMu.RUnlock() + + event := StatusEvent{Status: status, Err: err} + for _, ch := range m.subscribers[id] { + select { + case ch <- event: + default: + } + } +} diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go new file mode 100644 index 00000000..894be2bc --- /dev/null +++ b/lib/imagepush/manager_test.go @@ -0,0 +1,520 @@ +package imagepush + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/ocicache" + "github.com/kernel/hypeman/lib/paths" +) + +// fakeResolver resolves image names from a fixed map. +type fakeResolver struct { + images map[string]*images.Image + err error +} + +func (f *fakeResolver) GetImage(_ context.Context, name string) (*images.Image, error) { + if f.err != nil { + return nil, f.err + } + img, ok := f.images[name] + if !ok { + return nil, fmt.Errorf("%w: %s", images.ErrNotFound, name) + } + return img, nil +} + +// cacheFixture writes an OCI-native random image into a temp OCI cache and +// returns the paths and manifest digest. +func cacheFixture(t *testing.T) (*paths.Paths, string) { + t.Helper() + + p := paths.New(t.TempDir()) + randomImg, err := random.Image(256, 1) + if err != nil { + t.Fatalf("random image: %v", err) + } + img := mutate.MediaType(randomImg, types.OCIManifestSchema1) + + blobDir := p.OCICacheBlobDir() + if err := os.MkdirAll(blobDir, 0755); err != nil { + t.Fatalf("create blob dir: %v", err) + } + writeBlob := func(hash v1.Hash, data []byte) { + t.Helper() + if err := os.WriteFile(filepath.Join(blobDir, hash.Hex), data, 0644); err != nil { + t.Fatalf("write blob: %v", err) + } + } + + layers, err := img.Layers() + if err != nil { + t.Fatalf("layers: %v", err) + } + for _, layer := range layers { + rc, err := layer.Compressed() + if err != nil { + t.Fatalf("layer reader: %v", err) + } + data, err := io.ReadAll(rc) + rc.Close() + if err != nil { + t.Fatalf("read layer: %v", err) + } + hash, err := layer.Digest() + if err != nil { + t.Fatalf("layer digest: %v", err) + } + writeBlob(hash, data) + } + + rawConfig, err := img.RawConfigFile() + if err != nil { + t.Fatalf("raw config: %v", err) + } + configHash, err := img.ConfigName() + if err != nil { + t.Fatalf("config name: %v", err) + } + writeBlob(configHash, rawConfig) + + rawManifest, err := img.RawManifest() + if err != nil { + t.Fatalf("raw manifest: %v", err) + } + digest, err := img.Digest() + if err != nil { + t.Fatalf("digest: %v", err) + } + writeBlob(digest, rawManifest) + + return p, digest.String() +} + +// gatedRegistry serves an in-process registry whose requests block until the +// returned channel is closed. +func gatedRegistry(t *testing.T) (host string, gate chan struct{}) { + t.Helper() + gate = make(chan struct{}) + inner := registry.New() + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + <-gate + inner.ServeHTTP(w, r) + }) + srv := httptest.NewServer(handler) + t.Cleanup(srv.Close) + return strings.TrimPrefix(srv.URL, "http://"), gate +} + +// openRegistry serves an unblocked in-process registry. +func openRegistry(t *testing.T) string { + t.Helper() + srv := httptest.NewServer(registry.New()) + t.Cleanup(srv.Close) + return strings.TrimPrefix(srv.URL, "http://") +} + +func readyImage(name_, digest string) *images.Image { + return &images.Image{ + Name: name_, + Digest: digest, + Status: images.StatusReady, + } +} + +func TestCreatePushEndToEnd(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + + mgr, err := NewManager(p, resolver, nil, 2) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + target := host + "/export/app:v1" + push, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: target, Insecure: true}) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + if push.Status != StatusQueued && push.Status != StatusPushing { + t.Errorf("initial status = %s, want queued or pushing", push.Status) + } + + if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { + t.Fatalf("WaitForPush: %v", err) + } + + got, err := mgr.GetPush(context.Background(), push.ID) + if err != nil { + t.Fatalf("GetPush: %v", err) + } + if got.Status != StatusPushed { + t.Errorf("status = %s, want pushed (error: %v)", got.Status, got.Error) + } + if got.Digest != digest { + t.Errorf("digest = %s, want %s", got.Digest, digest) + } + if got.Bytes <= 0 || got.Layers != 1 { + t.Errorf("bytes/layers = %d/%d, want >0/1", got.Bytes, got.Layers) + } + if got.CompletedAt == nil { + t.Error("completed at not set") + } + + // The pushed image must be readable from the destination with the same digest. + dstRef, err := name.ParseReference(target, name.Insecure) + if err != nil { + t.Fatalf("parse target: %v", err) + } + desc, err := remote.Get(dstRef, remote.WithAuth(authn.Anonymous)) + if err != nil { + t.Fatalf("read back pushed image: %v", err) + } + if desc.Digest.String() != digest { + t.Errorf("pushed digest = %s, want %s", desc.Digest, digest) + } + + // No in-flight digests once done. + if digests := mgr.InProgressDigests(); len(digests) != 0 { + t.Errorf("InProgressDigests = %v, want empty", digests) + } +} + +func TestCreatePushDedupesInFlight(t *testing.T) { + p, digest := cacheFixture(t) + host, gate := gatedRegistry(t) + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + + mgr, err := NewManager(p, resolver, nil, 2) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + req := PushRequest{Image: "myapp:v1", Target: host + "/export/app:v1", Insecure: true} + first, err := mgr.CreatePush(context.Background(), req) + if err != nil { + t.Fatalf("first CreatePush: %v", err) + } + + // Same digest+target while in flight returns the same job. + second, err := mgr.CreatePush(context.Background(), req) + if err != nil { + t.Fatalf("second CreatePush: %v", err) + } + if second.ID != first.ID { + t.Errorf("duplicate push got new ID %s, want %s", second.ID, first.ID) + } + + if digests := mgr.InProgressDigests(); len(digests) != 1 || digests[0] != digest { + t.Errorf("InProgressDigests = %v, want [%s]", digests, digest) + } + + close(gate) + if err := mgr.WaitForPush(context.Background(), first.ID); err != nil { + t.Fatalf("WaitForPush: %v", err) + } +} + +func TestCreatePushQueuesBehindConcurrencyLimit(t *testing.T) { + p, digest := cacheFixture(t) + host, gate := gatedRegistry(t) + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + first, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: host + "/export/a:v1", Insecure: true, + }) + if err != nil { + t.Fatalf("first CreatePush: %v", err) + } + + // Wait until the first job is actually running so the second must queue. + deadline := time.Now().Add(5 * time.Second) + for { + got, err := mgr.GetPush(context.Background(), first.ID) + if err != nil { + t.Fatalf("GetPush: %v", err) + } + if got.Status == StatusPushing { + break + } + if time.Now().After(deadline) { + t.Fatal("first push never started") + } + time.Sleep(10 * time.Millisecond) + } + + second, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: host + "/export/b:v1", Insecure: true, + }) + if err != nil { + t.Fatalf("second CreatePush: %v", err) + } + if second.QueuePosition == nil || *second.QueuePosition != 1 { + t.Errorf("second queue position = %v, want 1", second.QueuePosition) + } + + close(gate) + if err := mgr.WaitForPush(context.Background(), first.ID); err != nil { + t.Fatalf("WaitForPush first: %v", err) + } + if err := mgr.WaitForPush(context.Background(), second.ID); err != nil { + t.Fatalf("WaitForPush second: %v", err) + } +} + +func TestCreatePushImageNotReady(t *testing.T) { + p, digest := cacheFixture(t) + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": {Name: "myapp:v1", Digest: digest, Status: images.StatusConverting}, + }} + + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + _, err = mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: "registry.example.com/app:v1"}) + if !errors.Is(err, ErrImageNotReady) { + t.Errorf("err = %v, want ErrImageNotReady", err) + } +} + +func TestCreatePushUnknownImage(t *testing.T) { + p, _ := cacheFixture(t) + resolver := &fakeResolver{images: map[string]*images.Image{}} + + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + _, err = mgr.CreatePush(context.Background(), PushRequest{Image: "missing:v1", Target: "registry.example.com/app:v1"}) + if !errors.Is(err, images.ErrNotFound) { + t.Errorf("err = %v, want images.ErrNotFound", err) + } +} + +func TestCreatePushInvalidTarget(t *testing.T) { + p, digest := cacheFixture(t) + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + _, err = mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: "!!!invalid"}) + if !errors.Is(err, ErrInvalidTarget) { + t.Errorf("err = %v, want ErrInvalidTarget", err) + } + + // Nothing was persisted for the invalid request. + pushes, err := mgr.ListPushes(context.Background()) + if err != nil { + t.Fatalf("ListPushes: %v", err) + } + if len(pushes) != 0 { + t.Errorf("len(pushes) = %d, want 0", len(pushes)) + } +} + +func TestCreatePushFailureRecorded(t *testing.T) { + p, digest := cacheFixture(t) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusInternalServerError) + })) + t.Cleanup(srv.Close) + host := strings.TrimPrefix(srv.URL, "http://") + + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: host + "/export/app:v1", Insecure: true, + }) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + + err = mgr.WaitForPush(context.Background(), push.ID) + if err == nil { + t.Fatal("WaitForPush should fail for a failed push") + } + + got, err := mgr.GetPush(context.Background(), push.ID) + if err != nil { + t.Fatalf("GetPush: %v", err) + } + if got.Status != StatusFailed { + t.Errorf("status = %s, want failed", got.Status) + } + if got.Error == nil || *got.Error == "" { + t.Error("error not recorded on failed push") + } +} + +func TestListPushesNewestFirst(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 2) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + first, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: host + "/export/a:v1", Insecure: true, + }) + if err != nil { + t.Fatalf("CreatePush a: %v", err) + } + if err := mgr.WaitForPush(context.Background(), first.ID); err != nil { + t.Fatalf("WaitForPush a: %v", err) + } + + second, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: host + "/export/b:v1", Insecure: true, + }) + if err != nil { + t.Fatalf("CreatePush b: %v", err) + } + if err := mgr.WaitForPush(context.Background(), second.ID); err != nil { + t.Fatalf("WaitForPush b: %v", err) + } + + pushes, err := mgr.ListPushes(context.Background()) + if err != nil { + t.Fatalf("ListPushes: %v", err) + } + if len(pushes) != 2 { + t.Fatalf("len(pushes) = %d, want 2", len(pushes)) + } + if pushes[0].ID != second.ID || pushes[1].ID != first.ID { + t.Errorf("ordering = %s,%s, want newest first", pushes[0].ID, pushes[1].ID) + } +} + +func TestRecoverInterruptedPushes(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + + // Simulate a push interrupted by a restart: metadata on disk, nothing queued. + meta := &pushMetadata{ + ID: "recovered-push", + Status: StatusPushing, + Image: "myapp:v1", + Digest: digest, + Target: host + "/export/recovered:v1", + Insecure: true, + CreatedAt: time.Now(), + } + if err := writeMetadata(p, meta); err != nil { + t.Fatalf("writeMetadata: %v", err) + } + + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + if err := mgr.WaitForPush(context.Background(), "recovered-push"); err != nil { + t.Fatalf("WaitForPush recovered: %v", err) + } + got, err := mgr.GetPush(context.Background(), "recovered-push") + if err != nil { + t.Fatalf("GetPush: %v", err) + } + if got.Status != StatusPushed { + t.Errorf("status = %s, want pushed", got.Status) + } +} + +func TestWaitForPushNotFound(t *testing.T) { + p, _ := cacheFixture(t) + resolver := &fakeResolver{images: map[string]*images.Image{}} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + err = mgr.WaitForPush(context.Background(), "missing") + if !errors.Is(err, ErrNotFound) { + t.Errorf("err = %v, want ErrNotFound", err) + } +} + +// Ensure ocicache errors surface through the manager when blobs disappear. +func TestCreatePushMissingBlobs(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + if err := os.RemoveAll(p.OCICacheBlobDir()); err != nil { + t.Fatalf("remove blobs: %v", err) + } + + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: host + "/export/app:v1", Insecure: true, + }) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + err = mgr.WaitForPush(context.Background(), push.ID) + if err == nil { + t.Fatal("WaitForPush should fail when cache blobs are missing") + } + // Depending on timing the failure is observed via the live event (typed) + // or via persisted metadata (string), so accept both forms. + if !errors.Is(err, ocicache.ErrNotFound) && !strings.Contains(err.Error(), ocicache.ErrNotFound.Error()) { + t.Errorf("err = %v, want ocicache.ErrNotFound", err) + } +} diff --git a/lib/imagepush/queue.go b/lib/imagepush/queue.go new file mode 100644 index 00000000..33a27737 --- /dev/null +++ b/lib/imagepush/queue.go @@ -0,0 +1,106 @@ +package imagepush + +import "sync" + +type queuedPush struct { + key string + startFn func() +} + +// pushQueue runs push jobs with a configurable concurrency limit. Jobs are +// keyed by digest+target so duplicate requests dedupe against in-flight work. +type pushQueue struct { + maxConcurrent int + active map[string]bool + pending []queuedPush + mu sync.Mutex +} + +func newPushQueue(maxConcurrent int) *pushQueue { + if maxConcurrent < 1 { + maxConcurrent = 1 + } + return &pushQueue{ + maxConcurrent: maxConcurrent, + active: make(map[string]bool), + pending: make([]queuedPush, 0), + } +} + +// Enqueue adds a job to the queue. Returns the queue position: 0 if started +// immediately, >0 if queued behind other jobs. If the key is already active +// or pending, returns the existing position without re-enqueueing. +func (q *pushQueue) Enqueue(key string, startFn func()) int { + q.mu.Lock() + defer q.mu.Unlock() + + if q.active[key] { + return 0 + } + for i, job := range q.pending { + if job.key == key { + return i + 1 + } + } + + wrappedFn := func() { + defer q.MarkComplete(key) + startFn() + } + + if len(q.active) < q.maxConcurrent { + q.active[key] = true + go wrappedFn() + return 0 + } + + q.pending = append(q.pending, queuedPush{key: key, startFn: wrappedFn}) + return len(q.pending) +} + +func (q *pushQueue) MarkComplete(key string) { + q.mu.Lock() + defer q.mu.Unlock() + + delete(q.active, key) + + if len(q.pending) > 0 && len(q.active) < q.maxConcurrent { + next := q.pending[0] + q.pending = q.pending[1:] + q.active[next.key] = true + go next.startFn() + } +} + +// GetPosition returns nil if the key is active or unknown, otherwise its +// 1-based position in the pending queue. +func (q *pushQueue) GetPosition(key string) *int { + q.mu.Lock() + defer q.mu.Unlock() + + if q.active[key] { + return nil + } + for i, job := range q.pending { + if job.key == key { + pos := i + 1 + return &pos + } + } + return nil +} + +// ActiveKeys returns the keys of currently running jobs. +func (q *pushQueue) ActiveKeys() []string { + q.mu.Lock() + defer q.mu.Unlock() + + keys := make([]string, 0, len(q.active)+len(q.pending)) + for key := range q.active { + keys = append(keys, key) + } + for _, job := range q.pending { + keys = append(keys, job.key) + } + return keys +} diff --git a/lib/imagepush/queue_test.go b/lib/imagepush/queue_test.go new file mode 100644 index 00000000..e9590c85 --- /dev/null +++ b/lib/imagepush/queue_test.go @@ -0,0 +1,122 @@ +package imagepush + +import ( + "sync" + "testing" + "time" +) + +func TestPushQueueConcurrencyLimit(t *testing.T) { + q := newPushQueue(1) + + var mu sync.Mutex + running := 0 + maxRunning := 0 + release := make(chan struct{}) + done := make(chan struct{}, 2) + + startFn := func(block bool) func() { + return func() { + mu.Lock() + running++ + if running > maxRunning { + maxRunning = running + } + mu.Unlock() + + if block { + <-release + } + + mu.Lock() + running-- + mu.Unlock() + done <- struct{}{} + } + } + + posA := q.Enqueue("a", startFn(true)) + posB := q.Enqueue("b", startFn(false)) + if posA != 0 { + t.Errorf("posA = %d, want 0 (started immediately)", posA) + } + if posB != 1 { + t.Errorf("posB = %d, want 1 (queued behind a)", posB) + } + + if pos := q.GetPosition("b"); pos == nil || *pos != 1 { + t.Errorf("GetPosition(b) = %v, want 1", pos) + } + if pos := q.GetPosition("a"); pos != nil { + t.Errorf("GetPosition(a) = %v, want nil (active)", pos) + } + + close(release) + for i := 0; i < 2; i++ { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for jobs") + } + } + + if maxRunning != 1 { + t.Errorf("maxRunning = %d, want 1", maxRunning) + } + if pos := q.GetPosition("b"); pos != nil { + t.Errorf("GetPosition(b) after completion = %v, want nil", pos) + } +} + +func TestPushQueueDedupesByKey(t *testing.T) { + q := newPushQueue(1) + release := make(chan struct{}) + + blocked := func() { <-release } + started := make(chan struct{}, 1) + first := func() { + started <- struct{}{} + <-release + } + + q.Enqueue("same", first) + <-started + + pos := q.Enqueue("same", blocked) + if pos != 0 { + t.Errorf("duplicate enqueue of active key = %d, want 0", pos) + } + pos = q.Enqueue("same", blocked) + if pos != 0 { + t.Errorf("second duplicate enqueue = %d, want 0 (still active)", pos) + } + + close(release) + time.Sleep(50 * time.Millisecond) + + // After completion the key is no longer tracked. + if pos := q.GetPosition("same"); pos != nil { + t.Errorf("GetPosition after completion = %v, want nil", pos) + } +} + +func TestPushQueueActiveKeys(t *testing.T) { + q := newPushQueue(1) + release := make(chan struct{}) + + q.Enqueue("a", func() { <-release }) + q.Enqueue("b", func() {}) + + keys := q.ActiveKeys() + if len(keys) != 2 { + t.Fatalf("ActiveKeys = %v, want both a and b", keys) + } + + close(release) + time.Sleep(50 * time.Millisecond) + + keys = q.ActiveKeys() + if len(keys) != 0 { + t.Errorf("ActiveKeys after completion = %v, want empty", keys) + } +} diff --git a/lib/imagepush/storage.go b/lib/imagepush/storage.go new file mode 100644 index 00000000..37389165 --- /dev/null +++ b/lib/imagepush/storage.go @@ -0,0 +1,136 @@ +package imagepush + +import ( + "encoding/json" + "fmt" + "os" + "sort" + "time" + + "github.com/kernel/hypeman/lib/paths" +) + +// pushMetadata is the internal representation stored on disk. +type pushMetadata struct { + ID string `json:"id"` + Status string `json:"status"` + Image string `json:"image"` + Digest string `json:"digest"` + Target string `json:"target"` + Insecure bool `json:"insecure"` + Error *string `json:"error,omitempty"` + Layers int `json:"layers,omitempty"` + Bytes int64 `json:"bytes,omitempty"` + CreatedAt time.Time `json:"created_at"` + CompletedAt *time.Time `json:"completed_at,omitempty"` +} + +func (m *pushMetadata) toPush() *Push { + return &Push{ + ID: m.ID, + Image: m.Image, + Digest: m.Digest, + Target: m.Target, + Status: m.Status, + Error: m.Error, + Layers: m.Layers, + Bytes: m.Bytes, + CreatedAt: m.CreatedAt, + CompletedAt: m.CompletedAt, + } +} + +// writeMetadata writes push metadata to disk atomically. +func writeMetadata(p *paths.Paths, meta *pushMetadata) error { + dir := p.PushDir(meta.ID) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("create push directory: %w", err) + } + + data, err := json.MarshalIndent(meta, "", " ") + if err != nil { + return fmt.Errorf("marshal metadata: %w", err) + } + + tempPath := p.PushMetadata(meta.ID) + ".tmp" + if err := os.WriteFile(tempPath, data, 0644); err != nil { + return fmt.Errorf("write temp metadata: %w", err) + } + + finalPath := p.PushMetadata(meta.ID) + if err := os.Rename(tempPath, finalPath); err != nil { + os.Remove(tempPath) + return fmt.Errorf("rename metadata: %w", err) + } + + return nil +} + +func readMetadata(p *paths.Paths, id string) (*pushMetadata, error) { + data, err := os.ReadFile(p.PushMetadata(id)) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("read metadata: %w", err) + } + + var meta pushMetadata + if err := json.Unmarshal(data, &meta); err != nil { + return nil, fmt.Errorf("unmarshal metadata: %w", err) + } + + return &meta, nil +} + +// listAllPushes returns all pushes sorted by creation time (newest first). +func listAllPushes(p *paths.Paths) ([]*pushMetadata, error) { + entries, err := os.ReadDir(p.PushesDir()) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read pushes directory: %w", err) + } + + metas := make([]*pushMetadata, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + meta, err := readMetadata(p, entry.Name()) + if err != nil { + continue // Skip invalid entries + } + metas = append(metas, meta) + } + + sort.Slice(metas, func(i, j int) bool { + return metas[i].CreatedAt.After(metas[j].CreatedAt) + }) + + return metas, nil +} + +// listPendingPushes returns pushes that did not reach a terminal state, +// oldest first for FIFO recovery. +func listPendingPushes(p *paths.Paths) ([]*pushMetadata, error) { + all, err := listAllPushes(p) + if err != nil { + return nil, err + } + + pending := make([]*pushMetadata, 0) + for _, meta := range all { + switch meta.Status { + case StatusQueued, StatusPushing: + pending = append(pending, meta) + } + } + + sort.Slice(pending, func(i, j int) bool { + return pending[i].CreatedAt.Before(pending[j].CreatedAt) + }) + + return pending, nil +} diff --git a/lib/imagepush/storage_test.go b/lib/imagepush/storage_test.go new file mode 100644 index 00000000..a2324044 --- /dev/null +++ b/lib/imagepush/storage_test.go @@ -0,0 +1,94 @@ +package imagepush + +import ( + "errors" + "testing" + "time" + + "github.com/kernel/hypeman/lib/paths" +) + +func TestPushMetadataRoundTrip(t *testing.T) { + p := paths.New(t.TempDir()) + + errMsg := "registry unreachable" + completed := time.Now().Truncate(time.Second) + meta := &pushMetadata{ + ID: "push-1", + Status: StatusFailed, + Image: "docker.io/library/alpine:latest", + Digest: "sha256:abc123", + Target: "registry.example.com/app:v1", + Insecure: true, + Error: &errMsg, + Layers: 3, + Bytes: 1234, + CreatedAt: completed.Add(-time.Minute), + CompletedAt: &completed, + } + + if err := writeMetadata(p, meta); err != nil { + t.Fatalf("writeMetadata: %v", err) + } + + got, err := readMetadata(p, "push-1") + if err != nil { + t.Fatalf("readMetadata: %v", err) + } + if got.Status != StatusFailed || got.Image != meta.Image || got.Target != meta.Target || !got.Insecure { + t.Errorf("round trip mismatch: %+v", got) + } + if got.Error == nil || *got.Error != errMsg { + t.Errorf("error = %v, want %q", got.Error, errMsg) + } + if got.Layers != 3 || got.Bytes != 1234 { + t.Errorf("layers/bytes = %d/%d, want 3/1234", got.Layers, got.Bytes) + } + if got.CompletedAt == nil || !got.CompletedAt.Equal(completed) { + t.Errorf("completed at = %v, want %v", got.CompletedAt, completed) + } +} + +func TestReadMetadataNotFound(t *testing.T) { + p := paths.New(t.TempDir()) + + _, err := readMetadata(p, "missing") + if !errors.Is(err, ErrNotFound) { + t.Errorf("err = %v, want ErrNotFound", err) + } +} + +func TestListPushesOrderingAndPendingFilter(t *testing.T) { + p := paths.New(t.TempDir()) + now := time.Now() + + metas := []*pushMetadata{ + {ID: "old-done", Status: StatusPushed, Digest: "sha256:1", Target: "t1", CreatedAt: now.Add(-3 * time.Minute)}, + {ID: "mid-queued", Status: StatusQueued, Digest: "sha256:2", Target: "t2", CreatedAt: now.Add(-2 * time.Minute)}, + {ID: "new-failed", Status: StatusFailed, Digest: "sha256:3", Target: "t3", CreatedAt: now.Add(-1 * time.Minute)}, + } + for _, meta := range metas { + if err := writeMetadata(p, meta); err != nil { + t.Fatalf("writeMetadata(%s): %v", meta.ID, err) + } + } + + all, err := listAllPushes(p) + if err != nil { + t.Fatalf("listAllPushes: %v", err) + } + if len(all) != 3 { + t.Fatalf("len(all) = %d, want 3", len(all)) + } + if all[0].ID != "new-failed" || all[2].ID != "old-done" { + t.Errorf("ordering = %s..%s, want newest first", all[0].ID, all[2].ID) + } + + pending, err := listPendingPushes(p) + if err != nil { + t.Fatalf("listPendingPushes: %v", err) + } + if len(pending) != 1 || pending[0].ID != "mid-queued" { + t.Errorf("pending = %v, want only mid-queued", pending) + } +} diff --git a/lib/paths/paths.go b/lib/paths/paths.go index 46ac741f..b0bbc205 100644 --- a/lib/paths/paths.go +++ b/lib/paths/paths.go @@ -407,6 +407,23 @@ func (p *Paths) BuildMetadata(id string) string { return filepath.Join(p.BuildDir(id), "metadata.json") } +// Push path methods + +// PushesDir returns the root pushes directory. +func (p *Paths) PushesDir() string { + return filepath.Join(p.dataDir, "pushes") +} + +// PushDir returns the directory for a specific push. +func (p *Paths) PushDir(id string) string { + return filepath.Join(p.PushesDir(), id) +} + +// PushMetadata returns the path to push metadata.json. +func (p *Paths) PushMetadata(id string) string { + return filepath.Join(p.PushDir(id), "metadata.json") +} + // BuildLogs returns the path to build logs directory. func (p *Paths) BuildLogs(id string) string { return filepath.Join(p.BuildDir(id), "logs") From b6ac5cc741147e10238800333487220f8eafe3e9 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 5 Aug 2026 18:12:28 +0000 Subject: [PATCH 02/21] Support borrowed per-push registry credentials Docker-aligned credential flow: the caller's registry login rides along with the push request instead of living on the server. Borrowed credentials are used only for that job, never persisted or logged, and a credentialed push interrupted by a restart fails with an explanation instead of retrying under different credentials. The manager's default provider remains the fallback when no credentials are supplied. --- lib/imagepush/imagepush.go | 11 ++- lib/imagepush/manager.go | 43 ++++++++--- lib/imagepush/manager_test.go | 134 ++++++++++++++++++++++++++++++++++ lib/imagepush/storage.go | 28 ++++--- 4 files changed, 192 insertions(+), 24 deletions(-) diff --git a/lib/imagepush/imagepush.go b/lib/imagepush/imagepush.go index 366a7b90..370395bf 100644 --- a/lib/imagepush/imagepush.go +++ b/lib/imagepush/imagepush.go @@ -12,6 +12,7 @@ import ( "errors" "time" + "github.com/google/go-containerregistry/pkg/authn" "github.com/kernel/hypeman/lib/images" ) @@ -36,6 +37,13 @@ type PushRequest struct { Target string // Insecure allows pushing to plain-HTTP registries. Insecure bool + // Credentials are borrowed for this push only, docker-style: the caller's + // registry login (e.g. resolved from the client's ~/.docker/config.json) + // rides along with the request instead of living on the server. They are + // never persisted or logged; a push interrupted across a restart cannot be + // recovered and fails instead. When nil, the manager's default provider + // resolves credentials. + Credentials *authn.AuthConfig } // Push is the state of one push job. @@ -68,7 +76,8 @@ type StatusEvent struct { type Manager interface { // CreatePush validates the request, persists a queued job, and enqueues it. // A request that matches an in-flight job (same digest and target) returns - // the existing job instead of creating a duplicate. + // the existing job instead of creating a duplicate; the in-flight job's + // credentials remain in effect. CreatePush(ctx context.Context, req PushRequest) (*Push, error) GetPush(ctx context.Context, id string) (*Push, error) // ListPushes returns all pushes, newest first. diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index bc70e986..d2bd97a0 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -96,13 +96,14 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error } meta := &pushMetadata{ - ID: cuid2.Generate(), - Status: StatusQueued, - Image: img.Name, - Digest: img.Digest, - Target: dstRef.String(), - Insecure: req.Insecure, - CreatedAt: time.Now(), + ID: cuid2.Generate(), + Status: StatusQueued, + Image: img.Name, + Digest: img.Digest, + Target: dstRef.String(), + Insecure: req.Insecure, + HadCredentials: req.Credentials != nil, + CreatedAt: time.Now(), } if err := writeMetadata(m.paths, meta); err != nil { m.mu.Unlock() @@ -111,9 +112,16 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest} m.mu.Unlock() + // Borrowed credentials live only in this closure: the job provider is + // built per push and never touches disk. + provider := m.provider + if req.Credentials != nil { + provider = ®istrypush.StaticProvider{Config: *req.Credentials} + } + metaCopy := *meta queuePos := m.queue.Enqueue(key, func() { - m.executePush(context.Background(), &metaCopy) + m.executePush(context.Background(), &metaCopy, provider) }) push := meta.toPush() @@ -123,7 +131,7 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error return push, nil } -func (m *manager) executePush(ctx context.Context, meta *pushMetadata) { +func (m *manager) executePush(ctx context.Context, meta *pushMetadata, provider registrypush.Provider) { key := pushKey(meta.Digest, meta.Target) defer func() { m.mu.Lock() @@ -134,7 +142,7 @@ func (m *manager) executePush(ctx context.Context, meta *pushMetadata) { meta.Status = StatusPushing writeMetadata(m.paths, meta) - result, err := registrypush.PushFromCache(ctx, m.paths, meta.Digest, meta.Target, m.provider, registrypush.Options{ + result, err := registrypush.PushFromCache(ctx, m.paths, meta.Digest, meta.Target, provider, registrypush.Options{ Insecure: meta.Insecure, }) now := time.Now() @@ -261,6 +269,19 @@ func (m *manager) recoverInterruptedPushes() { } for _, meta := range pending { + // Borrowed credentials do not survive a restart, so a credentialed + // push cannot be retried faithfully; fail it instead of re-enqueueing + // with the default provider. + if meta.HadCredentials { + meta.Status = StatusFailed + errorMsg := "push interrupted by restart: borrowed registry credentials are no longer available, retry the push" + meta.Error = &errorMsg + now := time.Now() + meta.CompletedAt = &now + writeMetadata(m.paths, meta) + continue + } + key := pushKey(meta.Digest, meta.Target) m.mu.Lock() @@ -269,7 +290,7 @@ func (m *manager) recoverInterruptedPushes() { metaCopy := *meta m.queue.Enqueue(key, func() { - m.executePush(context.Background(), &metaCopy) + m.executePush(context.Background(), &metaCopy, m.provider) }) } } diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index 894be2bc..34712e17 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -486,6 +486,140 @@ func TestWaitForPushNotFound(t *testing.T) { } } +// erroringProvider always fails, proving a push that succeeds used the +// request's borrowed credentials instead of the manager default. +type erroringProvider struct{} + +func (erroringProvider) Authenticator(_ context.Context, _ name.Reference) (authn.Authenticator, error) { + return nil, fmt.Errorf("default provider must not be used") +} + +func TestCreatePushWithBorrowedCredentials(t *testing.T) { + p, digest := cacheFixture(t) + + inner := registry.New() + gated := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer borrowed-token" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + inner.ServeHTTP(w, r) + }) + srv := httptest.NewServer(gated) + t.Cleanup(srv.Close) + host := strings.TrimPrefix(srv.URL, "http://") + + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, erroringProvider{}, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", + Target: host + "/export/app:v1", + Insecure: true, + Credentials: &authn.AuthConfig{RegistryToken: "borrowed-token"}, + }) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { + t.Fatalf("WaitForPush: %v", err) + } + + got, err := mgr.GetPush(context.Background(), push.ID) + if err != nil { + t.Fatalf("GetPush: %v", err) + } + if got.Status != StatusPushed { + t.Errorf("status = %s, want pushed (error: %v)", got.Status, got.Error) + } +} + +func TestCredentialsNeverPersisted(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + const secret = "super-secret-borrowed-password" + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", + Target: host + "/export/app:v1", + Insecure: true, + Credentials: &authn.AuthConfig{Username: "pusher", Password: secret}, + }) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { + t.Fatalf("WaitForPush: %v", err) + } + + data, err := os.ReadFile(p.PushMetadata(push.ID)) + if err != nil { + t.Fatalf("read metadata: %v", err) + } + if strings.Contains(string(data), secret) || strings.Contains(string(data), "pusher") { + t.Error("borrowed credentials were persisted to disk") + } +} + +func TestRecoveryFailsCredentialJobs(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + + meta := &pushMetadata{ + ID: "cred-push", + Status: StatusQueued, + Image: "myapp:v1", + Digest: digest, + Target: host + "/export/recovered:v1", + Insecure: true, + HadCredentials: true, + CreatedAt: time.Now(), + } + if err := writeMetadata(p, meta); err != nil { + t.Fatalf("writeMetadata: %v", err) + } + + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + got, err := mgr.GetPush(context.Background(), "cred-push") + if err != nil { + t.Fatalf("GetPush: %v", err) + } + if got.Status != StatusFailed { + t.Errorf("status = %s, want failed (credentialed pushes cannot be recovered)", got.Status) + } + if got.Error == nil || !strings.Contains(*got.Error, "credentials") { + t.Errorf("error = %v, want an explanation about borrowed credentials", got.Error) + } + + // Nothing was pushed to the destination. + dstRef, err := name.ParseReference(host+"/export/recovered:v1", name.Insecure) + if err != nil { + t.Fatalf("parse target: %v", err) + } + if _, err := remote.Get(dstRef, remote.WithAuth(authn.Anonymous)); err == nil { + t.Error("recovered credentialed push should not have pushed") + } +} + // Ensure ocicache errors surface through the manager when blobs disappear. func TestCreatePushMissingBlobs(t *testing.T) { p, digest := cacheFixture(t) diff --git a/lib/imagepush/storage.go b/lib/imagepush/storage.go index 37389165..a85cf331 100644 --- a/lib/imagepush/storage.go +++ b/lib/imagepush/storage.go @@ -10,19 +10,23 @@ import ( "github.com/kernel/hypeman/lib/paths" ) -// pushMetadata is the internal representation stored on disk. +// pushMetadata is the internal representation stored on disk. Credentials are +// deliberately absent: borrowed credentials live only in memory for the +// duration of the push. HadCredentials records that the job used them so +// recovery can fail it instead of retrying without them. type pushMetadata struct { - ID string `json:"id"` - Status string `json:"status"` - Image string `json:"image"` - Digest string `json:"digest"` - Target string `json:"target"` - Insecure bool `json:"insecure"` - Error *string `json:"error,omitempty"` - Layers int `json:"layers,omitempty"` - Bytes int64 `json:"bytes,omitempty"` - CreatedAt time.Time `json:"created_at"` - CompletedAt *time.Time `json:"completed_at,omitempty"` + ID string `json:"id"` + Status string `json:"status"` + Image string `json:"image"` + Digest string `json:"digest"` + Target string `json:"target"` + Insecure bool `json:"insecure"` + HadCredentials bool `json:"had_credentials,omitempty"` + Error *string `json:"error,omitempty"` + Layers int `json:"layers,omitempty"` + Bytes int64 `json:"bytes,omitempty"` + CreatedAt time.Time `json:"created_at"` + CompletedAt *time.Time `json:"completed_at,omitempty"` } func (m *pushMetadata) toPush() *Push { From c4a65c45773b654cb389307311b7b69640f0280f Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:42:06 +0000 Subject: [PATCH 03/21] Fix push job lifecycle races and orphan handling - Release the inflight registration only after the queue slot is freed, closing a window where a new push for the same key persisted metadata but was never started because the queue still held the slot. - CreatePush adopts a pending record already on disk for the same digest+target instead of duplicating it. - Recovery dedupes same-key records (oldest wins, rest marked failed) and removes records whose status cannot be persisted, so nothing is left permanently queued. - Terminal status writes retry once and drop the record on failure so disk state cannot diverge from the WaitForPush notification. --- lib/imagepush/manager.go | 126 ++++++++++++++++++++++++-------- lib/imagepush/manager_test.go | 131 ++++++++++++++++++++++++++++++++++ lib/imagepush/queue.go | 36 +++++++--- lib/imagepush/queue_test.go | 49 +++++++++++-- lib/imagepush/storage.go | 16 +++++ 5 files changed, 314 insertions(+), 44 deletions(-) diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index d2bd97a0..73d23f09 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -3,6 +3,7 @@ package imagepush import ( "context" "fmt" + "os" "sync" "time" @@ -85,9 +86,16 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error key := pushKey(img.Digest, dstRef.String()) - // Hold the lock across dedup check, metadata write, and registration so a - // concurrent request for the same digest+target cannot slip in between and - // leave an orphaned queued job behind. + // Borrowed credentials live only in this closure: the job provider is + // built per push and never touches disk. + provider := m.provider + if req.Credentials != nil { + provider = ®istrypush.StaticProvider{Config: *req.Credentials} + } + + // Hold the lock across dedup check, orphan adoption, metadata write, and + // registration so a concurrent request for the same digest+target cannot + // slip in between and leave an orphaned queued job behind. m.mu.Lock() if existing, ok := m.inflight[key]; ok { id := existing.id @@ -95,6 +103,31 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error return m.GetPush(ctx, id) } + // Adopt a pending record that exists on disk but is not tracked in memory + // (e.g. left behind by an interrupted recovery) instead of creating a + // duplicate job for the same digest+target. Adopted records cannot carry + // borrowed credentials: recovery fails those instead of re-enqueueing. + orphan, err := findPendingPush(m.paths, key) + if err != nil { + m.mu.Unlock() + return nil, fmt.Errorf("scan pending pushes: %w", err) + } + if orphan != nil { + m.inflight[key] = inflightPush{id: orphan.ID, digest: orphan.Digest} + m.mu.Unlock() + + orphanCopy := *orphan + queuePos := m.queue.Enqueue(key, func() { + m.executePush(context.Background(), &orphanCopy, m.provider) + }, m.releaseInflight(key)) + + push := orphan.toPush() + if queuePos > 0 { + push.QueuePosition = &queuePos + } + return push, nil + } + meta := &pushMetadata{ ID: cuid2.Generate(), Status: StatusQueued, @@ -112,17 +145,10 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest} m.mu.Unlock() - // Borrowed credentials live only in this closure: the job provider is - // built per push and never touches disk. - provider := m.provider - if req.Credentials != nil { - provider = ®istrypush.StaticProvider{Config: *req.Credentials} - } - metaCopy := *meta queuePos := m.queue.Enqueue(key, func() { m.executePush(context.Background(), &metaCopy, provider) - }) + }, m.releaseInflight(key)) push := meta.toPush() if queuePos > 0 { @@ -132,15 +158,10 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error } func (m *manager) executePush(ctx context.Context, meta *pushMetadata, provider registrypush.Provider) { - key := pushKey(meta.Digest, meta.Target) - defer func() { - m.mu.Lock() - delete(m.inflight, key) - m.mu.Unlock() - }() - meta.Status = StatusPushing - writeMetadata(m.paths, meta) + if err := writeMetadata(m.paths, meta); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to persist push status for %s: %v\n", meta.ID, err) + } result, err := registrypush.PushFromCache(ctx, m.paths, meta.Digest, meta.Target, provider, registrypush.Options{ Insecure: meta.Insecure, @@ -151,7 +172,7 @@ func (m *manager) executePush(ctx context.Context, meta *pushMetadata, provider meta.Status = StatusFailed meta.Error = &errorMsg meta.CompletedAt = &now - writeMetadata(m.paths, meta) + m.writeTerminal(meta) m.notify(meta.ID, StatusFailed, err) return } @@ -160,10 +181,39 @@ func (m *manager) executePush(ctx context.Context, meta *pushMetadata, provider meta.Layers = result.Layers meta.Bytes = result.Bytes meta.CompletedAt = &now - writeMetadata(m.paths, meta) + m.writeTerminal(meta) m.notify(meta.ID, StatusPushed, nil) } +// writeTerminal persists a terminal status. If the write fails even after a +// retry, the job directory is removed so the on-disk record cannot diverge +// from the notification that WaitForPush returns; a record that cannot be +// persisted cannot be tracked. +func (m *manager) writeTerminal(meta *pushMetadata) { + err := writeMetadata(m.paths, meta) + if err != nil { + err = writeMetadata(m.paths, meta) + } + if err == nil { + return + } + fmt.Fprintf(os.Stderr, "Warning: dropping push record %s, could not persist terminal status: %v\n", meta.ID, err) + os.RemoveAll(m.paths.PushDir(meta.ID)) +} + +// releaseInflight returns the queue completion hook that drops the job's +// inflight registration. The queue runs it only after the key leaves the +// active set, so a concurrent CreatePush never falls into a gap between job +// completion and slot release: it either sees the inflight entry and gets the +// finished job, or enqueues a fresh one that actually starts. +func (m *manager) releaseInflight(key string) func() { + return func() { + m.mu.Lock() + delete(m.inflight, key) + m.mu.Unlock() + } +} + func (m *manager) GetPush(ctx context.Context, id string) (*Push, error) { meta, err := readMetadata(m.paths, id) if err != nil { @@ -268,21 +318,26 @@ func (m *manager) recoverInterruptedPushes() { return // Best effort } + seen := make(map[string]string, len(pending)) // key -> recovered push ID for _, meta := range pending { + key := pushKey(meta.Digest, meta.Target) + // Borrowed credentials do not survive a restart, so a credentialed // push cannot be retried faithfully; fail it instead of re-enqueueing // with the default provider. if meta.HadCredentials { - meta.Status = StatusFailed - errorMsg := "push interrupted by restart: borrowed registry credentials are no longer available, retry the push" - meta.Error = &errorMsg - now := time.Now() - meta.CompletedAt = &now - writeMetadata(m.paths, meta) + m.failRecovered(meta, "push interrupted by restart: borrowed registry credentials are no longer available, retry the push") continue } - key := pushKey(meta.Digest, meta.Target) + // Duplicate records for one logical push can accumulate on disk; + // recover the oldest and close the rest so nothing is left forever + // queued. + if chosen, ok := seen[key]; ok { + m.failRecovered(meta, fmt.Sprintf("superseded by duplicate push job %s for the same image and target", chosen)) + continue + } + seen[key] = meta.ID m.mu.Lock() m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest} @@ -291,7 +346,20 @@ func (m *manager) recoverInterruptedPushes() { metaCopy := *meta m.queue.Enqueue(key, func() { m.executePush(context.Background(), &metaCopy, m.provider) - }) + }, m.releaseInflight(key)) + } +} + +// failRecovered marks a recovered job failed. If the status cannot be +// persisted, the record is removed instead of being left permanently queued. +func (m *manager) failRecovered(meta *pushMetadata, reason string) { + meta.Status = StatusFailed + meta.Error = &reason + now := time.Now() + meta.CompletedAt = &now + if err := writeMetadata(m.paths, meta); err != nil { + fmt.Fprintf(os.Stderr, "Warning: dropping unrecoverable push record %s: %v\n", meta.ID, err) + os.RemoveAll(m.paths.PushDir(meta.ID)) } } diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index 34712e17..12fa9b77 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -472,6 +472,137 @@ func TestRecoverInterruptedPushes(t *testing.T) { } } +func TestCreatePushAdoptsOrphanedPendingJob(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + target := host + "/export/orphan:v1" + + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + // Simulate an orphan: a queued record on disk that the manager does not + // track (it appeared after startup, e.g. left behind by a crashed + // process). A new push for the same image+target must adopt it instead of + // creating a duplicate job. + orphan := &pushMetadata{ + ID: "orphan-1", + Status: StatusQueued, + Image: "myapp:v1", + Digest: digest, + Target: target, + Insecure: true, + CreatedAt: time.Now(), + } + if err := writeMetadata(p, orphan); err != nil { + t.Fatalf("writeMetadata: %v", err) + } + + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: target, Insecure: true, + }) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + if push.ID != "orphan-1" { + t.Fatalf("push ID = %s, want orphan-1 (adopted, not duplicated)", push.ID) + } + if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { + t.Fatalf("WaitForPush: %v", err) + } + + got, err := mgr.GetPush(context.Background(), "orphan-1") + if err != nil { + t.Fatalf("GetPush: %v", err) + } + if got.Status != StatusPushed { + t.Errorf("status = %s, want pushed", got.Status) + } + + // Exactly one job exists for the key. + pushes, err := mgr.ListPushes(context.Background()) + if err != nil { + t.Fatalf("ListPushes: %v", err) + } + if len(pushes) != 1 { + t.Errorf("len(pushes) = %d, want 1", len(pushes)) + } +} + +func TestRecoveryDedupesSameKey(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + target := host + "/export/dup:v1" + now := time.Now() + + older := &pushMetadata{ID: "older", Status: StatusQueued, Image: "myapp:v1", Digest: digest, Target: target, Insecure: true, CreatedAt: now.Add(-time.Minute)} + newer := &pushMetadata{ID: "newer", Status: StatusQueued, Image: "myapp:v1", Digest: digest, Target: target, Insecure: true, CreatedAt: now} + for _, meta := range []*pushMetadata{older, newer} { + if err := writeMetadata(p, meta); err != nil { + t.Fatalf("writeMetadata(%s): %v", meta.ID, err) + } + } + + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + if err := mgr.WaitForPush(context.Background(), "older"); err != nil { + t.Fatalf("WaitForPush older: %v", err) + } + + got, err := mgr.GetPush(context.Background(), "newer") + if err != nil { + t.Fatalf("GetPush newer: %v", err) + } + if got.Status != StatusFailed { + t.Errorf("newer status = %s, want failed (superseded)", got.Status) + } + if got.Error == nil || !strings.Contains(*got.Error, "superseded") { + t.Errorf("newer error = %v, want superseded explanation", got.Error) + } +} + +func TestSequentialSameKeyPushesAllComplete(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + target := host + "/export/again:v1" + for i := 0; i < 5; i++ { + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: target, Insecure: true, + }) + if err != nil { + t.Fatalf("CreatePush #%d: %v", i, err) + } + if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { + t.Fatalf("WaitForPush #%d: %v", i, err) + } + got, err := mgr.GetPush(context.Background(), push.ID) + if err != nil { + t.Fatalf("GetPush #%d: %v", i, err) + } + if got.Status != StatusPushed { + t.Fatalf("push #%d status = %s, want pushed (stuck job?)", i, got.Status) + } + } +} + func TestWaitForPushNotFound(t *testing.T) { p, _ := cacheFixture(t) resolver := &fakeResolver{images: map[string]*images.Image{}} diff --git a/lib/imagepush/queue.go b/lib/imagepush/queue.go index 33a27737..1e2b459a 100644 --- a/lib/imagepush/queue.go +++ b/lib/imagepush/queue.go @@ -29,8 +29,12 @@ func newPushQueue(maxConcurrent int) *pushQueue { // Enqueue adds a job to the queue. Returns the queue position: 0 if started // immediately, >0 if queued behind other jobs. If the key is already active -// or pending, returns the existing position without re-enqueueing. -func (q *pushQueue) Enqueue(key string, startFn func()) int { +// or pending, returns the existing position without re-enqueueing. When the +// job finishes, done runs after the key leaves the active set; until done +// returns, Enqueue still reports the key as in flight, so callers can rely on +// their own bookkeeping being torn down only after the queue is done with the +// key. +func (q *pushQueue) Enqueue(key string, startFn func(), done func()) int { q.mu.Lock() defer q.mu.Unlock() @@ -44,8 +48,8 @@ func (q *pushQueue) Enqueue(key string, startFn func()) int { } wrappedFn := func() { - defer q.MarkComplete(key) startFn() + q.complete(key, done) } if len(q.active) < q.maxConcurrent { @@ -58,18 +62,34 @@ func (q *pushQueue) Enqueue(key string, startFn func()) int { return len(q.pending) } -func (q *pushQueue) MarkComplete(key string) { +// complete removes the key from the active set, starts the next pending job +// if there is capacity, and then runs done. done runs after the key is no +// longer active so a concurrent Enqueue cannot fall into the gap between +// "job finished" and "queue slot released". +func (q *pushQueue) complete(key string, done func()) { q.mu.Lock() - defer q.mu.Unlock() - delete(q.active, key) + var next *queuedPush if len(q.pending) > 0 && len(q.active) < q.maxConcurrent { - next := q.pending[0] + nextJob := q.pending[0] q.pending = q.pending[1:] - q.active[next.key] = true + q.active[nextJob.key] = true + next = &nextJob + } + q.mu.Unlock() + + if next != nil { go next.startFn() } + if done != nil { + done() + } +} + +// MarkComplete releases the key without running a completion callback. +func (q *pushQueue) MarkComplete(key string) { + q.complete(key, nil) } // GetPosition returns nil if the key is active or unknown, otherwise its diff --git a/lib/imagepush/queue_test.go b/lib/imagepush/queue_test.go index e9590c85..13561730 100644 --- a/lib/imagepush/queue_test.go +++ b/lib/imagepush/queue_test.go @@ -35,8 +35,8 @@ func TestPushQueueConcurrencyLimit(t *testing.T) { } } - posA := q.Enqueue("a", startFn(true)) - posB := q.Enqueue("b", startFn(false)) + posA := q.Enqueue("a", startFn(true), nil) + posB := q.Enqueue("b", startFn(false), nil) if posA != 0 { t.Errorf("posA = %d, want 0 (started immediately)", posA) } @@ -79,14 +79,14 @@ func TestPushQueueDedupesByKey(t *testing.T) { <-release } - q.Enqueue("same", first) + q.Enqueue("same", first, nil) <-started - pos := q.Enqueue("same", blocked) + pos := q.Enqueue("same", blocked, nil) if pos != 0 { t.Errorf("duplicate enqueue of active key = %d, want 0", pos) } - pos = q.Enqueue("same", blocked) + pos = q.Enqueue("same", blocked, nil) if pos != 0 { t.Errorf("second duplicate enqueue = %d, want 0 (still active)", pos) } @@ -100,12 +100,47 @@ func TestPushQueueDedupesByKey(t *testing.T) { } } +// TestPushQueueCompletionOrdering guards the invariant that the done +// callback runs only after the key leaves the active set: a concurrent +// Enqueue in between must never fall into a gap where the job has finished +// but the slot is still held. +func TestPushQueueCompletionOrdering(t *testing.T) { + q := newPushQueue(1) + + started := make(chan struct{}) + doneRan := make(chan struct{}) + q.Enqueue("k", func() { + close(started) + }, func() { + for _, key := range q.ActiveKeys() { + if key == "k" { + t.Error("key still active when done ran") + } + } + close(doneRan) + }) + + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("job never started") + } + select { + case <-doneRan: + case <-time.After(5 * time.Second): + t.Fatal("done never ran") + } + if keys := q.ActiveKeys(); len(keys) != 0 { + t.Errorf("ActiveKeys = %v, want empty", keys) + } +} + func TestPushQueueActiveKeys(t *testing.T) { q := newPushQueue(1) release := make(chan struct{}) - q.Enqueue("a", func() { <-release }) - q.Enqueue("b", func() {}) + q.Enqueue("a", func() { <-release }, nil) + q.Enqueue("b", func() {}, nil) keys := q.ActiveKeys() if len(keys) != 2 { diff --git a/lib/imagepush/storage.go b/lib/imagepush/storage.go index a85cf331..ad0f6f00 100644 --- a/lib/imagepush/storage.go +++ b/lib/imagepush/storage.go @@ -116,6 +116,22 @@ func listAllPushes(p *paths.Paths) ([]*pushMetadata, error) { return metas, nil } +// findPendingPush returns the oldest non-terminal push matching the key, or +// nil when there is none. Used to adopt orphaned records instead of +// duplicating them. +func findPendingPush(p *paths.Paths, key string) (*pushMetadata, error) { + pending, err := listPendingPushes(p) + if err != nil { + return nil, err + } + for _, meta := range pending { + if pushKey(meta.Digest, meta.Target) == key { + return meta, nil + } + } + return nil, nil +} + // listPendingPushes returns pushes that did not reach a terminal state, // oldest first for FIFO recovery. func listPendingPushes(p *paths.Paths) ([]*pushMetadata, error) { From 954aaee148b507b919055f796f92e2bc880cb728 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:56:29 +0000 Subject: [PATCH 04/21] Address review: panic-safe queue completion, credentialed orphans, persist-or-fail terminals - Defer queue completion so a panicking job still releases its slot and runs the completion hook. - An orphaned pending record that used borrowed credentials is closed with the recovery policy and replaced by a fresh job instead of being re-executed under the default provider. - When a terminal status cannot be persisted even after retry, drop the record and report the job failed with the persistence problem so WaitForPush and GetPush agree, logging the actual push outcome. --- lib/imagepush/manager.go | 65 +++++++++++++++++++++-------------- lib/imagepush/manager_test.go | 54 +++++++++++++++++++++++++++++ lib/imagepush/queue.go | 4 ++- 3 files changed, 97 insertions(+), 26 deletions(-) diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index 73d23f09..99a03a55 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "strings" "sync" "time" @@ -105,13 +106,19 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error // Adopt a pending record that exists on disk but is not tracked in memory // (e.g. left behind by an interrupted recovery) instead of creating a - // duplicate job for the same digest+target. Adopted records cannot carry - // borrowed credentials: recovery fails those instead of re-enqueueing. + // duplicate job for the same digest+target. Records that carried borrowed + // credentials cannot be re-executed with them: close them with the same + // policy as recovery and fall through to create a fresh job (the current + // request may lend new credentials). orphan, err := findPendingPush(m.paths, key) if err != nil { m.mu.Unlock() return nil, fmt.Errorf("scan pending pushes: %w", err) } + if orphan != nil && orphan.HadCredentials { + m.failRecovered(orphan, "push interrupted by restart: borrowed registry credentials are no longer available, retry the push") + orphan = nil + } if orphan != nil { m.inflight[key] = inflightPush{id: orphan.ID, digest: orphan.Digest} m.mu.Unlock() @@ -163,42 +170,50 @@ func (m *manager) executePush(ctx context.Context, meta *pushMetadata, provider fmt.Fprintf(os.Stderr, "Warning: failed to persist push status for %s: %v\n", meta.ID, err) } - result, err := registrypush.PushFromCache(ctx, m.paths, meta.Digest, meta.Target, provider, registrypush.Options{ + result, pushErr := registrypush.PushFromCache(ctx, m.paths, meta.Digest, meta.Target, provider, registrypush.Options{ Insecure: meta.Insecure, }) now := time.Now() - if err != nil { - errorMsg := err.Error() + if pushErr != nil { + errorMsg := pushErr.Error() meta.Status = StatusFailed meta.Error = &errorMsg - meta.CompletedAt = &now - m.writeTerminal(meta) - m.notify(meta.ID, StatusFailed, err) + } else { + meta.Status = StatusPushed + meta.Layers = result.Layers + meta.Bytes = result.Bytes + } + meta.CompletedAt = &now + + if err := m.writeTerminal(meta); err != nil { + // The outcome cannot be recorded: drop the record and report the job + // as failed with the persistence problem, so WaitForPush and GetPush + // agree instead of diverging into success-then-not-found. The actual + // push outcome goes to the log. + fmt.Fprintf(os.Stderr, "Warning: push %s to %s finished as %s but the job record could not be persisted: %v\n", meta.ID, meta.Target, strings.ToLower(meta.Status), err) + os.RemoveAll(m.paths.PushDir(meta.ID)) + persistErr := fmt.Errorf("job record could not be persisted: %w", err) + errorMsg := persistErr.Error() + meta.Status = StatusFailed + meta.Error = &errorMsg + m.notify(meta.ID, StatusFailed, persistErr) return } - meta.Status = StatusPushed - meta.Layers = result.Layers - meta.Bytes = result.Bytes - meta.CompletedAt = &now - m.writeTerminal(meta) - m.notify(meta.ID, StatusPushed, nil) + if pushErr != nil { + m.notify(meta.ID, StatusFailed, pushErr) + } else { + m.notify(meta.ID, StatusPushed, nil) + } } -// writeTerminal persists a terminal status. If the write fails even after a -// retry, the job directory is removed so the on-disk record cannot diverge -// from the notification that WaitForPush returns; a record that cannot be -// persisted cannot be tracked. -func (m *manager) writeTerminal(meta *pushMetadata) { +// writeTerminal persists a terminal status, retrying once. +func (m *manager) writeTerminal(meta *pushMetadata) error { err := writeMetadata(m.paths, meta) if err != nil { err = writeMetadata(m.paths, meta) } - if err == nil { - return - } - fmt.Fprintf(os.Stderr, "Warning: dropping push record %s, could not persist terminal status: %v\n", meta.ID, err) - os.RemoveAll(m.paths.PushDir(meta.ID)) + return err } // releaseInflight returns the queue completion hook that drops the job's @@ -357,7 +372,7 @@ func (m *manager) failRecovered(meta *pushMetadata, reason string) { meta.Error = &reason now := time.Now() meta.CompletedAt = &now - if err := writeMetadata(m.paths, meta); err != nil { + if err := m.writeTerminal(meta); err != nil { fmt.Fprintf(os.Stderr, "Warning: dropping unrecoverable push record %s: %v\n", meta.ID, err) os.RemoveAll(m.paths.PushDir(meta.ID)) } diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index 12fa9b77..51d56d53 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -533,6 +533,60 @@ func TestCreatePushAdoptsOrphanedPendingJob(t *testing.T) { } } +func TestCreatePushOrphanWithCredentialsIsFailedNotAdopted(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + target := host + "/export/cred-orphan:v1" + + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + // An orphaned record that used borrowed credentials cannot be + // re-executed with them: it must be closed, and a fresh job created. + orphan := &pushMetadata{ + ID: "cred-orphan", + Status: StatusQueued, + Image: "myapp:v1", + Digest: digest, + Target: target, + Insecure: true, + HadCredentials: true, + CreatedAt: time.Now(), + } + if err := writeMetadata(p, orphan); err != nil { + t.Fatalf("writeMetadata: %v", err) + } + + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: target, Insecure: true, + }) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + if push.ID == "cred-orphan" { + t.Fatal("credentialed orphan must not be adopted") + } + if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { + t.Fatalf("WaitForPush: %v", err) + } + + got, err := mgr.GetPush(context.Background(), "cred-orphan") + if err != nil { + t.Fatalf("GetPush orphan: %v", err) + } + if got.Status != StatusFailed { + t.Errorf("orphan status = %s, want failed", got.Status) + } + if got.Error == nil || !strings.Contains(*got.Error, "credentials") { + t.Errorf("orphan error = %v, want credentials explanation", got.Error) + } +} + func TestRecoveryDedupesSameKey(t *testing.T) { p, digest := cacheFixture(t) host := openRegistry(t) diff --git a/lib/imagepush/queue.go b/lib/imagepush/queue.go index 1e2b459a..c6b4c57e 100644 --- a/lib/imagepush/queue.go +++ b/lib/imagepush/queue.go @@ -48,8 +48,10 @@ func (q *pushQueue) Enqueue(key string, startFn func(), done func()) int { } wrappedFn := func() { + // Deferred so a panicking startFn still releases the slot and runs + // the completion hook; otherwise the key would stay active forever. + defer q.complete(key, done) startFn() - q.complete(key, done) } if len(q.active) < q.maxConcurrent { From 3a74966744858af5855f4c72604c5af4daee8400 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 5 Aug 2026 20:03:29 +0000 Subject: [PATCH 05/21] Notify subscribers when a recovered push is closed failRecovered persisted the failed status without notifying, so a WaitForPush racing the close could subscribe before the write and then wait for a notification that never comes. --- lib/imagepush/manager.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index 99a03a55..a8dd34c0 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -2,6 +2,7 @@ package imagepush import ( "context" + "errors" "fmt" "os" "strings" @@ -367,6 +368,7 @@ func (m *manager) recoverInterruptedPushes() { // failRecovered marks a recovered job failed. If the status cannot be // persisted, the record is removed instead of being left permanently queued. +// Subscribers are notified so a WaitForPush racing the close does not hang. func (m *manager) failRecovered(meta *pushMetadata, reason string) { meta.Status = StatusFailed meta.Error = &reason @@ -376,6 +378,7 @@ func (m *manager) failRecovered(meta *pushMetadata, reason string) { fmt.Fprintf(os.Stderr, "Warning: dropping unrecoverable push record %s: %v\n", meta.ID, err) os.RemoveAll(m.paths.PushDir(meta.ID)) } + m.notify(meta.ID, StatusFailed, errors.New(reason)) } func (m *manager) subscribe(id string, ch chan StatusEvent) { From 8f4fdd0958949c2ba757312f3abd87daa1483511 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:34:21 +0000 Subject: [PATCH 06/21] Fix remaining review findings on push lifecycle - Include the insecure flag in the dedup/recovery key: the same target pushed with different transport modes is distinct work. - Log loudly when startup recovery cannot list pending pushes instead of failing silently. - Contain panics in the push goroutine: record a failed terminal and notify waiters instead of leaving the job stuck as pushing. - A request that lends credentials supersedes a credential-less orphan instead of adopting it, so the push never runs under the wrong auth. --- lib/imagepush/manager.go | 48 ++++++++++-- lib/imagepush/manager_test.go | 137 ++++++++++++++++++++++++++++++++++ lib/imagepush/storage.go | 2 +- 3 files changed, 178 insertions(+), 9 deletions(-) diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index a8dd34c0..3a333630 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -28,7 +28,7 @@ type manager struct { queue *pushQueue mu sync.Mutex - inflight map[string]inflightPush // key = pushKey(digest, target) + inflight map[string]inflightPush // key = pushKey(digest, target, insecure) subscriberMu sync.RWMutex subscribers map[string][]chan StatusEvent // keyed by push ID @@ -58,8 +58,12 @@ func NewManager(p *paths.Paths, resolver ImageResolver, provider registrypush.Pr return m, nil } -// pushKey identifies in-flight work by digest and target. -func pushKey(digest, target string) string { +// pushKey identifies in-flight work by digest, target, and transport mode: +// the same target pushed with and without Insecure is distinct work. +func pushKey(digest, target string, insecure bool) string { + if insecure { + return digest + "->" + target + "+insecure" + } return digest + "->" + target } @@ -86,7 +90,7 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error return nil, fmt.Errorf("%w: %v", ErrInvalidTarget, err) } - key := pushKey(img.Digest, dstRef.String()) + key := pushKey(img.Digest, dstRef.String(), req.Insecure) // Borrowed credentials live only in this closure: the job provider is // built per push and never touches disk. @@ -120,6 +124,13 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error m.failRecovered(orphan, "push interrupted by restart: borrowed registry credentials are no longer available, retry the push") orphan = nil } + // A request that lends credentials cannot adopt a credential-less orphan: + // the adopted job would run under the default provider instead of the + // borrowed login. Supersede the orphan and create a fresh job instead. + if orphan != nil && req.Credentials != nil { + m.failRecovered(orphan, "superseded by a new push request for the same image and target") + orphan = nil + } if orphan != nil { m.inflight[key] = inflightPush{id: orphan.ID, digest: orphan.Digest} m.mu.Unlock() @@ -166,6 +177,24 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error } func (m *manager) executePush(ctx context.Context, meta *pushMetadata, provider registrypush.Provider) { + // Contain panics in the job goroutine: record a failed terminal and + // notify waiters instead of leaving the job stuck as pushing. The queue + // slot is released by its own deferred completion. + defer func() { + if r := recover(); r != nil { + fmt.Fprintf(os.Stderr, "Warning: push %s to %s panicked: %v\n", meta.ID, meta.Target, r) + now := time.Now() + errorMsg := fmt.Sprintf("push panicked: %v", r) + meta.Status = StatusFailed + meta.Error = &errorMsg + meta.CompletedAt = &now + if err := m.writeTerminal(meta); err != nil { + os.RemoveAll(m.paths.PushDir(meta.ID)) + } + m.notify(meta.ID, StatusFailed, fmt.Errorf("push panicked: %v", r)) + } + }() + meta.Status = StatusPushing if err := writeMetadata(m.paths, meta); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to persist push status for %s: %v\n", meta.ID, err) @@ -238,7 +267,7 @@ func (m *manager) GetPush(ctx context.Context, id string) (*Push, error) { push := meta.toPush() if meta.Status == StatusQueued { - push.QueuePosition = m.queue.GetPosition(pushKey(meta.Digest, meta.Target)) + push.QueuePosition = m.queue.GetPosition(pushKey(meta.Digest, meta.Target, meta.Insecure)) } return push, nil } @@ -253,7 +282,7 @@ func (m *manager) ListPushes(ctx context.Context) ([]Push, error) { for _, meta := range metas { push := meta.toPush() if meta.Status == StatusQueued { - push.QueuePosition = m.queue.GetPosition(pushKey(meta.Digest, meta.Target)) + push.QueuePosition = m.queue.GetPosition(pushKey(meta.Digest, meta.Target, meta.Insecure)) } pushes = append(pushes, *push) } @@ -331,12 +360,15 @@ func (m *manager) InProgressDigests() []string { func (m *manager) recoverInterruptedPushes() { pending, err := listPendingPushes(m.paths) if err != nil { - return // Best effort + // Loud on purpose: without recovery these records stay queued on disk + // with nothing to run them. + fmt.Fprintf(os.Stderr, "Warning: could not recover interrupted pushes: %v\n", err) + return } seen := make(map[string]string, len(pending)) // key -> recovered push ID for _, meta := range pending { - key := pushKey(meta.Digest, meta.Target) + key := pushKey(meta.Digest, meta.Target, meta.Insecure) // Borrowed credentials do not survive a restart, so a credentialed // push cannot be retried faithfully; fail it instead of re-enqueueing diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index 51d56d53..88335275 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -837,3 +837,140 @@ func TestCreatePushMissingBlobs(t *testing.T) { t.Errorf("err = %v, want ocicache.ErrNotFound", err) } } + +// panickingProvider blows up during credential resolution, standing in for a +// panic anywhere in the push path. +type panickingProvider struct{} + +func (panickingProvider) Authenticator(_ context.Context, _ name.Reference) (authn.Authenticator, error) { + panic("boom") +} + +func TestExecutePushContainsPanic(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, panickingProvider{}, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: host + "/export/panic:v1", Insecure: true, + }) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err = mgr.WaitForPush(ctx, push.ID) + if err == nil { + t.Fatal("WaitForPush should fail for a panicked push") + } + if !strings.Contains(err.Error(), "panicked") { + t.Errorf("err = %v, want panic explanation", err) + } + + got, err := mgr.GetPush(context.Background(), push.ID) + if err != nil { + t.Fatalf("GetPush: %v", err) + } + if got.Status != StatusFailed { + t.Errorf("status = %s, want failed", got.Status) + } +} + +func TestCreatePushWithCredentialsSupersedesOrphan(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + target := host + "/export/supersede:v1" + + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + // A credential-less orphan cannot be adopted by a request that lends + // credentials: it would run under the default provider instead of the + // borrowed login. + orphan := &pushMetadata{ + ID: "plain-orphan", + Status: StatusQueued, + Image: "myapp:v1", + Digest: digest, + Target: target, + Insecure: true, + CreatedAt: time.Now(), + } + if err := writeMetadata(p, orphan); err != nil { + t.Fatalf("writeMetadata: %v", err) + } + + push, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", + Target: target, + Insecure: true, + Credentials: &authn.AuthConfig{RegistryToken: "borrowed"}, + }) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + if push.ID == "plain-orphan" { + t.Fatal("credentialed request must not adopt a credential-less orphan") + } + if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { + t.Fatalf("WaitForPush: %v", err) + } + + got, err := mgr.GetPush(context.Background(), "plain-orphan") + if err != nil { + t.Fatalf("GetPush orphan: %v", err) + } + if got.Status != StatusFailed { + t.Errorf("orphan status = %s, want failed (superseded)", got.Status) + } +} + +func TestRecoveryTreatsInsecureAsDistinctKey(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + target := host + "/export/distinct:v1" + now := time.Now() + + secure := &pushMetadata{ID: "secure", Status: StatusQueued, Image: "myapp:v1", Digest: digest, Target: target, Insecure: false, CreatedAt: now.Add(-time.Minute)} + insecure := &pushMetadata{ID: "insecure", Status: StatusQueued, Image: "myapp:v1", Digest: digest, Target: target, Insecure: true, CreatedAt: now} + for _, meta := range []*pushMetadata{secure, insecure} { + if err := writeMetadata(p, meta); err != nil { + t.Fatalf("writeMetadata(%s): %v", meta.ID, err) + } + } + + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + // Same digest+target with different transport modes is distinct work: + // both jobs recover, neither is superseded. + for _, id := range []string{"secure", "insecure"} { + if err := mgr.WaitForPush(context.Background(), id); err != nil { + t.Fatalf("WaitForPush %s: %v", id, err) + } + got, err := mgr.GetPush(context.Background(), id) + if err != nil { + t.Fatalf("GetPush %s: %v", id, err) + } + if got.Status != StatusPushed { + t.Errorf("%s status = %s, want pushed (error: %v)", id, got.Status, got.Error) + } + } +} diff --git a/lib/imagepush/storage.go b/lib/imagepush/storage.go index ad0f6f00..67f380dc 100644 --- a/lib/imagepush/storage.go +++ b/lib/imagepush/storage.go @@ -125,7 +125,7 @@ func findPendingPush(p *paths.Paths, key string) (*pushMetadata, error) { return nil, err } for _, meta := range pending { - if pushKey(meta.Digest, meta.Target) == key { + if pushKey(meta.Digest, meta.Target, meta.Insecure) == key { return meta, nil } } From 107efc14a28e4cc04da10bdeddb7f7689c87937b Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Fri, 7 Aug 2026 16:34:33 +0000 Subject: [PATCH 07/21] imagepush: fix credential-blind dedup and drop dead orphan adoption CreatePush now refuses to merge a request into an in-flight job whose credential presence differs (one borrowed, one not), returning ErrCredentialConflict instead of silently running under the wrong auth. The orphan-adoption path in CreatePush was unreachable: startup recovery adopts every pending record before any CreatePush can run, and the create lock covers write+registration, so the per-request PushesDir scan was dead O(N) disk I/O under the lock. Remove it along with findPendingPush. listAllPushes now warns on unreadable metadata instead of swallowing it. --- lib/imagepush/imagepush.go | 10 +- lib/imagepush/manager.go | 82 ++++++------- lib/imagepush/manager_test.go | 211 +++++++++++++--------------------- lib/imagepush/storage.go | 22 +--- lib/imagepush/storage_test.go | 27 +++++ 5 files changed, 157 insertions(+), 195 deletions(-) diff --git a/lib/imagepush/imagepush.go b/lib/imagepush/imagepush.go index 370395bf..3fb6bd03 100644 --- a/lib/imagepush/imagepush.go +++ b/lib/imagepush/imagepush.go @@ -27,6 +27,12 @@ var ( ErrNotFound = errors.New("push not found") ErrImageNotReady = errors.New("image not ready for push") ErrInvalidTarget = errors.New("invalid push target") + // ErrCredentialConflict is returned when a push request matches an + // in-flight job but its credential intent differs from that job's. The + // manager never stores credential values, so it can only detect a + // presence mismatch; two requests that both borrow different credentials + // for the same target are indistinguishable and merge. + ErrCredentialConflict = errors.New("push already in flight with different credentials") ) // PushRequest describes a request to push a hypeman image to a remote registry. @@ -77,7 +83,9 @@ type Manager interface { // CreatePush validates the request, persists a queued job, and enqueues it. // A request that matches an in-flight job (same digest and target) returns // the existing job instead of creating a duplicate; the in-flight job's - // credentials remain in effect. + // credentials remain in effect. A request whose credential presence + // differs from the in-flight job's (one borrowed, one not) returns + // ErrCredentialConflict instead of silently merging under the wrong auth. CreatePush(ctx context.Context, req PushRequest) (*Push, error) GetPush(ctx context.Context, id string) (*Push, error) // ListPushes returns all pushes, newest first. diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index 3a333630..99276917 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -17,8 +17,9 @@ import ( ) type inflightPush struct { - id string - digest string + id string + digest string + hadCredentials bool } type manager struct { @@ -79,6 +80,11 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error if img.Status != images.StatusReady { return nil, fmt.Errorf("%w: %s is %s", ErrImageNotReady, img.Name, img.Status) } + // A ready image must carry a digest; without one the dedup key below + // would collide across unrelated images. + if img.Digest == "" { + return nil, fmt.Errorf("%w: image %s has no digest", ErrImageNotReady, img.Name) + } // Validate the target before persisting anything so typos fail fast. var refOpts []name.Option @@ -99,54 +105,29 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error provider = ®istrypush.StaticProvider{Config: *req.Credentials} } - // Hold the lock across dedup check, orphan adoption, metadata write, and - // registration so a concurrent request for the same digest+target cannot - // slip in between and leave an orphaned queued job behind. + // Hold the lock across the dedup check, metadata write, and registration so + // a concurrent request for the same digest+target cannot slip in between + // and create a duplicate job. m.mu.Lock() if existing, ok := m.inflight[key]; ok { + // Merge only when the credential intent matches the in-flight job. The + // manager never stores credential values, so it can only compare + // presence: a request that borrowed credentials cannot merge into an + // anonymous in-flight push (its auth would be silently dropped), and an + // anonymous request cannot merge into a credentialed one (it would + // silently inherit another caller's login). Both silently merge under + // the wrong auth otherwise; surface the conflict instead so the caller + // can retry once the in-flight job completes or match its credentials. + hadCreds := req.Credentials != nil + if existing.hadCredentials != hadCreds { + m.mu.Unlock() + return nil, fmt.Errorf("%w: a push of %s to %s is already in flight with different credentials; retry once it completes or match its credentials", ErrCredentialConflict, img.Digest, dstRef.String()) + } id := existing.id m.mu.Unlock() return m.GetPush(ctx, id) } - // Adopt a pending record that exists on disk but is not tracked in memory - // (e.g. left behind by an interrupted recovery) instead of creating a - // duplicate job for the same digest+target. Records that carried borrowed - // credentials cannot be re-executed with them: close them with the same - // policy as recovery and fall through to create a fresh job (the current - // request may lend new credentials). - orphan, err := findPendingPush(m.paths, key) - if err != nil { - m.mu.Unlock() - return nil, fmt.Errorf("scan pending pushes: %w", err) - } - if orphan != nil && orphan.HadCredentials { - m.failRecovered(orphan, "push interrupted by restart: borrowed registry credentials are no longer available, retry the push") - orphan = nil - } - // A request that lends credentials cannot adopt a credential-less orphan: - // the adopted job would run under the default provider instead of the - // borrowed login. Supersede the orphan and create a fresh job instead. - if orphan != nil && req.Credentials != nil { - m.failRecovered(orphan, "superseded by a new push request for the same image and target") - orphan = nil - } - if orphan != nil { - m.inflight[key] = inflightPush{id: orphan.ID, digest: orphan.Digest} - m.mu.Unlock() - - orphanCopy := *orphan - queuePos := m.queue.Enqueue(key, func() { - m.executePush(context.Background(), &orphanCopy, m.provider) - }, m.releaseInflight(key)) - - push := orphan.toPush() - if queuePos > 0 { - push.QueuePosition = &queuePos - } - return push, nil - } - meta := &pushMetadata{ ID: cuid2.Generate(), Status: StatusQueued, @@ -161,7 +142,7 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error m.mu.Unlock() return nil, fmt.Errorf("write initial metadata: %w", err) } - m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest} + m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest, hadCredentials: meta.HadCredentials} m.mu.Unlock() metaCopy := *meta @@ -196,6 +177,8 @@ func (m *manager) executePush(ctx context.Context, meta *pushMetadata, provider }() meta.Status = StatusPushing + // Best-effort: if this write fails the job still runs and the terminal + // record written on completion is the source of truth for recovery. if err := writeMetadata(m.paths, meta); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to persist push status for %s: %v\n", meta.ID, err) } @@ -251,6 +234,9 @@ func (m *manager) writeTerminal(meta *pushMetadata) error { // active set, so a concurrent CreatePush never falls into a gap between job // completion and slot release: it either sees the inflight entry and gets the // finished job, or enqueues a fresh one that actually starts. +// This relies on executePush having persisted the terminal status before it +// returns (including from its panic handler); otherwise the record on disk +// and the inflight map could disagree about whether the job is still running. func (m *manager) releaseInflight(key string) func() { return func() { m.mu.Lock() @@ -260,6 +246,9 @@ func (m *manager) releaseInflight(key string) func() { } func (m *manager) GetPush(ctx context.Context, id string) (*Push, error) { + if err := ctx.Err(); err != nil { + return nil, err + } meta, err := readMetadata(m.paths, id) if err != nil { return nil, err @@ -273,6 +262,9 @@ func (m *manager) GetPush(ctx context.Context, id string) (*Push, error) { } func (m *manager) ListPushes(ctx context.Context) ([]Push, error) { + if err := ctx.Err(); err != nil { + return nil, err + } metas, err := listAllPushes(m.paths) if err != nil { return nil, err @@ -388,7 +380,7 @@ func (m *manager) recoverInterruptedPushes() { seen[key] = meta.ID m.mu.Lock() - m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest} + m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest, hadCredentials: meta.HadCredentials} m.mu.Unlock() metaCopy := *meta diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index 88335275..1d6b4b8d 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" @@ -472,118 +473,118 @@ func TestRecoverInterruptedPushes(t *testing.T) { } } -func TestCreatePushAdoptsOrphanedPendingJob(t *testing.T) { +func TestCreatePushDedupesConcurrently(t *testing.T) { p, digest := cacheFixture(t) - host := openRegistry(t) - target := host + "/export/orphan:v1" - + host, gate := gatedRegistry(t) resolver := &fakeResolver{images: map[string]*images.Image{ "myapp:v1": readyImage("myapp:v1", digest), }} - mgr, err := NewManager(p, resolver, nil, 1) + + mgr, err := NewManager(p, resolver, nil, 2) if err != nil { t.Fatalf("NewManager: %v", err) } - // Simulate an orphan: a queued record on disk that the manager does not - // track (it appeared after startup, e.g. left behind by a crashed - // process). A new push for the same image+target must adopt it instead of - // creating a duplicate job. - orphan := &pushMetadata{ - ID: "orphan-1", - Status: StatusQueued, - Image: "myapp:v1", - Digest: digest, - Target: target, - Insecure: true, - CreatedAt: time.Now(), - } - if err := writeMetadata(p, orphan); err != nil { - t.Fatalf("writeMetadata: %v", err) - } - - push, err := mgr.CreatePush(context.Background(), PushRequest{ - Image: "myapp:v1", Target: target, Insecure: true, - }) - if err != nil { - t.Fatalf("CreatePush: %v", err) - } - if push.ID != "orphan-1" { - t.Fatalf("push ID = %s, want orphan-1 (adopted, not duplicated)", push.ID) - } - if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { - t.Fatalf("WaitForPush: %v", err) + req := PushRequest{Image: "myapp:v1", Target: host + "/export/app:v1", Insecure: true} + if _, err := mgr.CreatePush(context.Background(), req); err != nil { + t.Fatalf("seed CreatePush: %v", err) + } + + // Two goroutines racing CreatePush for the same digest+target must both + // land on the single in-flight job: one registers, the other merges. + const n = 2 + ids := make([]string, n) + errs := make([]error, n) + var wg sync.WaitGroup + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + push, err := mgr.CreatePush(context.Background(), req) + if push != nil { + ids[i] = push.ID + } + errs[i] = err + }(i) + } + wg.Wait() + + for i := 0; i < n; i++ { + if errs[i] != nil { + t.Fatalf("concurrent CreatePush #%d: %v", i, errs[i]) + } + if ids[i] != ids[0] { + t.Errorf("concurrent push #%d got ID %s, want %s (single job)", i, ids[i], ids[0]) + } } - got, err := mgr.GetPush(context.Background(), "orphan-1") - if err != nil { - t.Fatalf("GetPush: %v", err) - } - if got.Status != StatusPushed { - t.Errorf("status = %s, want pushed", got.Status) + if digests := mgr.InProgressDigests(); len(digests) != 1 || digests[0] != digest { + t.Errorf("InProgressDigests = %v, want [%s]", digests, digest) } - // Exactly one job exists for the key. - pushes, err := mgr.ListPushes(context.Background()) - if err != nil { - t.Fatalf("ListPushes: %v", err) - } - if len(pushes) != 1 { - t.Errorf("len(pushes) = %d, want 1", len(pushes)) + close(gate) + if err := mgr.WaitForPush(context.Background(), ids[0]); err != nil { + t.Fatalf("WaitForPush: %v", err) } } -func TestCreatePushOrphanWithCredentialsIsFailedNotAdopted(t *testing.T) { +func TestCreatePushCredentialConflict(t *testing.T) { p, digest := cacheFixture(t) - host := openRegistry(t) - target := host + "/export/cred-orphan:v1" - + hostA, gateA := gatedRegistry(t) + hostB, gateB := gatedRegistry(t) + targetA := hostA + "/export/a:v1" + targetB := hostB + "/export/b:v1" resolver := &fakeResolver{images: map[string]*images.Image{ "myapp:v1": readyImage("myapp:v1", digest), }} + mgr, err := NewManager(p, resolver, nil, 1) if err != nil { t.Fatalf("NewManager: %v", err) } - // An orphaned record that used borrowed credentials cannot be - // re-executed with them: it must be closed, and a fresh job created. - orphan := &pushMetadata{ - ID: "cred-orphan", - Status: StatusQueued, - Image: "myapp:v1", - Digest: digest, - Target: target, - Insecure: true, - HadCredentials: true, - CreatedAt: time.Now(), + // Credentialed in-flight push, then an anonymous request: merging would + // silently drop the request's intent and run under the in-flight auth. + seeded, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: targetA, Insecure: true, + Credentials: &authn.AuthConfig{RegistryToken: "borrowed-token"}, + }) + if err != nil { + t.Fatalf("seed CreatePush: %v", err) } - if err := writeMetadata(p, orphan); err != nil { - t.Fatalf("writeMetadata: %v", err) + if _, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: targetA, Insecure: true}); !errors.Is(err, ErrCredentialConflict) { + t.Errorf("anonymous duplicate err = %v, want ErrCredentialConflict", err) } - push, err := mgr.CreatePush(context.Background(), PushRequest{ - Image: "myapp:v1", Target: target, Insecure: true, - }) + // Reverse: anonymous in-flight push, then a credentialed request. + seeded2, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: targetB, Insecure: true}) if err != nil { - t.Fatalf("CreatePush: %v", err) + t.Fatalf("seed CreatePush 2: %v", err) } - if push.ID == "cred-orphan" { - t.Fatal("credentialed orphan must not be adopted") + if _, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: targetB, Insecure: true, + Credentials: &authn.AuthConfig{RegistryToken: "borrowed-token"}, + }); !errors.Is(err, ErrCredentialConflict) { + t.Errorf("credentialed duplicate err = %v, want ErrCredentialConflict", err) } - if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { - t.Fatalf("WaitForPush: %v", err) + + close(gateA) + close(gateB) + if err := mgr.WaitForPush(context.Background(), seeded.ID); err != nil { + t.Fatalf("WaitForPush seeded: %v", err) + } + if err := mgr.WaitForPush(context.Background(), seeded2.ID); err != nil { + t.Fatalf("WaitForPush seeded 2: %v", err) } - got, err := mgr.GetPush(context.Background(), "cred-orphan") + // The conflicted requests must not have created duplicate jobs: only the + // two seeds exist. + pushes, err := mgr.ListPushes(context.Background()) if err != nil { - t.Fatalf("GetPush orphan: %v", err) - } - if got.Status != StatusFailed { - t.Errorf("orphan status = %s, want failed", got.Status) + t.Fatalf("ListPushes: %v", err) } - if got.Error == nil || !strings.Contains(*got.Error, "credentials") { - t.Errorf("orphan error = %v, want credentials explanation", got.Error) + if len(pushes) != 2 { + t.Errorf("len(pushes) = %d, want 2 (conflicts created no jobs)", len(pushes)) } } @@ -883,60 +884,6 @@ func TestExecutePushContainsPanic(t *testing.T) { } } -func TestCreatePushWithCredentialsSupersedesOrphan(t *testing.T) { - p, digest := cacheFixture(t) - host := openRegistry(t) - target := host + "/export/supersede:v1" - - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - mgr, err := NewManager(p, resolver, nil, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) - } - - // A credential-less orphan cannot be adopted by a request that lends - // credentials: it would run under the default provider instead of the - // borrowed login. - orphan := &pushMetadata{ - ID: "plain-orphan", - Status: StatusQueued, - Image: "myapp:v1", - Digest: digest, - Target: target, - Insecure: true, - CreatedAt: time.Now(), - } - if err := writeMetadata(p, orphan); err != nil { - t.Fatalf("writeMetadata: %v", err) - } - - push, err := mgr.CreatePush(context.Background(), PushRequest{ - Image: "myapp:v1", - Target: target, - Insecure: true, - Credentials: &authn.AuthConfig{RegistryToken: "borrowed"}, - }) - if err != nil { - t.Fatalf("CreatePush: %v", err) - } - if push.ID == "plain-orphan" { - t.Fatal("credentialed request must not adopt a credential-less orphan") - } - if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { - t.Fatalf("WaitForPush: %v", err) - } - - got, err := mgr.GetPush(context.Background(), "plain-orphan") - if err != nil { - t.Fatalf("GetPush orphan: %v", err) - } - if got.Status != StatusFailed { - t.Errorf("orphan status = %s, want failed (superseded)", got.Status) - } -} - func TestRecoveryTreatsInsecureAsDistinctKey(t *testing.T) { p, digest := cacheFixture(t) host := openRegistry(t) diff --git a/lib/imagepush/storage.go b/lib/imagepush/storage.go index 67f380dc..9544c66e 100644 --- a/lib/imagepush/storage.go +++ b/lib/imagepush/storage.go @@ -104,7 +104,11 @@ func listAllPushes(p *paths.Paths) ([]*pushMetadata, error) { } meta, err := readMetadata(p, entry.Name()) if err != nil { - continue // Skip invalid entries + // Surface unreadable records instead of swallowing them: a corrupt + // or half-written metadata.json would otherwise vanish from + // listing and recovery while its push directory lingers on disk. + fmt.Fprintf(os.Stderr, "Warning: skipping push %s with unreadable metadata: %v\n", entry.Name(), err) + continue } metas = append(metas, meta) } @@ -116,22 +120,6 @@ func listAllPushes(p *paths.Paths) ([]*pushMetadata, error) { return metas, nil } -// findPendingPush returns the oldest non-terminal push matching the key, or -// nil when there is none. Used to adopt orphaned records instead of -// duplicating them. -func findPendingPush(p *paths.Paths, key string) (*pushMetadata, error) { - pending, err := listPendingPushes(p) - if err != nil { - return nil, err - } - for _, meta := range pending { - if pushKey(meta.Digest, meta.Target, meta.Insecure) == key { - return meta, nil - } - } - return nil, nil -} - // listPendingPushes returns pushes that did not reach a terminal state, // oldest first for FIFO recovery. func listPendingPushes(p *paths.Paths) ([]*pushMetadata, error) { diff --git a/lib/imagepush/storage_test.go b/lib/imagepush/storage_test.go index a2324044..f32156cf 100644 --- a/lib/imagepush/storage_test.go +++ b/lib/imagepush/storage_test.go @@ -2,6 +2,7 @@ package imagepush import ( "errors" + "os" "testing" "time" @@ -58,6 +59,32 @@ func TestReadMetadataNotFound(t *testing.T) { } } +func TestListPushesSkipsUnreadableMetadata(t *testing.T) { + p := paths.New(t.TempDir()) + now := time.Now() + + if err := writeMetadata(p, &pushMetadata{ID: "good", Status: StatusPushed, Digest: "sha256:1", Target: "t1", CreatedAt: now}); err != nil { + t.Fatalf("writeMetadata(good): %v", err) + } + // A corrupt record must not fail the whole listing, but must not vanish + // silently either: it is skipped with a warning so it stays visible. + badDir := p.PushDir("corrupt") + if err := os.MkdirAll(badDir, 0755); err != nil { + t.Fatalf("mkdir corrupt: %v", err) + } + if err := os.WriteFile(p.PushMetadata("corrupt"), []byte("{not json"), 0644); err != nil { + t.Fatalf("write corrupt metadata: %v", err) + } + + all, err := listAllPushes(p) + if err != nil { + t.Fatalf("listAllPushes: %v", err) + } + if len(all) != 1 || all[0].ID != "good" { + t.Errorf("all = %v, want only good", all) + } +} + func TestListPushesOrderingAndPendingFilter(t *testing.T) { p := paths.New(t.TempDir()) now := time.Now() From 5f7038f6532efab568723049372d3efbc02036fa Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Fri, 7 Aug 2026 18:36:45 +0000 Subject: [PATCH 08/21] imagepush: share a generic queue and slim the test harness Extract lib/queue, a minimal in-memory bounded queue with key dedup, a concurrency cap, and an optional completion hook, and use it from imagepush and images; delete their local queue implementations (pushQueue and images' BuildQueue). builds keeps its superset with serial keys. imagepush/manager_test.go: add a testManager fixture to collapse the repeated paths+cache+resolver+NewManager setup, and merge the not-ready/unknown/ invalid-target rejection tests into one table-driven test. Net: -23% PR size (2030 -> ~1930 insertions) with behavior preserved. --- lib/imagepush/manager.go | 5 +- lib/imagepush/manager_test.go | 204 ++++++++----------------- lib/imagepush/queue.go | 128 ---------------- lib/imagepush/queue_test.go | 157 ------------------- lib/images/manager.go | 9 +- lib/images/queue.go | 121 --------------- lib/images/recovery_regression_test.go | 3 +- lib/queue/queue.go | 116 ++++++++++++++ lib/queue/queue_test.go | 136 +++++++++++++++++ 9 files changed, 328 insertions(+), 551 deletions(-) delete mode 100644 lib/imagepush/queue.go delete mode 100644 lib/imagepush/queue_test.go delete mode 100644 lib/images/queue.go create mode 100644 lib/queue/queue.go create mode 100644 lib/queue/queue_test.go diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index 99276917..f5f48ce6 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -12,6 +12,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/queue" "github.com/kernel/hypeman/lib/registrypush" "github.com/nrednav/cuid2" ) @@ -26,7 +27,7 @@ type manager struct { paths *paths.Paths resolver ImageResolver provider registrypush.Provider - queue *pushQueue + queue *queue.Queue mu sync.Mutex inflight map[string]inflightPush // key = pushKey(digest, target, insecure) @@ -50,7 +51,7 @@ func NewManager(p *paths.Paths, resolver ImageResolver, provider registrypush.Pr paths: p, resolver: resolver, provider: provider, - queue: newPushQueue(maxConcurrent), + queue: queue.New(maxConcurrent), inflight: make(map[string]inflightPush), subscribers: make(map[string][]chan StatusEvent), } diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index 1d6b4b8d..13ad2966 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -25,6 +25,7 @@ import ( "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/ocicache" "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/registrypush" ) // fakeResolver resolves image names from a fixed map. @@ -142,17 +143,28 @@ func readyImage(name_, digest string) *images.Image { } } -func TestCreatePushEndToEnd(t *testing.T) { +// testManager wires the standard fixture: a temp OCI cache containing a ready +// random image and a resolver mapping "myapp:v1" to it. provider and resolver +// may be nil (default keychain provider / ready image map). Returns the +// manager and the image's manifest digest. +func testManager(t *testing.T, maxConcurrent int, provider registrypush.Provider, resolver ImageResolver) (Manager, string) { + t.Helper() p, digest := cacheFixture(t) - host := openRegistry(t) - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - - mgr, err := NewManager(p, resolver, nil, 2) + if resolver == nil { + resolver = &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + } + mgr, err := NewManager(p, resolver, provider, maxConcurrent) if err != nil { t.Fatalf("NewManager: %v", err) } + return mgr, digest +} + +func TestCreatePushEndToEnd(t *testing.T) { + mgr, digest := testManager(t, 2, nil, nil) + host := openRegistry(t) target := host + "/export/app:v1" push, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: target, Insecure: true}) @@ -204,16 +216,8 @@ func TestCreatePushEndToEnd(t *testing.T) { } func TestCreatePushDedupesInFlight(t *testing.T) { - p, digest := cacheFixture(t) + mgr, digest := testManager(t, 2, nil, nil) host, gate := gatedRegistry(t) - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - - mgr, err := NewManager(p, resolver, nil, 2) - if err != nil { - t.Fatalf("NewManager: %v", err) - } req := PushRequest{Image: "myapp:v1", Target: host + "/export/app:v1", Insecure: true} first, err := mgr.CreatePush(context.Background(), req) @@ -241,16 +245,8 @@ func TestCreatePushDedupesInFlight(t *testing.T) { } func TestCreatePushQueuesBehindConcurrencyLimit(t *testing.T) { - p, digest := cacheFixture(t) + mgr, _ := testManager(t, 1, nil, nil) host, gate := gatedRegistry(t) - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - - mgr, err := NewManager(p, resolver, nil, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) - } first, err := mgr.CreatePush(context.Background(), PushRequest{ Image: "myapp:v1", Target: host + "/export/a:v1", Insecure: true, @@ -294,55 +290,45 @@ func TestCreatePushQueuesBehindConcurrencyLimit(t *testing.T) { } } -func TestCreatePushImageNotReady(t *testing.T) { - p, digest := cacheFixture(t) - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": {Name: "myapp:v1", Digest: digest, Status: images.StatusConverting}, - }} - - mgr, err := NewManager(p, resolver, nil, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) - } - - _, err = mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: "registry.example.com/app:v1"}) - if !errors.Is(err, ErrImageNotReady) { - t.Errorf("err = %v, want ErrImageNotReady", err) - } -} - -func TestCreatePushUnknownImage(t *testing.T) { - p, _ := cacheFixture(t) - resolver := &fakeResolver{images: map[string]*images.Image{}} - - mgr, err := NewManager(p, resolver, nil, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) - } - - _, err = mgr.CreatePush(context.Background(), PushRequest{Image: "missing:v1", Target: "registry.example.com/app:v1"}) - if !errors.Is(err, images.ErrNotFound) { - t.Errorf("err = %v, want images.ErrNotFound", err) - } -} - -func TestCreatePushInvalidTarget(t *testing.T) { - p, digest := cacheFixture(t) - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - - mgr, err := NewManager(p, resolver, nil, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) +func TestCreatePushRejectsInvalidRequests(t *testing.T) { + cases := []struct { + name string + resolver ImageResolver + req PushRequest + want error + }{ + { + name: "image not ready", + resolver: &fakeResolver{images: map[string]*images.Image{"myapp:v1": {Name: "myapp:v1", Digest: "sha256:0", Status: images.StatusConverting}}}, + req: PushRequest{Image: "myapp:v1", Target: "registry.example.com/app:v1"}, + want: ErrImageNotReady, + }, + { + name: "unknown image", + resolver: &fakeResolver{images: map[string]*images.Image{}}, + req: PushRequest{Image: "missing:v1", Target: "registry.example.com/app:v1"}, + want: images.ErrNotFound, + }, + { + name: "invalid target", + req: PushRequest{Image: "myapp:v1", Target: "!!!invalid"}, + want: ErrInvalidTarget, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + mgr, _ := testManager(t, 1, nil, tc.resolver) + if _, err := mgr.CreatePush(context.Background(), tc.req); !errors.Is(err, tc.want) { + t.Errorf("err = %v, want %v", err, tc.want) + } + }) } - _, err = mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: "!!!invalid"}) - if !errors.Is(err, ErrInvalidTarget) { - t.Errorf("err = %v, want ErrInvalidTarget", err) + // Nothing was persisted for an invalid request. + mgr, _ := testManager(t, 1, nil, nil) + if _, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: "!!!invalid"}); !errors.Is(err, ErrInvalidTarget) { + t.Fatalf("err = %v, want ErrInvalidTarget", err) } - - // Nothing was persisted for the invalid request. pushes, err := mgr.ListPushes(context.Background()) if err != nil { t.Fatalf("ListPushes: %v", err) @@ -353,21 +339,13 @@ func TestCreatePushInvalidTarget(t *testing.T) { } func TestCreatePushFailureRecorded(t *testing.T) { - p, digest := cacheFixture(t) + mgr, _ := testManager(t, 1, nil, nil) srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, "boom", http.StatusInternalServerError) })) t.Cleanup(srv.Close) host := strings.TrimPrefix(srv.URL, "http://") - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - mgr, err := NewManager(p, resolver, nil, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) - } - push, err := mgr.CreatePush(context.Background(), PushRequest{ Image: "myapp:v1", Target: host + "/export/app:v1", Insecure: true, }) @@ -393,15 +371,8 @@ func TestCreatePushFailureRecorded(t *testing.T) { } func TestListPushesNewestFirst(t *testing.T) { - p, digest := cacheFixture(t) + mgr, _ := testManager(t, 2, nil, nil) host := openRegistry(t) - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - mgr, err := NewManager(p, resolver, nil, 2) - if err != nil { - t.Fatalf("NewManager: %v", err) - } first, err := mgr.CreatePush(context.Background(), PushRequest{ Image: "myapp:v1", Target: host + "/export/a:v1", Insecure: true, @@ -474,16 +445,8 @@ func TestRecoverInterruptedPushes(t *testing.T) { } func TestCreatePushDedupesConcurrently(t *testing.T) { - p, digest := cacheFixture(t) + mgr, digest := testManager(t, 2, nil, nil) host, gate := gatedRegistry(t) - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - - mgr, err := NewManager(p, resolver, nil, 2) - if err != nil { - t.Fatalf("NewManager: %v", err) - } req := PushRequest{Image: "myapp:v1", Target: host + "/export/app:v1", Insecure: true} if _, err := mgr.CreatePush(context.Background(), req); err != nil { @@ -529,19 +492,11 @@ func TestCreatePushDedupesConcurrently(t *testing.T) { } func TestCreatePushCredentialConflict(t *testing.T) { - p, digest := cacheFixture(t) + mgr, _ := testManager(t, 1, nil, nil) hostA, gateA := gatedRegistry(t) hostB, gateB := gatedRegistry(t) targetA := hostA + "/export/a:v1" targetB := hostB + "/export/b:v1" - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - - mgr, err := NewManager(p, resolver, nil, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) - } // Credentialed in-flight push, then an anonymous request: merging would // silently drop the request's intent and run under the in-flight auth. @@ -627,15 +582,8 @@ func TestRecoveryDedupesSameKey(t *testing.T) { } func TestSequentialSameKeyPushesAllComplete(t *testing.T) { - p, digest := cacheFixture(t) + mgr, _ := testManager(t, 1, nil, nil) host := openRegistry(t) - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - mgr, err := NewManager(p, resolver, nil, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) - } target := host + "/export/again:v1" for i := 0; i < 5; i++ { @@ -659,14 +607,9 @@ func TestSequentialSameKeyPushesAllComplete(t *testing.T) { } func TestWaitForPushNotFound(t *testing.T) { - p, _ := cacheFixture(t) - resolver := &fakeResolver{images: map[string]*images.Image{}} - mgr, err := NewManager(p, resolver, nil, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) - } + mgr, _ := testManager(t, 1, nil, nil) - err = mgr.WaitForPush(context.Background(), "missing") + err := mgr.WaitForPush(context.Background(), "missing") if !errors.Is(err, ErrNotFound) { t.Errorf("err = %v, want ErrNotFound", err) } @@ -681,7 +624,7 @@ func (erroringProvider) Authenticator(_ context.Context, _ name.Reference) (auth } func TestCreatePushWithBorrowedCredentials(t *testing.T) { - p, digest := cacheFixture(t) + mgr, _ := testManager(t, 1, erroringProvider{}, nil) inner := registry.New() gated := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -695,14 +638,6 @@ func TestCreatePushWithBorrowedCredentials(t *testing.T) { t.Cleanup(srv.Close) host := strings.TrimPrefix(srv.URL, "http://") - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - mgr, err := NewManager(p, resolver, erroringProvider{}, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) - } - push, err := mgr.CreatePush(context.Background(), PushRequest{ Image: "myapp:v1", Target: host + "/export/app:v1", @@ -848,15 +783,8 @@ func (panickingProvider) Authenticator(_ context.Context, _ name.Reference) (aut } func TestExecutePushContainsPanic(t *testing.T) { - p, digest := cacheFixture(t) + mgr, _ := testManager(t, 1, panickingProvider{}, nil) host := openRegistry(t) - resolver := &fakeResolver{images: map[string]*images.Image{ - "myapp:v1": readyImage("myapp:v1", digest), - }} - mgr, err := NewManager(p, resolver, panickingProvider{}, 1) - if err != nil { - t.Fatalf("NewManager: %v", err) - } push, err := mgr.CreatePush(context.Background(), PushRequest{ Image: "myapp:v1", Target: host + "/export/panic:v1", Insecure: true, diff --git a/lib/imagepush/queue.go b/lib/imagepush/queue.go deleted file mode 100644 index c6b4c57e..00000000 --- a/lib/imagepush/queue.go +++ /dev/null @@ -1,128 +0,0 @@ -package imagepush - -import "sync" - -type queuedPush struct { - key string - startFn func() -} - -// pushQueue runs push jobs with a configurable concurrency limit. Jobs are -// keyed by digest+target so duplicate requests dedupe against in-flight work. -type pushQueue struct { - maxConcurrent int - active map[string]bool - pending []queuedPush - mu sync.Mutex -} - -func newPushQueue(maxConcurrent int) *pushQueue { - if maxConcurrent < 1 { - maxConcurrent = 1 - } - return &pushQueue{ - maxConcurrent: maxConcurrent, - active: make(map[string]bool), - pending: make([]queuedPush, 0), - } -} - -// Enqueue adds a job to the queue. Returns the queue position: 0 if started -// immediately, >0 if queued behind other jobs. If the key is already active -// or pending, returns the existing position without re-enqueueing. When the -// job finishes, done runs after the key leaves the active set; until done -// returns, Enqueue still reports the key as in flight, so callers can rely on -// their own bookkeeping being torn down only after the queue is done with the -// key. -func (q *pushQueue) Enqueue(key string, startFn func(), done func()) int { - q.mu.Lock() - defer q.mu.Unlock() - - if q.active[key] { - return 0 - } - for i, job := range q.pending { - if job.key == key { - return i + 1 - } - } - - wrappedFn := func() { - // Deferred so a panicking startFn still releases the slot and runs - // the completion hook; otherwise the key would stay active forever. - defer q.complete(key, done) - startFn() - } - - if len(q.active) < q.maxConcurrent { - q.active[key] = true - go wrappedFn() - return 0 - } - - q.pending = append(q.pending, queuedPush{key: key, startFn: wrappedFn}) - return len(q.pending) -} - -// complete removes the key from the active set, starts the next pending job -// if there is capacity, and then runs done. done runs after the key is no -// longer active so a concurrent Enqueue cannot fall into the gap between -// "job finished" and "queue slot released". -func (q *pushQueue) complete(key string, done func()) { - q.mu.Lock() - delete(q.active, key) - - var next *queuedPush - if len(q.pending) > 0 && len(q.active) < q.maxConcurrent { - nextJob := q.pending[0] - q.pending = q.pending[1:] - q.active[nextJob.key] = true - next = &nextJob - } - q.mu.Unlock() - - if next != nil { - go next.startFn() - } - if done != nil { - done() - } -} - -// MarkComplete releases the key without running a completion callback. -func (q *pushQueue) MarkComplete(key string) { - q.complete(key, nil) -} - -// GetPosition returns nil if the key is active or unknown, otherwise its -// 1-based position in the pending queue. -func (q *pushQueue) GetPosition(key string) *int { - q.mu.Lock() - defer q.mu.Unlock() - - if q.active[key] { - return nil - } - for i, job := range q.pending { - if job.key == key { - pos := i + 1 - return &pos - } - } - return nil -} - -// ActiveKeys returns the keys of currently running jobs. -func (q *pushQueue) ActiveKeys() []string { - q.mu.Lock() - defer q.mu.Unlock() - - keys := make([]string, 0, len(q.active)+len(q.pending)) - for key := range q.active { - keys = append(keys, key) - } - for _, job := range q.pending { - keys = append(keys, job.key) - } - return keys -} diff --git a/lib/imagepush/queue_test.go b/lib/imagepush/queue_test.go deleted file mode 100644 index 13561730..00000000 --- a/lib/imagepush/queue_test.go +++ /dev/null @@ -1,157 +0,0 @@ -package imagepush - -import ( - "sync" - "testing" - "time" -) - -func TestPushQueueConcurrencyLimit(t *testing.T) { - q := newPushQueue(1) - - var mu sync.Mutex - running := 0 - maxRunning := 0 - release := make(chan struct{}) - done := make(chan struct{}, 2) - - startFn := func(block bool) func() { - return func() { - mu.Lock() - running++ - if running > maxRunning { - maxRunning = running - } - mu.Unlock() - - if block { - <-release - } - - mu.Lock() - running-- - mu.Unlock() - done <- struct{}{} - } - } - - posA := q.Enqueue("a", startFn(true), nil) - posB := q.Enqueue("b", startFn(false), nil) - if posA != 0 { - t.Errorf("posA = %d, want 0 (started immediately)", posA) - } - if posB != 1 { - t.Errorf("posB = %d, want 1 (queued behind a)", posB) - } - - if pos := q.GetPosition("b"); pos == nil || *pos != 1 { - t.Errorf("GetPosition(b) = %v, want 1", pos) - } - if pos := q.GetPosition("a"); pos != nil { - t.Errorf("GetPosition(a) = %v, want nil (active)", pos) - } - - close(release) - for i := 0; i < 2; i++ { - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("timed out waiting for jobs") - } - } - - if maxRunning != 1 { - t.Errorf("maxRunning = %d, want 1", maxRunning) - } - if pos := q.GetPosition("b"); pos != nil { - t.Errorf("GetPosition(b) after completion = %v, want nil", pos) - } -} - -func TestPushQueueDedupesByKey(t *testing.T) { - q := newPushQueue(1) - release := make(chan struct{}) - - blocked := func() { <-release } - started := make(chan struct{}, 1) - first := func() { - started <- struct{}{} - <-release - } - - q.Enqueue("same", first, nil) - <-started - - pos := q.Enqueue("same", blocked, nil) - if pos != 0 { - t.Errorf("duplicate enqueue of active key = %d, want 0", pos) - } - pos = q.Enqueue("same", blocked, nil) - if pos != 0 { - t.Errorf("second duplicate enqueue = %d, want 0 (still active)", pos) - } - - close(release) - time.Sleep(50 * time.Millisecond) - - // After completion the key is no longer tracked. - if pos := q.GetPosition("same"); pos != nil { - t.Errorf("GetPosition after completion = %v, want nil", pos) - } -} - -// TestPushQueueCompletionOrdering guards the invariant that the done -// callback runs only after the key leaves the active set: a concurrent -// Enqueue in between must never fall into a gap where the job has finished -// but the slot is still held. -func TestPushQueueCompletionOrdering(t *testing.T) { - q := newPushQueue(1) - - started := make(chan struct{}) - doneRan := make(chan struct{}) - q.Enqueue("k", func() { - close(started) - }, func() { - for _, key := range q.ActiveKeys() { - if key == "k" { - t.Error("key still active when done ran") - } - } - close(doneRan) - }) - - select { - case <-started: - case <-time.After(5 * time.Second): - t.Fatal("job never started") - } - select { - case <-doneRan: - case <-time.After(5 * time.Second): - t.Fatal("done never ran") - } - if keys := q.ActiveKeys(); len(keys) != 0 { - t.Errorf("ActiveKeys = %v, want empty", keys) - } -} - -func TestPushQueueActiveKeys(t *testing.T) { - q := newPushQueue(1) - release := make(chan struct{}) - - q.Enqueue("a", func() { <-release }, nil) - q.Enqueue("b", func() {}, nil) - - keys := q.ActiveKeys() - if len(keys) != 2 { - t.Fatalf("ActiveKeys = %v, want both a and b", keys) - } - - close(release) - time.Sleep(50 * time.Millisecond) - - keys = q.ActiveKeys() - if len(keys) != 0 { - t.Errorf("ActiveKeys after completion = %v, want empty", keys) - } -} diff --git a/lib/images/manager.go b/lib/images/manager.go index 85be8b48..58a1bac0 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -13,6 +13,7 @@ import ( "time" "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/queue" "github.com/kernel/hypeman/lib/tags" "go.opentelemetry.io/otel/metric" ) @@ -54,7 +55,7 @@ type Manager interface { type manager struct { paths *paths.Paths ociClient *ociClient - queue *BuildQueue + queue *queue.Queue createMu sync.Mutex diskUsageMu sync.RWMutex diskUsageLoaded bool @@ -78,7 +79,7 @@ func NewManager(p *paths.Paths, maxConcurrentBuilds int, meter metric.Meter) (Ma m := &manager{ paths: p, ociClient: ociClient, - queue: NewBuildQueue(maxConcurrentBuilds), + queue: queue.New(maxConcurrentBuilds), readySubscribers: make(map[string][]chan StatusEvent), } @@ -265,7 +266,7 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, } // Enqueue the build using digest as the queue key for deduplication - queuePos := m.queue.Enqueue(ref.Digest(), storedReq, func() { + queuePos := m.queue.Enqueue(ref.Digest(), func() { m.buildImage(context.Background(), ref) }) @@ -483,7 +484,7 @@ func (m *manager) RecoverInterruptedBuilds() { } // Create a ResolvedRef since we already have the digest from metadata ref := NewResolvedRef(normalized, metaCopy.Digest) - m.queue.Enqueue(metaCopy.Digest, *metaCopy.Request, func() { + m.queue.Enqueue(metaCopy.Digest, func() { m.buildImage(context.Background(), ref) }) } diff --git a/lib/images/queue.go b/lib/images/queue.go deleted file mode 100644 index 5d08100a..00000000 --- a/lib/images/queue.go +++ /dev/null @@ -1,121 +0,0 @@ -package images - -import "sync" - -type QueuedBuild struct { - ImageName string - Request CreateImageRequest - StartFn func() -} - -// BuildQueue manages concurrent image builds with a configurable limit -type BuildQueue struct { - maxConcurrent int - active map[string]bool - pending []QueuedBuild - mu sync.Mutex -} - -func NewBuildQueue(maxConcurrent int) *BuildQueue { - if maxConcurrent < 1 { - maxConcurrent = 1 - } - return &BuildQueue{ - maxConcurrent: maxConcurrent, - active: make(map[string]bool), - pending: make([]QueuedBuild, 0), - } -} - -// Enqueue adds a build to the queue. Returns queue position (0 if started immediately, >0 if queued). -// If the image is already building or queued, returns its current position without re-enqueueing. -func (q *BuildQueue) Enqueue(imageName string, req CreateImageRequest, startFn func()) int { - q.mu.Lock() - defer q.mu.Unlock() - - // Check if already building (position 0, actively running) - if q.active[imageName] { - return 0 - } - - // Check if already in pending queue - for i, build := range q.pending { - if build.ImageName == imageName { - return i + 1 // Return existing queue position - } - } - - // Wrap the function to auto-complete - wrappedFn := func() { - defer q.MarkComplete(imageName) - startFn() - } - - build := QueuedBuild{ - ImageName: imageName, - Request: req, - StartFn: wrappedFn, - } - - if len(q.active) < q.maxConcurrent { - q.active[imageName] = true - go wrappedFn() - return 0 - } - - q.pending = append(q.pending, build) - return len(q.pending) -} - -func (q *BuildQueue) MarkComplete(imageName string) { - q.mu.Lock() - defer q.mu.Unlock() - - delete(q.active, imageName) - - if len(q.pending) > 0 && len(q.active) < q.maxConcurrent { - next := q.pending[0] - q.pending = q.pending[1:] - q.active[next.ImageName] = true - go next.StartFn() - } -} - -func (q *BuildQueue) GetPosition(imageName string) *int { - q.mu.Lock() - defer q.mu.Unlock() - - if q.active[imageName] { - return nil - } - - for i, build := range q.pending { - if build.ImageName == imageName { - pos := i + 1 - return &pos - } - } - - return nil -} - -// ActiveCount returns number of actively building images -func (q *BuildQueue) ActiveCount() int { - q.mu.Lock() - defer q.mu.Unlock() - return len(q.active) -} - -// PendingCount returns number of queued builds -func (q *BuildQueue) PendingCount() int { - q.mu.Lock() - defer q.mu.Unlock() - return len(q.pending) -} - -// QueueLength returns the total number of builds (active + pending) -func (q *BuildQueue) QueueLength() int { - q.mu.Lock() - defer q.mu.Unlock() - return len(q.active) + len(q.pending) -} diff --git a/lib/images/recovery_regression_test.go b/lib/images/recovery_regression_test.go index 2f832914..b979939a 100644 --- a/lib/images/recovery_regression_test.go +++ b/lib/images/recovery_regression_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/queue" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -45,7 +46,7 @@ func TestRecoverInterruptedBuildsCapturedFixtureMarksBuildFailed(t *testing.T) { m := &manager{ paths: p, ociClient: client, - queue: NewBuildQueue(1), + queue: queue.New(1), readySubscribers: make(map[string][]chan StatusEvent), } diff --git a/lib/queue/queue.go b/lib/queue/queue.go new file mode 100644 index 00000000..1dd59b0d --- /dev/null +++ b/lib/queue/queue.go @@ -0,0 +1,116 @@ +// Package queue provides a minimal in-memory bounded queue for keyed work +// with dedup: jobs are identified by a key, and enqueueing a key that is +// already active or pending returns its position instead of re-enqueueing. +package queue + +import "sync" + +type job struct { + key string + startFn func() +} + +// Queue runs jobs with a configurable concurrency limit. Queue state is +// in-memory; callers persist their own job metadata and recover on startup. +type Queue struct { + maxConcurrent int + active map[string]bool + pending []job + mu sync.Mutex +} + +// New creates a Queue with the given concurrency limit (minimum 1). +func New(maxConcurrent int) *Queue { + if maxConcurrent < 1 { + maxConcurrent = 1 + } + return &Queue{ + maxConcurrent: maxConcurrent, + active: make(map[string]bool), + pending: make([]job, 0), + } +} + +// Enqueue adds a job keyed by key. Returns the queue position: 0 if it +// started immediately, >0 if queued behind other jobs. If the key is already +// active or pending, returns its current position without re-enqueueing. +// An optional completion hook runs after the key leaves the active set, so a +// caller's bookkeeping for the key is torn down only once the queue is done +// with it. +func (q *Queue) Enqueue(key string, startFn func(), done ...func()) int { + q.mu.Lock() + defer q.mu.Unlock() + + if q.active[key] { + return 0 + } + for i, j := range q.pending { + if j.key == key { + return i + 1 + } + } + + wrappedFn := func() { + // complete runs first (last-registered defer runs first), so done + // only fires after the key has left the active set. + if len(done) == 1 && done[0] != nil { + defer done[0]() + } + defer q.complete(key) + startFn() + } + + if len(q.active) < q.maxConcurrent { + q.active[key] = true + go wrappedFn() + return 0 + } + + q.pending = append(q.pending, job{key: key, startFn: wrappedFn}) + return len(q.pending) +} + +// complete removes the key from the active set and starts the next pending +// job if there is capacity. +func (q *Queue) complete(key string) { + q.mu.Lock() + delete(q.active, key) + + var next *job + if len(q.pending) > 0 && len(q.active) < q.maxConcurrent { + nextJob := q.pending[0] + q.pending = q.pending[1:] + q.active[nextJob.key] = true + next = &nextJob + } + q.mu.Unlock() + + if next != nil { + go next.startFn() + } +} + +// GetPosition returns nil if the key is active or unknown, otherwise its +// 1-based position in the pending queue. +func (q *Queue) GetPosition(key string) *int { + q.mu.Lock() + defer q.mu.Unlock() + + if q.active[key] { + return nil + } + for i, j := range q.pending { + if j.key == key { + pos := i + 1 + return &pos + } + } + return nil +} + +// QueueLength returns the number of tracked jobs (active + pending). +func (q *Queue) QueueLength() int { + q.mu.Lock() + defer q.mu.Unlock() + return len(q.active) + len(q.pending) +} diff --git a/lib/queue/queue_test.go b/lib/queue/queue_test.go new file mode 100644 index 00000000..ca8b2fde --- /dev/null +++ b/lib/queue/queue_test.go @@ -0,0 +1,136 @@ +package queue + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +func TestConcurrencyLimit(t *testing.T) { + q := New(1) + + var mu sync.Mutex + running := 0 + maxRunning := 0 + release := make(chan struct{}) + done := make(chan struct{}, 2) + + startFn := func(block bool) func() { + return func() { + mu.Lock() + running++ + if running > maxRunning { + maxRunning = running + } + mu.Unlock() + + if block { + <-release + } + + mu.Lock() + running-- + mu.Unlock() + done <- struct{}{} + } + } + + posA := q.Enqueue("a", startFn(true)) + posB := q.Enqueue("b", startFn(false)) + if posA != 0 { + t.Errorf("posA = %d, want 0 (started immediately)", posA) + } + if posB != 1 { + t.Errorf("posB = %d, want 1 (queued behind a)", posB) + } + + if pos := q.GetPosition("b"); pos == nil || *pos != 1 { + t.Errorf("GetPosition(b) = %v, want 1", pos) + } + if pos := q.GetPosition("a"); pos != nil { + t.Errorf("GetPosition(a) = %v, want nil (active)", pos) + } + + close(release) + for i := 0; i < 2; i++ { + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for jobs") + } + } + + if maxRunning != 1 { + t.Errorf("maxRunning = %d, want 1", maxRunning) + } + if pos := q.GetPosition("b"); pos != nil { + t.Errorf("GetPosition(b) after completion = %v, want nil", pos) + } +} + +func TestDedupesByKey(t *testing.T) { + q := New(1) + release := make(chan struct{}) + started := make(chan struct{}, 1) + var ran int64 + + q.Enqueue("same", func() { + started <- struct{}{} + atomic.AddInt64(&ran, 1) + <-release + }) + <-started + + // Duplicate enqueues of an active key must not start another job. + q.Enqueue("same", func() { atomic.AddInt64(&ran, 1) }) + q.Enqueue("same", func() { atomic.AddInt64(&ran, 1) }) + if pos := q.GetPosition("same"); pos != nil { + t.Errorf("GetPosition(same) = %v, want nil (active)", pos) + } + + close(release) + time.Sleep(50 * time.Millisecond) + + if atomic.LoadInt64(&ran) != 1 { + t.Errorf("job ran %d times, want 1", ran) + } + if pos := q.GetPosition("same"); pos != nil { + t.Errorf("GetPosition after completion = %v, want nil", pos) + } +} + +// TestDoneRunsAfterKeyReleased guards the completion-hook ordering: done must +// fire only after the key has left the active set, otherwise a caller's +// "job finished" bookkeeping would race a concurrent re-enqueue of the key. +func TestDoneRunsAfterKeyReleased(t *testing.T) { + q := New(1) + + release := make(chan struct{}) + stateAtDone := make(chan int, 1) + + q.Enqueue("a", func() { <-release }, func() { + stateAtDone <- q.QueueLength() + }) + // "b" occupies the freed slot when "a" completes, so if the key "a" were + // still tracked at done time the queue would report length 2 (a + b). + q.Enqueue("b", func() {}) + + close(release) + select { + case n := <-stateAtDone: + if n != 1 { + t.Errorf("queue length when done ran = %d, want 1 (key a released, only b tracked)", n) + } + case <-time.After(5 * time.Second): + t.Fatal("done never ran") + } + + deadline := time.Now().Add(5 * time.Second) + for q.QueueLength() != 0 { + if time.Now().After(deadline) { + t.Fatal("b never completed") + } + time.Sleep(10 * time.Millisecond) + } +} From 888c9f3ba96871d024717cb20a42a222619a3525 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Fri, 7 Aug 2026 19:18:39 +0000 Subject: [PATCH 09/21] imagepush: dedupe test wait/assert scaffolding Add mustPushed and writePushes helpers to collapse the repeated wait-for-push + get + assert-pushed blocks (6 sites) and the recovery fixture metadata writes, without changing coverage. --- lib/imagepush/manager_test.go | 95 ++++++++++++----------------------- 1 file changed, 33 insertions(+), 62 deletions(-) diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index 13ad2966..a0cfc07e 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -162,6 +162,32 @@ func testManager(t *testing.T, maxConcurrent int, provider registrypush.Provider return mgr, digest } +// mustPushed waits for the push to reach a terminal pushed state and returns +// it, failing the test otherwise. +func mustPushed(t *testing.T, mgr Manager, id string) *Push { + t.Helper() + if err := mgr.WaitForPush(context.Background(), id); err != nil { + t.Fatalf("WaitForPush %s: %v", id, err) + } + got, err := mgr.GetPush(context.Background(), id) + if err != nil { + t.Fatalf("GetPush %s: %v", id, err) + } + if got.Status != StatusPushed { + t.Fatalf("push %s status = %s, want pushed (error: %v)", id, got.Status, got.Error) + } + return got +} + +func writePushes(t *testing.T, p *paths.Paths, metas ...*pushMetadata) { + t.Helper() + for _, meta := range metas { + if err := writeMetadata(p, meta); err != nil { + t.Fatalf("writeMetadata(%s): %v", meta.ID, err) + } + } +} + func TestCreatePushEndToEnd(t *testing.T) { mgr, digest := testManager(t, 2, nil, nil) host := openRegistry(t) @@ -175,17 +201,7 @@ func TestCreatePushEndToEnd(t *testing.T) { t.Errorf("initial status = %s, want queued or pushing", push.Status) } - if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { - t.Fatalf("WaitForPush: %v", err) - } - - got, err := mgr.GetPush(context.Background(), push.ID) - if err != nil { - t.Fatalf("GetPush: %v", err) - } - if got.Status != StatusPushed { - t.Errorf("status = %s, want pushed (error: %v)", got.Status, got.Error) - } + got := mustPushed(t, mgr, push.ID) if got.Digest != digest { t.Errorf("digest = %s, want %s", got.Digest, digest) } @@ -432,16 +448,7 @@ func TestRecoverInterruptedPushes(t *testing.T) { t.Fatalf("NewManager: %v", err) } - if err := mgr.WaitForPush(context.Background(), "recovered-push"); err != nil { - t.Fatalf("WaitForPush recovered: %v", err) - } - got, err := mgr.GetPush(context.Background(), "recovered-push") - if err != nil { - t.Fatalf("GetPush: %v", err) - } - if got.Status != StatusPushed { - t.Errorf("status = %s, want pushed", got.Status) - } + mustPushed(t, mgr, "recovered-push") } func TestCreatePushDedupesConcurrently(t *testing.T) { @@ -551,11 +558,7 @@ func TestRecoveryDedupesSameKey(t *testing.T) { older := &pushMetadata{ID: "older", Status: StatusQueued, Image: "myapp:v1", Digest: digest, Target: target, Insecure: true, CreatedAt: now.Add(-time.Minute)} newer := &pushMetadata{ID: "newer", Status: StatusQueued, Image: "myapp:v1", Digest: digest, Target: target, Insecure: true, CreatedAt: now} - for _, meta := range []*pushMetadata{older, newer} { - if err := writeMetadata(p, meta); err != nil { - t.Fatalf("writeMetadata(%s): %v", meta.ID, err) - } - } + writePushes(t, p, older, newer) resolver := &fakeResolver{images: map[string]*images.Image{ "myapp:v1": readyImage("myapp:v1", digest), @@ -593,16 +596,7 @@ func TestSequentialSameKeyPushesAllComplete(t *testing.T) { if err != nil { t.Fatalf("CreatePush #%d: %v", i, err) } - if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { - t.Fatalf("WaitForPush #%d: %v", i, err) - } - got, err := mgr.GetPush(context.Background(), push.ID) - if err != nil { - t.Fatalf("GetPush #%d: %v", i, err) - } - if got.Status != StatusPushed { - t.Fatalf("push #%d status = %s, want pushed (stuck job?)", i, got.Status) - } + mustPushed(t, mgr, push.ID) } } @@ -647,17 +641,7 @@ func TestCreatePushWithBorrowedCredentials(t *testing.T) { if err != nil { t.Fatalf("CreatePush: %v", err) } - if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { - t.Fatalf("WaitForPush: %v", err) - } - - got, err := mgr.GetPush(context.Background(), push.ID) - if err != nil { - t.Fatalf("GetPush: %v", err) - } - if got.Status != StatusPushed { - t.Errorf("status = %s, want pushed (error: %v)", got.Status, got.Error) - } + mustPushed(t, mgr, push.ID) } func TestCredentialsNeverPersisted(t *testing.T) { @@ -820,11 +804,7 @@ func TestRecoveryTreatsInsecureAsDistinctKey(t *testing.T) { secure := &pushMetadata{ID: "secure", Status: StatusQueued, Image: "myapp:v1", Digest: digest, Target: target, Insecure: false, CreatedAt: now.Add(-time.Minute)} insecure := &pushMetadata{ID: "insecure", Status: StatusQueued, Image: "myapp:v1", Digest: digest, Target: target, Insecure: true, CreatedAt: now} - for _, meta := range []*pushMetadata{secure, insecure} { - if err := writeMetadata(p, meta); err != nil { - t.Fatalf("writeMetadata(%s): %v", meta.ID, err) - } - } + writePushes(t, p, secure, insecure) resolver := &fakeResolver{images: map[string]*images.Image{ "myapp:v1": readyImage("myapp:v1", digest), @@ -837,15 +817,6 @@ func TestRecoveryTreatsInsecureAsDistinctKey(t *testing.T) { // Same digest+target with different transport modes is distinct work: // both jobs recover, neither is superseded. for _, id := range []string{"secure", "insecure"} { - if err := mgr.WaitForPush(context.Background(), id); err != nil { - t.Fatalf("WaitForPush %s: %v", id, err) - } - got, err := mgr.GetPush(context.Background(), id) - if err != nil { - t.Fatalf("GetPush %s: %v", id, err) - } - if got.Status != StatusPushed { - t.Errorf("%s status = %s, want pushed (error: %v)", id, got.Status, got.Error) - } + mustPushed(t, mgr, id) } } From d98dec5723c0425d21e19051246d17f1abb080a1 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:16:28 +0000 Subject: [PATCH 10/21] imagepush: address parallel-review findings - lib/queue: replace the variadic done hook with a plain nil-able param, document that a dedup'd enqueue does not run it, and note the under-lock launch is safe - treat an empty non-nil credential config as anonymous (credsPresent) - persist push metadata durably: fsync the temp file before rename - use a distinct (non-wrapped) error for empty-digest-on-ready, a corrupted-record state rather than not-ready - sort InProgressDigests for determinism Tests: assert pending QueuePosition via the manager read surface, verify WaitForPush on a superseded recovered job fails, and cover recovery when blobs were reclaimed by GC between crash and restart. --- lib/imagepush/imagepush.go | 9 +++++++ lib/imagepush/manager.go | 14 ++++++---- lib/imagepush/manager_test.go | 50 +++++++++++++++++++++++++++++++++++ lib/imagepush/storage.go | 19 ++++++++++++- lib/images/manager.go | 4 +-- lib/queue/queue.go | 17 +++++++----- lib/queue/queue_test.go | 12 ++++----- 7 files changed, 104 insertions(+), 21 deletions(-) diff --git a/lib/imagepush/imagepush.go b/lib/imagepush/imagepush.go index 3fb6bd03..b4bc8545 100644 --- a/lib/imagepush/imagepush.go +++ b/lib/imagepush/imagepush.go @@ -52,6 +52,15 @@ type PushRequest struct { Credentials *authn.AuthConfig } +// credsPresent reports whether a credential config carries any credential +// material; an empty non-nil config is treated as anonymous. +func credsPresent(c *authn.AuthConfig) bool { + if c == nil { + return false + } + return c.Username != "" || c.Password != "" || c.Auth != "" || c.IdentityToken != "" || c.RegistryToken != "" +} + // Push is the state of one push job. type Push struct { ID string diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index f5f48ce6..d0be5063 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "sort" "strings" "sync" "time" @@ -82,9 +83,11 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error return nil, fmt.Errorf("%w: %s is %s", ErrImageNotReady, img.Name, img.Status) } // A ready image must carry a digest; without one the dedup key below - // would collide across unrelated images. + // would collide across unrelated images. This is a corrupted-record + // state rather than "not ready", so it gets a distinct (non-wrapped) + // error. if img.Digest == "" { - return nil, fmt.Errorf("%w: image %s has no digest", ErrImageNotReady, img.Name) + return nil, fmt.Errorf("image %s is ready but has no digest", img.Name) } // Validate the target before persisting anything so typos fail fast. @@ -102,7 +105,7 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error // Borrowed credentials live only in this closure: the job provider is // built per push and never touches disk. provider := m.provider - if req.Credentials != nil { + if credsPresent(req.Credentials) { provider = ®istrypush.StaticProvider{Config: *req.Credentials} } @@ -119,7 +122,7 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error // silently inherit another caller's login). Both silently merge under // the wrong auth otherwise; surface the conflict instead so the caller // can retry once the in-flight job completes or match its credentials. - hadCreds := req.Credentials != nil + hadCreds := credsPresent(req.Credentials) if existing.hadCredentials != hadCreds { m.mu.Unlock() return nil, fmt.Errorf("%w: a push of %s to %s is already in flight with different credentials; retry once it completes or match its credentials", ErrCredentialConflict, img.Digest, dstRef.String()) @@ -136,7 +139,7 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error Digest: img.Digest, Target: dstRef.String(), Insecure: req.Insecure, - HadCredentials: req.Credentials != nil, + HadCredentials: credsPresent(req.Credentials), CreatedAt: time.Now(), } if err := writeMetadata(m.paths, meta); err != nil { @@ -347,6 +350,7 @@ func (m *manager) InProgressDigests() []string { seen[job.digest] = struct{}{} digests = append(digests, job.digest) } + sort.Strings(digests) return digests } diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index a0cfc07e..04566eb3 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -296,6 +296,14 @@ func TestCreatePushQueuesBehindConcurrencyLimit(t *testing.T) { if second.QueuePosition == nil || *second.QueuePosition != 1 { t.Errorf("second queue position = %v, want 1", second.QueuePosition) } + // The manager read surface reports the same pending position. + got, err := mgr.GetPush(context.Background(), second.ID) + if err != nil { + t.Fatalf("GetPush pending: %v", err) + } + if got.QueuePosition == nil || *got.QueuePosition != 1 { + t.Errorf("GetPush pending queue position = %v, want 1", got.QueuePosition) + } close(gate) if err := mgr.WaitForPush(context.Background(), first.ID); err != nil { @@ -582,6 +590,10 @@ func TestRecoveryDedupesSameKey(t *testing.T) { if got.Error == nil || !strings.Contains(*got.Error, "superseded") { t.Errorf("newer error = %v, want superseded explanation", got.Error) } + // WaitForPush on the superseded job surfaces the failure rather than hanging. + if err := mgr.WaitForPush(context.Background(), "newer"); err == nil { + t.Error("WaitForPush on superseded job should fail") + } } func TestSequentialSameKeyPushesAllComplete(t *testing.T) { @@ -758,6 +770,44 @@ func TestCreatePushMissingBlobs(t *testing.T) { } } +// A push interrupted before a restart whose blobs were reclaimed by GC in the +// meantime fails the same way on recovery instead of hanging or wedging the slot. +func TestRecoveryFailsWhenBlobsReclaimed(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + + meta := &pushMetadata{ + ID: "recovered-missing-blobs", + Status: StatusPushing, + Image: "myapp:v1", + Digest: digest, + Target: host + "/export/recovered:v1", + Insecure: true, + CreatedAt: time.Now(), + } + writePushes(t, p, meta) + if err := os.RemoveAll(p.OCICacheBlobDir()); err != nil { + t.Fatalf("remove blobs: %v", err) + } + + mgr, err := NewManager(p, &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }}, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err = mgr.WaitForPush(ctx, "recovered-missing-blobs") + if err == nil { + t.Fatal("WaitForPush should fail when cache blobs were reclaimed") + } + if !errors.Is(err, ocicache.ErrNotFound) && !strings.Contains(err.Error(), ocicache.ErrNotFound.Error()) { + t.Errorf("err = %v, want ocicache.ErrNotFound", err) + } +} + // panickingProvider blows up during credential resolution, standing in for a // panic anywhere in the push path. type panickingProvider struct{} diff --git a/lib/imagepush/storage.go b/lib/imagepush/storage.go index 9544c66e..f32eacc2 100644 --- a/lib/imagepush/storage.go +++ b/lib/imagepush/storage.go @@ -57,9 +57,26 @@ func writeMetadata(p *paths.Paths, meta *pushMetadata) error { } tempPath := p.PushMetadata(meta.ID) + ".tmp" - if err := os.WriteFile(tempPath, data, 0644); err != nil { + file, err := os.OpenFile(tempPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("create temp metadata: %w", err) + } + if _, err := file.Write(data); err != nil { + file.Close() + os.Remove(tempPath) return fmt.Errorf("write temp metadata: %w", err) } + // Sync before rename so a crash cannot leave an empty/partial final file + // in place of a durably-written one. + if err := file.Sync(); err != nil { + file.Close() + os.Remove(tempPath) + return fmt.Errorf("sync temp metadata: %w", err) + } + if err := file.Close(); err != nil { + os.Remove(tempPath) + return fmt.Errorf("close temp metadata: %w", err) + } finalPath := p.PushMetadata(meta.ID) if err := os.Rename(tempPath, finalPath); err != nil { diff --git a/lib/images/manager.go b/lib/images/manager.go index 58a1bac0..e70146ce 100644 --- a/lib/images/manager.go +++ b/lib/images/manager.go @@ -268,7 +268,7 @@ func (m *manager) createAndQueueImage(ref *ResolvedRef, req CreateImageRequest, // Enqueue the build using digest as the queue key for deduplication queuePos := m.queue.Enqueue(ref.Digest(), func() { m.buildImage(context.Background(), ref) - }) + }, nil) img := meta.toImage() if queuePos > 0 { @@ -486,7 +486,7 @@ func (m *manager) RecoverInterruptedBuilds() { ref := NewResolvedRef(normalized, metaCopy.Digest) m.queue.Enqueue(metaCopy.Digest, func() { m.buildImage(context.Background(), ref) - }) + }, nil) } } } diff --git a/lib/queue/queue.go b/lib/queue/queue.go index 1dd59b0d..f9cb84e6 100644 --- a/lib/queue/queue.go +++ b/lib/queue/queue.go @@ -33,11 +33,12 @@ func New(maxConcurrent int) *Queue { // Enqueue adds a job keyed by key. Returns the queue position: 0 if it // started immediately, >0 if queued behind other jobs. If the key is already -// active or pending, returns its current position without re-enqueueing. -// An optional completion hook runs after the key leaves the active set, so a -// caller's bookkeeping for the key is torn down only once the queue is done -// with it. -func (q *Queue) Enqueue(key string, startFn func(), done ...func()) int { +// active or pending, Enqueue dedups and returns its current position without +// re-enqueueing. A completion hook may be provided; when non-nil it runs +// after the key leaves the active set (so a caller's bookkeeping for the key +// is torn down only once the queue is done with it), but only when this call +// actually started the job — a dedup'd enqueue does not run it. +func (q *Queue) Enqueue(key string, startFn func(), done func()) int { q.mu.Lock() defer q.mu.Unlock() @@ -53,8 +54,8 @@ func (q *Queue) Enqueue(key string, startFn func(), done ...func()) int { wrappedFn := func() { // complete runs first (last-registered defer runs first), so done // only fires after the key has left the active set. - if len(done) == 1 && done[0] != nil { - defer done[0]() + if done != nil { + defer done() } defer q.complete(key) startFn() @@ -62,6 +63,8 @@ func (q *Queue) Enqueue(key string, startFn func(), done ...func()) int { if len(q.active) < q.maxConcurrent { q.active[key] = true + // Safe to launch under the lock: wrappedFn blocks on complete's lock + // until Enqueue returns, so no key can start before its slot is held. go wrappedFn() return 0 } diff --git a/lib/queue/queue_test.go b/lib/queue/queue_test.go index ca8b2fde..811ad0cb 100644 --- a/lib/queue/queue_test.go +++ b/lib/queue/queue_test.go @@ -36,8 +36,8 @@ func TestConcurrencyLimit(t *testing.T) { } } - posA := q.Enqueue("a", startFn(true)) - posB := q.Enqueue("b", startFn(false)) + posA := q.Enqueue("a", startFn(true), nil) + posB := q.Enqueue("b", startFn(false), nil) if posA != 0 { t.Errorf("posA = %d, want 0 (started immediately)", posA) } @@ -79,12 +79,12 @@ func TestDedupesByKey(t *testing.T) { started <- struct{}{} atomic.AddInt64(&ran, 1) <-release - }) + }, nil) <-started // Duplicate enqueues of an active key must not start another job. - q.Enqueue("same", func() { atomic.AddInt64(&ran, 1) }) - q.Enqueue("same", func() { atomic.AddInt64(&ran, 1) }) + q.Enqueue("same", func() { atomic.AddInt64(&ran, 1) }, nil) + q.Enqueue("same", func() { atomic.AddInt64(&ran, 1) }, nil) if pos := q.GetPosition("same"); pos != nil { t.Errorf("GetPosition(same) = %v, want nil (active)", pos) } @@ -114,7 +114,7 @@ func TestDoneRunsAfterKeyReleased(t *testing.T) { }) // "b" occupies the freed slot when "a" completes, so if the key "a" were // still tracked at done time the queue would report length 2 (a + b). - q.Enqueue("b", func() {}) + q.Enqueue("b", func() {}, nil) close(release) select { From da865ae6f2b00f1fdb3c979f3fa2aca4cc10e0be Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:50:19 +0000 Subject: [PATCH 11/21] imagepush: deflake queue completion-hook ordering test --- lib/queue/queue_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/queue/queue_test.go b/lib/queue/queue_test.go index 811ad0cb..c4cb602a 100644 --- a/lib/queue/queue_test.go +++ b/lib/queue/queue_test.go @@ -108,13 +108,17 @@ func TestDoneRunsAfterKeyReleased(t *testing.T) { release := make(chan struct{}) stateAtDone := make(chan int, 1) + doneSampled := make(chan struct{}) q.Enqueue("a", func() { <-release }, func() { stateAtDone <- q.QueueLength() + close(doneSampled) }) // "b" occupies the freed slot when "a" completes, so if the key "a" were - // still tracked at done time the queue would report length 2 (a + b). - q.Enqueue("b", func() {}, nil) + // still tracked at done time the queue would report length 2 (a + b). It + // blocks until the done hook has sampled so it cannot finish early and + // flake the assertion. + q.Enqueue("b", func() { <-doneSampled }, nil) close(release) select { From 2c7f833cff6ca07dc0bb75eaaf9cf708e7dcbbb8 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:36:41 +0000 Subject: [PATCH 12/21] images: fix build-phase metrics test after build-queue consolidation --- lib/images/metrics_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/images/metrics_test.go b/lib/images/metrics_test.go index 01e70e99..93007b30 100644 --- a/lib/images/metrics_test.go +++ b/lib/images/metrics_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/queue" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel/attribute" otelmetric "go.opentelemetry.io/otel/sdk/metric" @@ -17,7 +18,7 @@ func TestImageBuildPhaseMetrics(t *testing.T) { provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) m := &manager{ paths: paths.New(t.TempDir()), - queue: NewBuildQueue(1), + queue: queue.New(1), } metrics, err := newMetrics(provider.Meter("test"), m) From 68b750e417ee8b9e3155d41801abb4c62338636c Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:14:36 +0000 Subject: [PATCH 13/21] imagepush: address review findings on dedup, recovery, and tests --- lib/imagepush/imagepush.go | 14 +++ lib/imagepush/manager.go | 132 +++++++++++++++++-------- lib/imagepush/manager_test.go | 176 ++++++++++++++++++++++++++++++++-- lib/imagepush/storage.go | 36 ++++++- lib/queue/queue.go | 7 +- lib/queue/queue_test.go | 7 +- 6 files changed, 319 insertions(+), 53 deletions(-) diff --git a/lib/imagepush/imagepush.go b/lib/imagepush/imagepush.go index b4bc8545..3ed18f1d 100644 --- a/lib/imagepush/imagepush.go +++ b/lib/imagepush/imagepush.go @@ -9,7 +9,10 @@ package imagepush import ( "context" + "crypto/sha256" + "encoding/hex" "errors" + "strings" "time" "github.com/google/go-containerregistry/pkg/authn" @@ -61,6 +64,17 @@ func credsPresent(c *authn.AuthConfig) bool { return c.Username != "" || c.Password != "" || c.Auth != "" || c.IdentityToken != "" || c.RegistryToken != "" } +// credFingerprint hashes borrowed credentials so in-flight dedup can tell +// "same login" from "different login" without retaining the secret material. +// Anonymous configs (nil or empty) all share the empty fingerprint. +func credFingerprint(c *authn.AuthConfig) string { + if !credsPresent(c) { + return "" + } + sum := sha256.Sum256([]byte(strings.Join([]string{c.Username, c.Password, c.Auth, c.IdentityToken, c.RegistryToken}, "\x00"))) + return hex.EncodeToString(sum[:]) +} + // Push is the state of one push job. type Push struct { ID string diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index d0be5063..2da0d10b 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -19,9 +19,9 @@ import ( ) type inflightPush struct { - id string - digest string - hadCredentials bool + id string + digest string + credFingerprint string } type manager struct { @@ -101,6 +101,7 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error } key := pushKey(img.Digest, dstRef.String(), req.Insecure) + fingerprint := credFingerprint(req.Credentials) // Borrowed credentials live only in this closure: the job provider is // built per push and never touches disk. @@ -111,43 +112,62 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error // Hold the lock across the dedup check, metadata write, and registration so // a concurrent request for the same digest+target cannot slip in between - // and create a duplicate job. - m.mu.Lock() - if existing, ok := m.inflight[key]; ok { - // Merge only when the credential intent matches the in-flight job. The - // manager never stores credential values, so it can only compare - // presence: a request that borrowed credentials cannot merge into an - // anonymous in-flight push (its auth would be silently dropped), and an - // anonymous request cannot merge into a credentialed one (it would - // silently inherit another caller's login). Both silently merge under - // the wrong auth otherwise; surface the conflict instead so the caller - // can retry once the in-flight job completes or match its credentials. - hadCreds := credsPresent(req.Credentials) - if existing.hadCredentials != hadCreds { + // and create a duplicate job. The write is one small fsync'd file; keeping + // it under the lock is what lets the dedup path hand back a durable record, + // and it only briefly stalls InProgressDigests — cheap next to the registry + // I/O that dominates a push. + var meta *pushMetadata + for { + m.mu.Lock() + if existing, ok := m.inflight[key]; ok { + // Merge only when the in-flight job runs under the same credentials + // as the request. The manager never stores credential values, so it + // compares fingerprints: a request that borrowed credentials cannot + // merge into an anonymous in-flight push (its auth would be silently + // dropped), an anonymous request cannot merge into a credentialed one + // (it would silently inherit another caller's login), and two + // requests that borrowed different logins cannot merge either — one + // would run under the other caller's auth, and an instance can serve + // more than one principal. Surface the conflict instead so the caller + // can retry once the in-flight job completes or match its credentials. + if existing.credFingerprint != fingerprint { + m.mu.Unlock() + return nil, fmt.Errorf("%w: a push of %s to %s is already in flight with different credentials; retry once it completes or match its credentials", ErrCredentialConflict, img.Digest, dstRef.String()) + } + id := existing.id m.mu.Unlock() - return nil, fmt.Errorf("%w: a push of %s to %s is already in flight with different credentials; retry once it completes or match its credentials", ErrCredentialConflict, img.Digest, dstRef.String()) + push, err := m.GetPush(ctx, id) + if errors.Is(err, ErrNotFound) { + // The job's terminal record could not be persisted and its + // directory was dropped; the queue completion hook releases the + // inflight entry moments later. Wait it out and retry the dedup + // rather than surface a bare ErrNotFound from a create call. + if err := m.waitForInflightRelease(ctx, key); err != nil { + return nil, err + } + continue + } + return push, err } - id := existing.id - m.mu.Unlock() - return m.GetPush(ctx, id) - } - meta := &pushMetadata{ - ID: cuid2.Generate(), - Status: StatusQueued, - Image: img.Name, - Digest: img.Digest, - Target: dstRef.String(), - Insecure: req.Insecure, - HadCredentials: credsPresent(req.Credentials), - CreatedAt: time.Now(), - } - if err := writeMetadata(m.paths, meta); err != nil { + meta = &pushMetadata{ + ID: cuid2.Generate(), + Status: StatusQueued, + Image: img.Name, + Digest: img.Digest, + Target: dstRef.String(), + Insecure: req.Insecure, + HadCredentials: credsPresent(req.Credentials), + CreatedAt: time.Now(), + } + if err := writeMetadata(m.paths, meta); err != nil { + m.mu.Unlock() + return nil, fmt.Errorf("write initial metadata: %w", err) + } + m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest, credFingerprint: fingerprint} m.mu.Unlock() - return nil, fmt.Errorf("write initial metadata: %w", err) + break } - m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest, hadCredentials: meta.HadCredentials} - m.mu.Unlock() metaCopy := *meta queuePos := m.queue.Enqueue(key, func() { @@ -249,6 +269,28 @@ func (m *manager) releaseInflight(key string) func() { } } +// waitForInflightRelease blocks until the key's inflight entry is dropped. +// The queue releases the key's active slot before it runs the completion +// hook, so once the entry is gone a fresh Enqueue for the key starts +// immediately. Only useful in the narrow window where a job's record was +// dropped by the persist-failure path; the hook runs moments later, and the +// caller's context bounds the wait. +func (m *manager) waitForInflightRelease(ctx context.Context, key string) error { + for { + m.mu.Lock() + _, ok := m.inflight[key] + m.mu.Unlock() + if !ok { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(5 * time.Millisecond): + } + } +} + func (m *manager) GetPush(ctx context.Context, id string) (*Push, error) { if err := ctx.Err(); err != nil { return nil, err @@ -355,6 +397,11 @@ func (m *manager) InProgressDigests() []string { } func (m *manager) recoverInterruptedPushes() { + // A crash between the push-dir MkdirAll and the metadata rename leaves an + // empty dir no listing can read and no recovery can act on; sweep it now, + // while no CreatePush can be mid-write. + removeOrphanedPushDirs(m.paths) + pending, err := listPendingPushes(m.paths) if err != nil { // Loud on purpose: without recovery these records stay queued on disk @@ -375,17 +422,22 @@ func (m *manager) recoverInterruptedPushes() { continue } - // Duplicate records for one logical push can accumulate on disk; - // recover the oldest and close the rest so nothing is left forever - // queued. + // Duplicate records for one logical push can accumulate on disk. + // listPendingPushes runs oldest-first, so the first record seen for a + // key is the original request: recover that one and close the rest so + // nothing is left forever queued. Oldest wins because it is the record + // a waiter is most likely to hold, and a stable choice matters more + // than which duplicate happened to be written last. if chosen, ok := seen[key]; ok { - m.failRecovered(meta, fmt.Sprintf("superseded by duplicate push job %s for the same image and target", chosen)) + m.failRecovered(meta, fmt.Sprintf("duplicate of push job %s for the same image and target (oldest record wins)", chosen)) continue } seen[key] = meta.ID + // Recovered jobs are anonymous by policy (credentialed ones are failed + // above), so their credential fingerprint is the empty string. m.mu.Lock() - m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest, hadCredentials: meta.HadCredentials} + m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest} m.mu.Unlock() metaCopy := *meta diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index 04566eb3..2583231d 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -464,12 +464,11 @@ func TestCreatePushDedupesConcurrently(t *testing.T) { host, gate := gatedRegistry(t) req := PushRequest{Image: "myapp:v1", Target: host + "/export/app:v1", Insecure: true} - if _, err := mgr.CreatePush(context.Background(), req); err != nil { - t.Fatalf("seed CreatePush: %v", err) - } - // Two goroutines racing CreatePush for the same digest+target must both - // land on the single in-flight job: one registers, the other merges. + // Two goroutines racing CreatePush for the same digest+target — including + // the registration itself — must both land on the single job: one + // registers, the other merges. The gated registry keeps the winner in + // flight so the race cannot resolve before both calls return. const n = 2 ids := make([]string, n) errs := make([]error, n) @@ -558,6 +557,137 @@ func TestCreatePushCredentialConflict(t *testing.T) { } } +func TestCreatePushCredentialMismatch(t *testing.T) { + mgr, _ := testManager(t, 1, nil, nil) + host, gate := gatedRegistry(t) + target := host + "/export/app:v1" + + seeded, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: target, Insecure: true, + Credentials: &authn.AuthConfig{RegistryToken: "token-a"}, + }) + if err != nil { + t.Fatalf("seed CreatePush: %v", err) + } + + // A different borrowed login for the same in-flight work must conflict: + // merging would run the request under token-a's principal. + if _, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: target, Insecure: true, + Credentials: &authn.AuthConfig{RegistryToken: "token-b"}, + }); !errors.Is(err, ErrCredentialConflict) { + t.Errorf("mismatched-credential duplicate err = %v, want ErrCredentialConflict", err) + } + + // The same borrowed login merges into the in-flight job. + dup, err := mgr.CreatePush(context.Background(), PushRequest{ + Image: "myapp:v1", Target: target, Insecure: true, + Credentials: &authn.AuthConfig{RegistryToken: "token-a"}, + }) + if err != nil { + t.Fatalf("same-credential duplicate: %v", err) + } + if dup.ID != seeded.ID { + t.Errorf("same-credential duplicate got ID %s, want %s", dup.ID, seeded.ID) + } + + close(gate) + if err := mgr.WaitForPush(context.Background(), seeded.ID); err != nil { + t.Fatalf("WaitForPush: %v", err) + } +} + +func TestCreatePushDedupSurvivesTornDownKey(t *testing.T) { + mgr, digest := testManager(t, 1, nil, nil) + host := openRegistry(t) + target := host + "/export/app:v1" + + dstRef, err := name.ParseReference(target, name.Insecure) + if err != nil { + t.Fatalf("ParseReference: %v", err) + } + key := pushKey(digest, dstRef.String(), true) + + // Simulate the persist-failure teardown window: the job's record is gone + // from disk but its inflight entry is still registered, released by the + // queue completion hook moments later. + m := mgr.(*manager) + m.mu.Lock() + m.inflight[key] = inflightPush{id: "ghost", digest: digest} + m.mu.Unlock() + go func() { + time.Sleep(50 * time.Millisecond) + m.mu.Lock() + delete(m.inflight, key) + m.mu.Unlock() + }() + + // The dedup path must wait out the torn-down key and create a fresh job, + // not surface ErrNotFound from a create. + push, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: target, Insecure: true}) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + if push.ID == "ghost" { + t.Fatal("CreatePush returned the torn-down job") + } + mustPushed(t, mgr, push.ID) +} + +func TestWaitForPushCancellation(t *testing.T) { + mgr, _ := testManager(t, 1, nil, nil) + host, gate := gatedRegistry(t) + + push, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: host + "/export/app:v1", Insecure: true}) + if err != nil { + t.Fatalf("CreatePush: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + errCh := make(chan error, 1) + go func() { errCh <- mgr.WaitForPush(ctx, push.ID) }() + cancel() + if err := <-errCh; !errors.Is(err, context.Canceled) { + t.Errorf("WaitForPush err = %v, want context.Canceled", err) + } + + // Let the in-flight job finish so its writes land before the fixture's + // TempDir cleanup. + close(gate) + if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { + t.Fatalf("WaitForPush after cancel: %v", err) + } +} + +func TestInProgressDigestsDedupesAcrossTargets(t *testing.T) { + mgr, digest := testManager(t, 2, nil, nil) + hostA, gateA := gatedRegistry(t) + hostB, gateB := gatedRegistry(t) + + // The same image pushed to two targets is two jobs but one live digest: + // the GC needs the digest kept alive once, not per target. + pushes := make([]string, 0, 2) + for _, target := range []string{hostA + "/export/a:v1", hostB + "/export/b:v1"} { + push, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: target, Insecure: true}) + if err != nil { + t.Fatalf("CreatePush %s: %v", target, err) + } + pushes = append(pushes, push.ID) + } + + if digests := mgr.InProgressDigests(); len(digests) != 1 || digests[0] != digest { + t.Errorf("InProgressDigests = %v, want [%s]", digests, digest) + } + + // Drain the gated jobs so their writes land before the fixture's TempDir + // cleanup. + close(gateA) + close(gateB) + for _, id := range pushes { + mustPushed(t, mgr, id) + } +} + func TestRecoveryDedupesSameKey(t *testing.T) { p, digest := cacheFixture(t) host := openRegistry(t) @@ -587,8 +717,8 @@ func TestRecoveryDedupesSameKey(t *testing.T) { if got.Status != StatusFailed { t.Errorf("newer status = %s, want failed (superseded)", got.Status) } - if got.Error == nil || !strings.Contains(*got.Error, "superseded") { - t.Errorf("newer error = %v, want superseded explanation", got.Error) + if got.Error == nil || !strings.Contains(*got.Error, "duplicate of push job older") { + t.Errorf("newer error = %v, want duplicate-of-older explanation", got.Error) } // WaitForPush on the superseded job surfaces the failure rather than hanging. if err := mgr.WaitForPush(context.Background(), "newer"); err == nil { @@ -865,8 +995,38 @@ func TestRecoveryTreatsInsecureAsDistinctKey(t *testing.T) { } // Same digest+target with different transport modes is distinct work: - // both jobs recover, neither is superseded. + // both jobs recover, neither is superseded. Both pushes still go over + // plain HTTP here: go-containerregistry's Registry.Scheme maps loopback + // and RFC1918 hosts to http on its own, so an httptest registry cannot + // exercise the secure transport — what this pins down is the key + // distinction, not the wire behavior. for _, id := range []string{"secure", "insecure"} { mustPushed(t, mgr, id) } } + +func TestRecoverySweepsOrphanDirs(t *testing.T) { + p, digest := cacheFixture(t) + host := openRegistry(t) + + meta := &pushMetadata{ID: "real", Status: StatusQueued, Image: "myapp:v1", Digest: digest, Target: host + "/export/real:v1", Insecure: true, CreatedAt: time.Now()} + writePushes(t, p, meta) + // A crash between the push-dir MkdirAll and the metadata rename leaves an + // empty dir; startup recovery sweeps it since it holds no record. + if err := os.MkdirAll(p.PushDir("orphan"), 0755); err != nil { + t.Fatalf("mkdir orphan: %v", err) + } + + resolver := &fakeResolver{images: map[string]*images.Image{ + "myapp:v1": readyImage("myapp:v1", digest), + }} + mgr, err := NewManager(p, resolver, nil, 1) + if err != nil { + t.Fatalf("NewManager: %v", err) + } + + if _, err := os.Stat(p.PushDir("orphan")); !os.IsNotExist(err) { + t.Errorf("orphan dir still exists after recovery (stat err = %v)", err) + } + mustPushed(t, mgr, "real") +} diff --git a/lib/imagepush/storage.go b/lib/imagepush/storage.go index f32eacc2..a269f374 100644 --- a/lib/imagepush/storage.go +++ b/lib/imagepush/storage.go @@ -2,6 +2,7 @@ package imagepush import ( "encoding/json" + "errors" "fmt" "os" "sort" @@ -121,6 +122,13 @@ func listAllPushes(p *paths.Paths) ([]*pushMetadata, error) { } meta, err := readMetadata(p, entry.Name()) if err != nil { + // A missing metadata.json is not an anomaly: the dir belongs to a + // CreatePush writing right now or to a crash orphan swept at the + // next startup. Warning here would fire on every list for as long + // as the dir lingers. + if errors.Is(err, ErrNotFound) { + continue + } // Surface unreadable records instead of swallowing them: a corrupt // or half-written metadata.json would otherwise vanish from // listing and recovery while its push directory lingers on disk. @@ -137,6 +145,26 @@ func listAllPushes(p *paths.Paths) ([]*pushMetadata, error) { return metas, nil } +// removeOrphanedPushDirs deletes push directories that never received a +// metadata.json (a crash between MkdirAll and the metadata rename). They hold +// no record to recover and would otherwise linger on disk forever. Only safe +// at startup, before any CreatePush can be mid-write. +func removeOrphanedPushDirs(p *paths.Paths) { + entries, err := os.ReadDir(p.PushesDir()) + if err != nil { + return // nothing to sweep; listing errors surface in recovery proper + } + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if _, err := os.Stat(p.PushMetadata(entry.Name())); errors.Is(err, os.ErrNotExist) { + fmt.Fprintf(os.Stderr, "Warning: removing orphaned push directory %s (no metadata.json)\n", entry.Name()) + os.RemoveAll(p.PushDir(entry.Name())) + } + } +} + // listPendingPushes returns pushes that did not reach a terminal state, // oldest first for FIFO recovery. func listPendingPushes(p *paths.Paths) ([]*pushMetadata, error) { @@ -153,7 +181,13 @@ func listPendingPushes(p *paths.Paths) ([]*pushMetadata, error) { } } - sort.Slice(pending, func(i, j int) bool { + sort.SliceStable(pending, func(i, j int) bool { + // CreatedAt ties are broken by ID so recovery order — and with it the + // "oldest record wins" dedup in recoverInterruptedPushes — is + // deterministic across restarts. + if pending[i].CreatedAt.Equal(pending[j].CreatedAt) { + return pending[i].ID < pending[j].ID + } return pending[i].CreatedAt.Before(pending[j].CreatedAt) }) diff --git a/lib/queue/queue.go b/lib/queue/queue.go index f9cb84e6..3cd580c1 100644 --- a/lib/queue/queue.go +++ b/lib/queue/queue.go @@ -63,8 +63,11 @@ func (q *Queue) Enqueue(key string, startFn func(), done func()) int { if len(q.active) < q.maxConcurrent { q.active[key] = true - // Safe to launch under the lock: wrappedFn blocks on complete's lock - // until Enqueue returns, so no key can start before its slot is held. + // The key enters the active set before the goroutine launches, both + // under the lock, so a concurrent Enqueue always observes the slot and + // dedups. The goroutine may run to completion before Enqueue returns, + // but its deferred complete then blocks on q.mu, so every state + // transition still serializes after this call. go wrappedFn() return 0 } diff --git a/lib/queue/queue_test.go b/lib/queue/queue_test.go index c4cb602a..95b838b2 100644 --- a/lib/queue/queue_test.go +++ b/lib/queue/queue_test.go @@ -73,13 +73,14 @@ func TestDedupesByKey(t *testing.T) { q := New(1) release := make(chan struct{}) started := make(chan struct{}, 1) + done := make(chan struct{}) var ran int64 q.Enqueue("same", func() { started <- struct{}{} atomic.AddInt64(&ran, 1) <-release - }, nil) + }, func() { close(done) }) <-started // Duplicate enqueues of an active key must not start another job. @@ -90,7 +91,9 @@ func TestDedupesByKey(t *testing.T) { } close(release) - time.Sleep(50 * time.Millisecond) + // The completion hook runs after the key leaves the active set, so once + // it fires the job is fully drained — no sleep-based guesswork. + <-done if atomic.LoadInt64(&ran) != 1 { t.Errorf("job ran %d times, want 1", ran) From f461185ad05ebc8641f7da63b6216d95415757f5 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:42:41 +0000 Subject: [PATCH 14/21] imagepush: merge dedup waiters into successor instead of re-pushing --- lib/imagepush/manager.go | 29 ++++++++++------- lib/imagepush/manager_test.go | 60 +++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 12 deletions(-) diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index 2da0d10b..21f4d891 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -140,9 +140,12 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error if errors.Is(err, ErrNotFound) { // The job's terminal record could not be persisted and its // directory was dropped; the queue completion hook releases the - // inflight entry moments later. Wait it out and retry the dedup - // rather than surface a bare ErrNotFound from a create call. - if err := m.waitForInflightRelease(ctx, key); err != nil { + // inflight entry moments later. Wait for that entry (not just the + // key) to go away, then retry the dedup: a concurrent waiter that + // got here first may already have registered a successor, which + // this retry merges into instead of surfacing a bare ErrNotFound + // from a create call. + if err := m.waitForInflightRelease(ctx, key, id); err != nil { return nil, err } continue @@ -269,18 +272,20 @@ func (m *manager) releaseInflight(key string) func() { } } -// waitForInflightRelease blocks until the key's inflight entry is dropped. -// The queue releases the key's active slot before it runs the completion -// hook, so once the entry is gone a fresh Enqueue for the key starts -// immediately. Only useful in the narrow window where a job's record was -// dropped by the persist-failure path; the hook runs moments later, and the -// caller's context bounds the wait. -func (m *manager) waitForInflightRelease(ctx context.Context, key string) error { +// waitForInflightRelease blocks until the key's torn-down inflight entry is +// dropped — or replaced by a successor job a concurrent create registered +// first, which the caller then merges into by retrying the dedup. Waiting on +// the entry's id rather than the key's absence is what keeps a second waiter +// from parking until the successor finishes and then starting a duplicate +// push. The queue releases the key's active slot before it runs the +// completion hook, so once the torn-down entry is gone a fresh Enqueue for +// the key starts immediately. The caller's context bounds the wait. +func (m *manager) waitForInflightRelease(ctx context.Context, key, tornDownID string) error { for { m.mu.Lock() - _, ok := m.inflight[key] + existing, ok := m.inflight[key] m.mu.Unlock() - if !ok { + if !ok || existing.id != tornDownID { return nil } select { diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index 2583231d..d77e9145 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -634,6 +634,66 @@ func TestCreatePushDedupSurvivesTornDownKey(t *testing.T) { mustPushed(t, mgr, push.ID) } +func TestCreatePushDedupWaitersMergeIntoSuccessor(t *testing.T) { + mgr, digest := testManager(t, 1, nil, nil) + host := openRegistry(t) + target := host + "/export/app:v1" + + dstRef, err := name.ParseReference(target, name.Insecure) + if err != nil { + t.Fatalf("ParseReference: %v", err) + } + key := pushKey(digest, dstRef.String(), true) + + // Two concurrent creates racing the same torn-down key: one must create + // the successor job and the other must merge into it — not wait out the + // successor and then start a duplicate push. + m := mgr.(*manager) + m.mu.Lock() + m.inflight[key] = inflightPush{id: "ghost", digest: digest} + m.mu.Unlock() + go func() { + time.Sleep(50 * time.Millisecond) + m.mu.Lock() + delete(m.inflight, key) + m.mu.Unlock() + }() + + ids := make([]string, 2) + errs := make([]error, 2) + var wg sync.WaitGroup + for i := range ids { + wg.Add(1) + go func(i int) { + defer wg.Done() + push, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: target, Insecure: true}) + if push != nil { + ids[i] = push.ID + } + errs[i] = err + }(i) + } + wg.Wait() + + for i := range ids { + if errs[i] != nil { + t.Fatalf("CreatePush #%d: %v", i, errs[i]) + } + } + if ids[0] != ids[1] { + t.Errorf("concurrent creates got IDs %s and %s, want one shared successor job", ids[0], ids[1]) + } + mustPushed(t, mgr, ids[0]) + + pushes, err := mgr.ListPushes(context.Background()) + if err != nil { + t.Fatalf("ListPushes: %v", err) + } + if len(pushes) != 1 { + t.Errorf("len(pushes) = %d, want 1 (no duplicate after the successor)", len(pushes)) + } +} + func TestWaitForPushCancellation(t *testing.T) { mgr, _ := testManager(t, 1, nil, nil) host, gate := gatedRegistry(t) From 6edf9548685a1f8ebeb2fd96fc78378540c5e023 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:11:01 +0000 Subject: [PATCH 15/21] Add pushes API for exporting images to remote registries POST /pushes creates a push job exporting a ready hypeman image from the OCI cache to a remote registry; GET /pushes and GET /pushes/{id} expose job state. Requests may lend registry credentials, which the push manager borrows for that job only and never persists; without them the server's own credentials resolve via the Docker keychain. Routes use the existing image:read/image:write scopes, the push queue concurrency is configurable, and in-flight push digests are composed into the OCI cache GC roots. --- cmd/api/api/api.go | 4 + cmd/api/api/pushes.go | 142 ++++ cmd/api/api/pushes_test.go | 235 ++++++ cmd/api/config/config.go | 2 + cmd/api/main.go | 18 +- cmd/api/wire.go | 3 + cmd/api/wire_gen.go | 9 +- config.example.yaml | 1 + lib/imagepush/imagepush.go | 3 + lib/imagepush/manager.go | 6 + lib/oapi/oapi.go | 1513 ++++++++++++++++++++++++++++-------- lib/providers/providers.go | 9 + lib/scopes/scopes.go | 5 + openapi.yaml | 217 +++++- 14 files changed, 1839 insertions(+), 328 deletions(-) create mode 100644 cmd/api/api/pushes.go create mode 100644 cmd/api/api/pushes_test.go diff --git a/cmd/api/api/api.go b/cmd/api/api/api.go index 20db9461..62e4a647 100644 --- a/cmd/api/api/api.go +++ b/cmd/api/api/api.go @@ -8,6 +8,7 @@ import ( "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/imagepush" "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/network" @@ -28,6 +29,7 @@ type ApiService struct { DeviceManager devices.Manager IngressManager ingress.Manager BuildManager builds.Manager + PushManager imagepush.Manager ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller AutoStandbyController *autostandby.Controller @@ -47,6 +49,7 @@ func New( deviceManager devices.Manager, ingressManager ingress.Manager, buildManager builds.Manager, + pushManager imagepush.Manager, resourceManager *resources.Manager, guestMemoryController guestmemory.Controller, autoStandbyController *autostandby.Controller, @@ -62,6 +65,7 @@ func New( DeviceManager: deviceManager, IngressManager: ingressManager, BuildManager: buildManager, + PushManager: pushManager, ResourceManager: resourceManager, GuestMemoryController: guestMemoryController, AutoStandbyController: autoStandbyController, diff --git a/cmd/api/api/pushes.go b/cmd/api/api/pushes.go new file mode 100644 index 00000000..f1cff277 --- /dev/null +++ b/cmd/api/api/pushes.go @@ -0,0 +1,142 @@ +package api + +import ( + "context" + "errors" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/kernel/hypeman/lib/imagepush" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/logger" + "github.com/kernel/hypeman/lib/oapi" +) + +func (s *ApiService) CreatePush(ctx context.Context, request oapi.CreatePushRequestObject) (oapi.CreatePushResponseObject, error) { + log := logger.FromContext(ctx) + + domainReq := imagepush.PushRequest{ + Image: request.Body.Image, + Target: request.Body.Target, + Credentials: pushCredentialsToAuthn(request.Body.Credentials), + } + if request.Body.Insecure != nil { + domainReq.Insecure = *request.Body.Insecure + } + + push, err := s.PushManager.CreatePush(ctx, domainReq) + if err != nil { + switch { + case errors.Is(err, images.ErrInvalidName): + return oapi.CreatePush400JSONResponse{ + Code: "invalid_name", + Message: err.Error(), + }, nil + case errors.Is(err, imagepush.ErrInvalidTarget): + return oapi.CreatePush400JSONResponse{ + Code: "invalid_target", + Message: err.Error(), + }, nil + case errors.Is(err, images.ErrNotFound): + return oapi.CreatePush404JSONResponse{ + Code: "not_found", + Message: "image not found", + }, nil + case errors.Is(err, imagepush.ErrImageNotReady): + return oapi.CreatePush409JSONResponse{ + Code: "image_not_ready", + Message: err.Error(), + }, nil + default: + log.ErrorContext(ctx, "failed to create push", "error", err) + return oapi.CreatePush500JSONResponse{ + Code: "internal_error", + Message: "failed to create push", + }, nil + } + } + + return oapi.CreatePush202JSONResponse(pushToOAPI(*push)), nil +} + +func (s *ApiService) GetPush(ctx context.Context, request oapi.GetPushRequestObject) (oapi.GetPushResponseObject, error) { + log := logger.FromContext(ctx) + + push, err := s.PushManager.GetPush(ctx, request.Id) + if err != nil { + if errors.Is(err, imagepush.ErrNotFound) { + return oapi.GetPush404JSONResponse{ + Code: "not_found", + Message: "push not found", + }, nil + } + log.ErrorContext(ctx, "failed to get push", "error", err) + return oapi.GetPush500JSONResponse{ + Code: "internal_error", + Message: "failed to get push", + }, nil + } + + return oapi.GetPush200JSONResponse(pushToOAPI(*push)), nil +} + +func (s *ApiService) ListPushes(ctx context.Context, request oapi.ListPushesRequestObject) (oapi.ListPushesResponseObject, error) { + log := logger.FromContext(ctx) + + pushes, err := s.PushManager.ListPushes(ctx) + if err != nil { + log.ErrorContext(ctx, "failed to list pushes", "error", err) + return oapi.ListPushes500JSONResponse{ + Code: "internal_error", + Message: "failed to list pushes", + }, nil + } + + out := make([]oapi.Push, 0, len(pushes)) + for _, push := range pushes { + out = append(out, pushToOAPI(push)) + } + return oapi.ListPushes200JSONResponse(out), nil +} + +// pushCredentialsToAuthn maps API credentials to the go-containerregistry +// auth config. Returns nil when absent so the push falls back to the +// server's default credential resolution. +func pushCredentialsToAuthn(creds *oapi.PushCredentials) *authn.AuthConfig { + if creds == nil { + return nil + } + cfg := &authn.AuthConfig{} + if creds.Username != nil { + cfg.Username = *creds.Username + } + if creds.Password != nil { + cfg.Password = *creds.Password + } + if creds.RegistryToken != nil { + cfg.RegistryToken = *creds.RegistryToken + } + return cfg +} + +func pushToOAPI(push imagepush.Push) oapi.Push { + out := oapi.Push{ + Id: push.ID, + Image: push.Image, + Digest: push.Digest, + Target: push.Target, + Status: oapi.PushStatus(push.Status), + QueuePosition: push.QueuePosition, + Error: push.Error, + CreatedAt: push.CreatedAt, + CompletedAt: push.CompletedAt, + } + if push.Layers > 0 { + layers := push.Layers + out.Layers = &layers + } + if push.Bytes > 0 { + bytes := push.Bytes + out.Bytes = &bytes + } + return out +} diff --git a/cmd/api/api/pushes_test.go b/cmd/api/api/pushes_test.go new file mode 100644 index 00000000..69de7dec --- /dev/null +++ b/cmd/api/api/pushes_test.go @@ -0,0 +1,235 @@ +package api + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/kernel/hypeman/lib/imagepush" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/oapi" + "github.com/stretchr/testify/require" +) + +// fakePushManager implements imagepush.Manager for handler tests. +type fakePushManager struct { + createErr error + createdReq imagepush.PushRequest + push *imagepush.Push + getErr error + listErr error + pushes []imagepush.Push +} + +func (f *fakePushManager) CreatePush(_ context.Context, req imagepush.PushRequest) (*imagepush.Push, error) { + f.createdReq = req + if f.createErr != nil { + return nil, f.createErr + } + return f.push, nil +} + +func (f *fakePushManager) GetPush(_ context.Context, _ string) (*imagepush.Push, error) { + if f.getErr != nil { + return nil, f.getErr + } + return f.push, nil +} + +func (f *fakePushManager) ListPushes(_ context.Context) ([]imagepush.Push, error) { + if f.listErr != nil { + return nil, f.listErr + } + return f.pushes, nil +} + +func (f *fakePushManager) WaitForPush(_ context.Context, _ string) error { return nil } + +func (f *fakePushManager) InProgressDigests() []string { return nil } +func (f *fakePushManager) LiveCacheManifestDigests() []string { return nil } + +func TestCreatePush_MapsRequestAndCredentials(t *testing.T) { + t.Parallel() + + now := time.Now().Truncate(time.Second) + fake := &fakePushManager{push: &imagepush.Push{ + ID: "push-1", + Image: "docker.io/library/alpine:latest", + Digest: "sha256:abc", + Target: "registry.example.com/app:v1", + Status: imagepush.StatusQueued, + CreatedAt: now, + }} + svc := &ApiService{PushManager: fake} + + insecure := true + username, password, token := "pusher", "hunter2", "bearer-tok" + resp, err := svc.CreatePush(context.Background(), oapi.CreatePushRequestObject{ + Body: &oapi.CreatePushRequest{ + Image: "alpine:latest", + Target: "registry.example.com/app:v1", + Insecure: &insecure, + Credentials: &oapi.PushCredentials{ + Username: &username, + Password: &password, + RegistryToken: &token, + }, + }, + }) + require.NoError(t, err) + require.IsType(t, oapi.CreatePush202JSONResponse{}, resp) + + got := resp.(oapi.CreatePush202JSONResponse) + require.Equal(t, "push-1", got.Id) + require.Equal(t, oapi.PushStatus(imagepush.StatusQueued), got.Status) + require.Equal(t, now, got.CreatedAt) + + // Borrowed credentials must reach the manager as an auth config. + require.Equal(t, "alpine:latest", fake.createdReq.Image) + require.True(t, fake.createdReq.Insecure) + require.NotNil(t, fake.createdReq.Credentials) + require.Equal(t, &authn.AuthConfig{ + Username: "pusher", + Password: "hunter2", + RegistryToken: "bearer-tok", + }, fake.createdReq.Credentials) +} + +func TestCreatePush_NoCredentialsStaysNil(t *testing.T) { + t.Parallel() + + fake := &fakePushManager{push: &imagepush.Push{ID: "push-1", Status: imagepush.StatusQueued}} + svc := &ApiService{PushManager: fake} + + resp, err := svc.CreatePush(context.Background(), oapi.CreatePushRequestObject{ + Body: &oapi.CreatePushRequest{Image: "alpine:latest", Target: "registry.example.com/app:v1"}, + }) + require.NoError(t, err) + require.IsType(t, oapi.CreatePush202JSONResponse{}, resp) + require.Nil(t, fake.createdReq.Credentials) +} + +func TestCreatePush_ErrorStatusMapping(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + err error + wantType any + wantCode string + }{ + { + name: "invalid name -> 400", + err: fmt.Errorf("lookup: %w", images.ErrInvalidName), + wantType: oapi.CreatePush400JSONResponse{}, + wantCode: "invalid_name", + }, + { + name: "invalid target -> 400", + err: fmt.Errorf("parse: %w", imagepush.ErrInvalidTarget), + wantType: oapi.CreatePush400JSONResponse{}, + wantCode: "invalid_target", + }, + { + name: "image not found -> 404", + err: fmt.Errorf("lookup: %w", images.ErrNotFound), + wantType: oapi.CreatePush404JSONResponse{}, + wantCode: "not_found", + }, + { + name: "image not ready -> 409", + err: fmt.Errorf("lookup: %w", imagepush.ErrImageNotReady), + wantType: oapi.CreatePush409JSONResponse{}, + wantCode: "image_not_ready", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + svc := &ApiService{PushManager: &fakePushManager{createErr: tc.err}} + + resp, err := svc.CreatePush(context.Background(), oapi.CreatePushRequestObject{ + Body: &oapi.CreatePushRequest{Image: "alpine:latest", Target: "registry.example.com/app:v1"}, + }) + require.NoError(t, err) + require.IsType(t, tc.wantType, resp) + require.Equal(t, tc.wantCode, pushErrorCodeOf(resp)) + }) + } +} + +func TestGetPush_NotFound(t *testing.T) { + t.Parallel() + + svc := &ApiService{PushManager: &fakePushManager{getErr: imagepush.ErrNotFound}} + resp, err := svc.GetPush(context.Background(), oapi.GetPushRequestObject{Id: "missing"}) + require.NoError(t, err) + require.IsType(t, oapi.GetPush404JSONResponse{}, resp) + require.Equal(t, "not_found", pushErrorCodeOf(resp)) +} + +func TestGetPush_OmitsEmptyCounters(t *testing.T) { + t.Parallel() + + fake := &fakePushManager{push: &imagepush.Push{ + ID: "push-1", + Status: imagepush.StatusQueued, + }} + svc := &ApiService{PushManager: fake} + resp, err := svc.GetPush(context.Background(), oapi.GetPushRequestObject{Id: "push-1"}) + require.NoError(t, err) + got, ok := resp.(oapi.GetPush200JSONResponse) + require.True(t, ok) + require.Nil(t, got.Layers) + require.Nil(t, got.Bytes) +} + +func TestListPushes_Empty(t *testing.T) { + t.Parallel() + + svc := &ApiService{PushManager: &fakePushManager{}} + resp, err := svc.ListPushes(context.Background(), oapi.ListPushesRequestObject{}) + require.NoError(t, err) + got, ok := resp.(oapi.ListPushes200JSONResponse) + require.True(t, ok) + require.Empty(t, got) +} + +func TestListPushes_ReturnsAll(t *testing.T) { + t.Parallel() + + fake := &fakePushManager{pushes: []imagepush.Push{ + {ID: "push-2", Status: imagepush.StatusPushed, Layers: 3, Bytes: 1024}, + {ID: "push-1", Status: imagepush.StatusFailed}, + }} + svc := &ApiService{PushManager: fake} + resp, err := svc.ListPushes(context.Background(), oapi.ListPushesRequestObject{}) + require.NoError(t, err) + got, ok := resp.(oapi.ListPushes200JSONResponse) + require.True(t, ok) + require.Len(t, got, 2) + require.Equal(t, "push-2", got[0].Id) + require.NotNil(t, got[0].Layers) + require.Equal(t, 3, *got[0].Layers) + require.NotNil(t, got[0].Bytes) + require.Equal(t, int64(1024), *got[0].Bytes) +} + +// pushErrorCodeOf extracts the Code field from any CreatePush/GetPush error response. +func pushErrorCodeOf(resp any) string { + switch r := resp.(type) { + case oapi.CreatePush400JSONResponse: + return r.Code + case oapi.CreatePush404JSONResponse: + return r.Code + case oapi.CreatePush409JSONResponse: + return r.Code + case oapi.GetPush404JSONResponse: + return r.Code + default: + return "" + } +} diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 8a22c060..460cf6d2 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -187,6 +187,7 @@ type LimitsConfig struct { MaxMemoryPerInstance string `koanf:"max_memory_per_instance"` MaxTotalVolumeStorage string `koanf:"max_total_volume_storage"` MaxConcurrentBuilds int `koanf:"max_concurrent_builds"` + MaxConcurrentPushes int `koanf:"max_concurrent_pushes"` MaxOverlaySize string `koanf:"max_overlay_size"` MaxImageStorage float64 `koanf:"max_image_storage"` } @@ -430,6 +431,7 @@ func defaultConfig() *Config { MaxMemoryPerInstance: "32GB", MaxTotalVolumeStorage: "", MaxConcurrentBuilds: 1, + MaxConcurrentPushes: 2, MaxOverlaySize: "100GB", MaxImageStorage: 0.2, }, diff --git a/cmd/api/main.go b/cmd/api/main.go index 98d7be72..50a6a4e6 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -119,6 +119,22 @@ type ociCacheGCRunner interface { Run(ctx context.Context) error } +// compositeOCICacheRoots fans the GC's extra-root query out to every source +// that tracks cache blobs outside index.json: the embedded registry's +// BuildKit cache tags and the push manager's in-flight push digests. +type compositeOCICacheRoots []ocicachegc.RootsProvider + +func (c compositeOCICacheRoots) LiveCacheManifestDigests() []string { + out := make([]string, 0) + for _, roots := range c { + if roots == nil { + continue + } + out = append(out, roots.LiveCacheManifestDigests()...) + } + return out +} + func configureOCICacheGC(cfg *config.Config, roots ocicachegc.RootsProvider, logger *slog.Logger, meter metric.Meter, tracer trace.Tracer) (ociCacheGCRunner, error) { if cfg == nil || !cfg.Images.OCICacheGC.Enabled { return nil, nil @@ -576,7 +592,7 @@ func run() error { ociGC, err := configureOCICacheGC( app.Config, - app.Registry, + compositeOCICacheRoots{app.Registry, app.PushManager}, logger, otelProvider.MeterFor(loglib.SubsystemImages), otelProvider.TracerFor(loglib.SubsystemImages), diff --git a/cmd/api/wire.go b/cmd/api/wire.go index 95c5b00e..cc4870a5 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -15,6 +15,7 @@ import ( "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/imagepush" "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/network" @@ -40,6 +41,7 @@ type application struct { BuilderManager builders.Manager IngressManager ingress.Manager BuildManager builds.Manager + PushManager imagepush.Manager ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller AutoStandbyController *autostandby.Controller @@ -65,6 +67,7 @@ func initializeApp() (*application, func(), error) { providers.ProvideBuilderManager, providers.ProvideIngressManager, providers.ProvideBuildManager, + providers.ProvidePushManager, providers.ProvideResourceManager, providers.ProvideGuestMemoryController, providers.ProvideAutoStandbyController, diff --git a/cmd/api/wire_gen.go b/cmd/api/wire_gen.go index 3fa5198e..9eb13c6e 100644 --- a/cmd/api/wire_gen.go +++ b/cmd/api/wire_gen.go @@ -15,6 +15,7 @@ import ( "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" + "github.com/kernel/hypeman/lib/imagepush" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/instances" @@ -67,6 +68,10 @@ func initializeApp() (*application, func(), error) { if err != nil { return nil, nil, err } + imagepushManager, err := providers.ProvidePushManager(paths, config, manager) + if err != nil { + return nil, nil, err + } resourcesManager, err := providers.ProvideResourceManager(context, config, paths, manager, instancesManager, volumesManager) if err != nil { return nil, nil, err @@ -85,7 +90,7 @@ func initializeApp() (*application, func(), error) { if err != nil { return nil, nil, err } - apiService := api.New(config, manager, instancesManager, volumesManager, buildersManager, networkManager, devicesManager, ingressManager, buildsManager, resourcesManager, controller, autostandbyController, vm_metricsManager) + apiService := api.New(config, manager, instancesManager, volumesManager, buildersManager, networkManager, devicesManager, ingressManager, buildsManager, imagepushManager, resourcesManager, controller, autostandbyController, vm_metricsManager) mainApplication := &application{ Ctx: context, Logger: logger, @@ -99,6 +104,7 @@ func initializeApp() (*application, func(), error) { BuilderManager: buildersManager, IngressManager: ingressManager, BuildManager: buildsManager, + PushManager: imagepushManager, ResourceManager: resourcesManager, GuestMemoryController: controller, AutoStandbyController: autostandbyController, @@ -127,6 +133,7 @@ type application struct { BuilderManager builders.Manager IngressManager ingress.Manager BuildManager builds.Manager + PushManager imagepush.Manager ResourceManager *resources.Manager GuestMemoryController guestmemory.Controller AutoStandbyController *autostandby.Controller diff --git a/config.example.yaml b/config.example.yaml index 4cab6e88..ac908464 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -168,4 +168,5 @@ data_dir: /var/lib/hypeman # max_memory_per_instance: 32GB # max_total_volume_storage: "" # 0 or empty = unlimited # max_concurrent_builds: 1 +# max_concurrent_pushes: 2 # max_overlay_size: 100GB diff --git a/lib/imagepush/imagepush.go b/lib/imagepush/imagepush.go index 3ed18f1d..236602d5 100644 --- a/lib/imagepush/imagepush.go +++ b/lib/imagepush/imagepush.go @@ -119,4 +119,7 @@ type Manager interface { // InProgressDigests returns the manifest digests of queued and pushing // jobs so the OCI cache GC can keep their blobs alive mid-push. InProgressDigests() []string + // LiveCacheManifestDigests implements ocicachegc.RootsProvider by + // delegating to InProgressDigests. + LiveCacheManifestDigests() []string } diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index 21f4d891..5f8c56cd 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -401,6 +401,12 @@ func (m *manager) InProgressDigests() []string { return digests } +// LiveCacheManifestDigests implements ocicachegc.RootsProvider so in-flight +// push digests are treated as live alongside the OCI layout index. +func (m *manager) LiveCacheManifestDigests() []string { + return m.InProgressDigests() +} + func (m *manager) recoverInterruptedPushes() { // A crash between the push-dir MkdirAll and the metadata rename leaves an // empty dir no listing can read and no recovery can act on; sweep it now, diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 398daeab..576fa723 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -193,6 +193,14 @@ const ( MemoryReclaimResponseHostPressureStatePressure MemoryReclaimResponseHostPressureState = "pressure" ) +// Defines values for PushStatus. +const ( + Failed PushStatus = "failed" + Pushed PushStatus = "pushed" + Pushing PushStatus = "pushing" + Queued PushStatus = "queued" +) + // Defines values for RestartPolicyPolicy. const ( Always RestartPolicyPolicy = "always" @@ -685,6 +693,26 @@ type CreateInstanceRequestNetworkEgressEnforcement struct { // while `http_https_only` rejects direct egress only on TCP ports 80 and 443. type CreateInstanceRequestNetworkEgressEnforcementMode string +// CreatePushRequest defines model for CreatePushRequest. +type CreatePushRequest struct { + // Credentials Registry credentials borrowed for this push only, docker-style: the + // caller's registry login (e.g. from the client's ~/.docker/config.json) + // rides along with the request instead of living on the server. Never + // persisted or logged; a push interrupted by a restart fails instead of + // retrying without them. When omitted, the server's own registry + // credentials are used. + Credentials *PushCredentials `json:"credentials,omitempty"` + + // Image Hypeman image name to push (tag or digest form) + Image string `json:"image"` + + // Insecure Allow pushing to plain-HTTP registries + Insecure *bool `json:"insecure,omitempty"` + + // Target Full remote reference to push to + Target string `json:"target"` +} + // CreateSnapshotRequest defines model for CreateSnapshotRequest. type CreateSnapshotRequest struct { Compression *SnapshotCompressionConfig `json:"compression,omitempty"` @@ -1335,6 +1363,56 @@ type PathInfo struct { Size *int64 `json:"size,omitempty"` } +// Push defines model for Push. +type Push struct { + // Bytes Total compressed layer bytes pushed (only when status is pushed) + Bytes *int64 `json:"bytes,omitempty"` + CompletedAt *time.Time `json:"completed_at"` + CreatedAt time.Time `json:"created_at"` + + // Digest Cached manifest digest being pushed + Digest string `json:"digest"` + + // Error Error message (only when status is failed) + Error *string `json:"error"` + + // Id Push job identifier + Id string `json:"id"` + + // Image Hypeman image name (normalized ref) + Image string `json:"image"` + + // Layers Number of layers pushed (only when status is pushed) + Layers *int `json:"layers,omitempty"` + + // QueuePosition Position in the push queue (only when status is queued) + QueuePosition *int `json:"queue_position"` + Status PushStatus `json:"status"` + + // Target Remote reference the image is pushed to + Target string `json:"target"` +} + +// PushCredentials Registry credentials borrowed for this push only, docker-style: the +// caller's registry login (e.g. from the client's ~/.docker/config.json) +// rides along with the request instead of living on the server. Never +// persisted or logged; a push interrupted by a restart fails instead of +// retrying without them. When omitted, the server's own registry +// credentials are used. +type PushCredentials struct { + // Password Registry password or access token + Password *string `json:"password,omitempty"` + + // RegistryToken Bearer token sent as-is in the Authorization header + RegistryToken *string `json:"registry_token,omitempty"` + + // Username Registry username + Username *string `json:"username,omitempty"` +} + +// PushStatus defines model for PushStatus. +type PushStatus string + // ResourceAllocation defines model for ResourceAllocation. type ResourceAllocation struct { // Cpu vCPUs allocated @@ -1922,6 +2000,9 @@ type StartInstanceJSONRequestBody StartInstanceJSONBody // AttachVolumeJSONRequestBody defines body for AttachVolume for application/json ContentType. type AttachVolumeJSONRequestBody = AttachVolumeRequest +// CreatePushJSONRequestBody defines body for CreatePush for application/json ContentType. +type CreatePushJSONRequestBody = CreatePushRequest + // ReclaimMemoryJSONRequestBody defines body for ReclaimMemory for application/json ContentType. type ReclaimMemoryJSONRequestBody = MemoryReclaimRequest @@ -2171,6 +2252,17 @@ type ClientInterface interface { // WaitForInstanceState request WaitForInstanceState(ctx context.Context, id string, params *WaitForInstanceStateParams, reqEditors ...RequestEditorFn) (*http.Response, error) + // ListPushes request + ListPushes(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) + + // CreatePushWithBody request with any body + CreatePushWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + CreatePush(ctx context.Context, body CreatePushJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + + // GetPush request + GetPush(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetResources request GetResources(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -2931,6 +3023,54 @@ func (c *Client) WaitForInstanceState(ctx context.Context, id string, params *Wa return c.Client.Do(req) } +func (c *Client) ListPushes(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewListPushesRequest(c.Server) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) CreatePushWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreatePushRequestWithBody(c.Server, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) CreatePush(ctx context.Context, body CreatePushJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewCreatePushRequest(c.Server, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) GetPush(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetPushRequest(c.Server, id) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + func (c *Client) GetResources(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { req, err := NewGetResourcesRequest(c.Server) if err != nil { @@ -5083,6 +5223,107 @@ func NewWaitForInstanceStateRequest(server string, id string, params *WaitForIns return req, nil } +// NewListPushesRequest generates requests for ListPushes +func NewListPushesRequest(server string) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/pushes") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewCreatePushRequest calls the generic CreatePush builder with application/json body +func NewCreatePushRequest(server string, body CreatePushJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewCreatePushRequestWithBody(server, "application/json", bodyReader) +} + +// NewCreatePushRequestWithBody generates requests for CreatePush with any type of body +func NewCreatePushRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/pushes") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + return req, nil +} + +// NewGetPushRequest generates requests for GetPush +func NewGetPushRequest(server string, id string) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithLocation("simple", false, "id", runtime.ParamLocationPath, id) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/pushes/%s", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("GET", queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewGetResourcesRequest generates requests for GetResources func NewGetResourcesRequest(server string) (*http.Request, error) { var err error @@ -5820,6 +6061,17 @@ type ClientWithResponsesInterface interface { // WaitForInstanceStateWithResponse request WaitForInstanceStateWithResponse(ctx context.Context, id string, params *WaitForInstanceStateParams, reqEditors ...RequestEditorFn) (*WaitForInstanceStateResponse, error) + // ListPushesWithResponse request + ListPushesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListPushesResponse, error) + + // CreatePushWithBodyWithResponse request with any body + CreatePushWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreatePushResponse, error) + + CreatePushWithResponse(ctx context.Context, body CreatePushJSONRequestBody, reqEditors ...RequestEditorFn) (*CreatePushResponse, error) + + // GetPushWithResponse request + GetPushWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetPushResponse, error) + // GetResourcesWithResponse request GetResourcesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetResourcesResponse, error) @@ -7024,6 +7276,82 @@ func (r WaitForInstanceStateResponse) StatusCode() int { return 0 } +type ListPushesResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *[]Push + JSON401 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r ListPushesResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r ListPushesResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type CreatePushResponse struct { + Body []byte + HTTPResponse *http.Response + JSON202 *Push + JSON400 *Error + JSON401 *Error + JSON404 *Error + JSON409 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r CreatePushResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r CreatePushResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +type GetPushResponse struct { + Body []byte + HTTPResponse *http.Response + JSON200 *Push + JSON401 *Error + JSON404 *Error + JSON500 *Error +} + +// Status returns HTTPResponse.Status +func (r GetPushResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetPushResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + type GetResourcesResponse struct { Body []byte HTTPResponse *http.Response @@ -7820,6 +8148,41 @@ func (c *ClientWithResponses) WaitForInstanceStateWithResponse(ctx context.Conte return ParseWaitForInstanceStateResponse(rsp) } +// ListPushesWithResponse request returning *ListPushesResponse +func (c *ClientWithResponses) ListPushesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*ListPushesResponse, error) { + rsp, err := c.ListPushes(ctx, reqEditors...) + if err != nil { + return nil, err + } + return ParseListPushesResponse(rsp) +} + +// CreatePushWithBodyWithResponse request with arbitrary body returning *CreatePushResponse +func (c *ClientWithResponses) CreatePushWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*CreatePushResponse, error) { + rsp, err := c.CreatePushWithBody(ctx, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseCreatePushResponse(rsp) +} + +func (c *ClientWithResponses) CreatePushWithResponse(ctx context.Context, body CreatePushJSONRequestBody, reqEditors ...RequestEditorFn) (*CreatePushResponse, error) { + rsp, err := c.CreatePush(ctx, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseCreatePushResponse(rsp) +} + +// GetPushWithResponse request returning *GetPushResponse +func (c *ClientWithResponses) GetPushWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*GetPushResponse, error) { + rsp, err := c.GetPush(ctx, id, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetPushResponse(rsp) +} + // GetResourcesWithResponse request returning *GetResourcesResponse func (c *ClientWithResponses) GetResourcesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetResourcesResponse, error) { rsp, err := c.GetResources(ctx, reqEditors...) @@ -10075,28 +10438,176 @@ func ParseWaitForInstanceStateResponse(rsp *http.Response) (*WaitForInstanceStat return response, nil } -// ParseGetResourcesResponse parses an HTTP response from a GetResourcesWithResponse call -func ParseGetResourcesResponse(rsp *http.Response) (*GetResourcesResponse, error) { +// ParseListPushesResponse parses an HTTP response from a ListPushesWithResponse call +func ParseListPushesResponse(rsp *http.Response) (*ListPushesResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) defer func() { _ = rsp.Body.Close() }() if err != nil { return nil, err } - response := &GetResourcesResponse{ + response := &ListPushesResponse{ Body: bodyBytes, HTTPResponse: rsp, } switch { case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: - var dest Resources + var dest []Push if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err } response.JSON200 = &dest - case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + +// ParseCreatePushResponse parses an HTTP response from a CreatePushWithResponse call +func ParseCreatePushResponse(rsp *http.Response) (*CreatePushResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &CreatePushResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 202: + var dest Push + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON202 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON409 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + +// ParseGetPushResponse parses an HTTP response from a GetPushWithResponse call +func ParseGetPushResponse(rsp *http.Response) (*GetPushResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetPushResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest Push + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + +// ParseGetResourcesResponse parses an HTTP response from a GetResourcesWithResponse call +func ParseGetResourcesResponse(rsp *http.Response) (*GetResourcesResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetResourcesResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest Resources + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { return nil, err @@ -10693,6 +11204,15 @@ type ServerInterface interface { // Wait for instance to reach a target state // (GET /instances/{id}/wait) WaitForInstanceState(w http.ResponseWriter, r *http.Request, id string, params WaitForInstanceStateParams) + // List pushes + // (GET /pushes) + ListPushes(w http.ResponseWriter, r *http.Request) + // Push an image to a remote registry + // (POST /pushes) + CreatePush(w http.ResponseWriter, r *http.Request) + // Get push details + // (GET /pushes/{id}) + GetPush(w http.ResponseWriter, r *http.Request, id string) // Get host resource capacity and allocations // (GET /resources) GetResources(w http.ResponseWriter, r *http.Request) @@ -11014,6 +11534,24 @@ func (_ Unimplemented) WaitForInstanceState(w http.ResponseWriter, r *http.Reque w.WriteHeader(http.StatusNotImplemented) } +// List pushes +// (GET /pushes) +func (_ Unimplemented) ListPushes(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) +} + +// Push an image to a remote registry +// (POST /pushes) +func (_ Unimplemented) CreatePush(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotImplemented) +} + +// Get push details +// (GET /pushes/{id}) +func (_ Unimplemented) GetPush(w http.ResponseWriter, r *http.Request, id string) { + w.WriteHeader(http.StatusNotImplemented) +} + // Get host resource capacity and allocations // (GET /resources) func (_ Unimplemented) GetResources(w http.ResponseWriter, r *http.Request) { @@ -12589,6 +13127,77 @@ func (siw *ServerInterfaceWrapper) WaitForInstanceState(w http.ResponseWriter, r handler.ServeHTTP(w, r) } +// ListPushes operation middleware +func (siw *ServerInterfaceWrapper) ListPushes(w http.ResponseWriter, r *http.Request) { + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.ListPushes(w, r) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + +// CreatePush operation middleware +func (siw *ServerInterfaceWrapper) CreatePush(w http.ResponseWriter, r *http.Request) { + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.CreatePush(w, r) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + +// GetPush operation middleware +func (siw *ServerInterfaceWrapper) GetPush(w http.ResponseWriter, r *http.Request) { + + var err error + + // ------------- Path parameter "id" ------------- + var id string + + err = runtime.BindStyledParameterWithOptions("simple", "id", chi.URLParam(r, "id"), &id, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true}) + if err != nil { + siw.ErrorHandlerFunc(w, r, &InvalidParamFormatError{ParamName: "id", Err: err}) + return + } + + ctx := r.Context() + + ctx = context.WithValue(ctx, BearerAuthScopes, []string{}) + + r = r.WithContext(ctx) + + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + siw.Handler.GetPush(w, r, id) + })) + + for _, middleware := range siw.HandlerMiddlewares { + handler = middleware(handler) + } + + handler.ServeHTTP(w, r) +} + // GetResources operation middleware func (siw *ServerInterfaceWrapper) GetResources(w http.ResponseWriter, r *http.Request) { @@ -13219,6 +13828,15 @@ func HandlerWithOptions(si ServerInterface, options ChiServerOptions) http.Handl r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/instances/{id}/wait", wrapper.WaitForInstanceState) }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/pushes", wrapper.ListPushes) + }) + r.Group(func(r chi.Router) { + r.Post(options.BaseURL+"/pushes", wrapper.CreatePush) + }) + r.Group(func(r chi.Router) { + r.Get(options.BaseURL+"/pushes/{id}", wrapper.GetPush) + }) r.Group(func(r chi.Router) { r.Get(options.BaseURL+"/resources", wrapper.GetResources) }) @@ -15332,6 +15950,146 @@ func (response WaitForInstanceState500JSONResponse) VisitWaitForInstanceStateRes return json.NewEncoder(w).Encode(response) } +type ListPushesRequestObject struct { +} + +type ListPushesResponseObject interface { + VisitListPushesResponse(w http.ResponseWriter) error +} + +type ListPushes200JSONResponse []Push + +func (response ListPushes200JSONResponse) VisitListPushesResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type ListPushes401JSONResponse Error + +func (response ListPushes401JSONResponse) VisitListPushesResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(401) + + return json.NewEncoder(w).Encode(response) +} + +type ListPushes500JSONResponse Error + +func (response ListPushes500JSONResponse) VisitListPushesResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + +type CreatePushRequestObject struct { + Body *CreatePushJSONRequestBody +} + +type CreatePushResponseObject interface { + VisitCreatePushResponse(w http.ResponseWriter) error +} + +type CreatePush202JSONResponse Push + +func (response CreatePush202JSONResponse) VisitCreatePushResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(202) + + return json.NewEncoder(w).Encode(response) +} + +type CreatePush400JSONResponse Error + +func (response CreatePush400JSONResponse) VisitCreatePushResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + +type CreatePush401JSONResponse Error + +func (response CreatePush401JSONResponse) VisitCreatePushResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(401) + + return json.NewEncoder(w).Encode(response) +} + +type CreatePush404JSONResponse Error + +func (response CreatePush404JSONResponse) VisitCreatePushResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type CreatePush409JSONResponse Error + +func (response CreatePush409JSONResponse) VisitCreatePushResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(409) + + return json.NewEncoder(w).Encode(response) +} + +type CreatePush500JSONResponse Error + +func (response CreatePush500JSONResponse) VisitCreatePushResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + +type GetPushRequestObject struct { + Id string `json:"id"` +} + +type GetPushResponseObject interface { + VisitGetPushResponse(w http.ResponseWriter) error +} + +type GetPush200JSONResponse Push + +func (response GetPush200JSONResponse) VisitGetPushResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(200) + + return json.NewEncoder(w).Encode(response) +} + +type GetPush401JSONResponse Error + +func (response GetPush401JSONResponse) VisitGetPushResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(401) + + return json.NewEncoder(w).Encode(response) +} + +type GetPush404JSONResponse Error + +func (response GetPush404JSONResponse) VisitGetPushResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + + return json.NewEncoder(w).Encode(response) +} + +type GetPush500JSONResponse Error + +func (response GetPush500JSONResponse) VisitGetPushResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(500) + + return json.NewEncoder(w).Encode(response) +} + type GetResourcesRequestObject struct { } @@ -15922,6 +16680,15 @@ type StrictServerInterface interface { // Wait for instance to reach a target state // (GET /instances/{id}/wait) WaitForInstanceState(ctx context.Context, request WaitForInstanceStateRequestObject) (WaitForInstanceStateResponseObject, error) + // List pushes + // (GET /pushes) + ListPushes(ctx context.Context, request ListPushesRequestObject) (ListPushesResponseObject, error) + // Push an image to a remote registry + // (POST /pushes) + CreatePush(ctx context.Context, request CreatePushRequestObject) (CreatePushResponseObject, error) + // Get push details + // (GET /pushes/{id}) + GetPush(ctx context.Context, request GetPushRequestObject) (GetPushResponseObject, error) // Get host resource capacity and allocations // (GET /resources) GetResources(ctx context.Context, request GetResourcesRequestObject) (GetResourcesResponseObject, error) @@ -17297,6 +18064,87 @@ func (sh *strictHandler) WaitForInstanceState(w http.ResponseWriter, r *http.Req } } +// ListPushes operation middleware +func (sh *strictHandler) ListPushes(w http.ResponseWriter, r *http.Request) { + var request ListPushesRequestObject + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.ListPushes(ctx, request.(ListPushesRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "ListPushes") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(ListPushesResponseObject); ok { + if err := validResponse.VisitListPushesResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + +// CreatePush operation middleware +func (sh *strictHandler) CreatePush(w http.ResponseWriter, r *http.Request) { + var request CreatePushRequestObject + + var body CreatePushJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.CreatePush(ctx, request.(CreatePushRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "CreatePush") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(CreatePushResponseObject); ok { + if err := validResponse.VisitCreatePushResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + +// GetPush operation middleware +func (sh *strictHandler) GetPush(w http.ResponseWriter, r *http.Request, id string) { + var request GetPushRequestObject + + request.Id = id + + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { + return sh.ssi.GetPush(ctx, request.(GetPushRequestObject)) + } + for _, middleware := range sh.middlewares { + handler = middleware(handler, "GetPush") + } + + response, err := handler(r.Context(), w, r, request) + + if err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } else if validResponse, ok := response.(GetPushResponseObject); ok { + if err := validResponse.VisitGetPushResponse(w); err != nil { + sh.options.ResponseErrorHandlerFunc(w, r, err) + } + } else if response != nil { + sh.options.ResponseErrorHandlerFunc(w, r, fmt.Errorf("unexpected response type: %T", response)) + } +} + // GetResources operation middleware func (sh *strictHandler) GetResources(w http.ResponseWriter, r *http.Request) { var request GetResourcesRequestObject @@ -17603,326 +18451,341 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y9+3IbOZI3+ioIntkYaYakqKttdXR8R5bcbm232lrL9pzdpg8FVoEkRlVANYCiRHf4", - "332AfcR9ki+QAOqKIouyLtbYsRvTMqsKl0QikZnI/OWfnYDHCWeEKdk5/LMjgxmJMfx5pBQOZh94lMbk", - "LfkjJVLpnxPBEyIUJfBSzFOmRglWM/2vkMhA0ERRzjqHnXOsZuh6RgRBc2gFyRlPoxCNCYLvSNjpdsgN", - "jpOIdA47WzFTWyFWuNPtqEWif5JKUDbtfO52BMEhZ9HCdDPBaaQ6hxMcSdKtdHumm0ZYIv1JD77J2htz", - "HhHMOp+hxT9SKkjYOfy9OI2P2ct8/E8SKN35Uar4hcIsHC/OeUSDRX2yv1KW3kBvCKeKx1jRAEnzDUrg", - "IzTGkoSIM4QDRecEUTbmKQvRu+NzFHDGSKAbk0PGx5KIOQnRRPAYqRlBMy4VvKMEDq6QwuOI9Ies062s", - "B2H6SbiaSv+YETUjwjNYKpFtBU24QGpGJaJMPw1Iv7hgSqSkTtluh4YRGSkaE56qOqF+5tco4mwK03Lt", - "ojiVCs3wnKBPRHD0R4ojOllQNm0m0phMuCDo50VCYsxQEuGASEQVokxxNxtDo5zH9mMfc9Ep44KMQiIV", - "ZVi3P0q4MDuiPPo38AeOUOFdGBq8j9QMK8fljCt0RUhSnii+xldlMv6+s9N9MRgMPnY7VJHYbCt8Q+M0", - "7hwe7O/v7nc7MWXm39vZ6ClTZEqEHr79BQuBF4XpSJ6KgIwCGoplMwkiSphCx6cnb285gc72oA//t/W8", - "0+1sv9jpbx88h39vH3SK06oRvjzyz8u33oXCKpV1GWR208gyyqjAJPVZ/5bGYyIQn6AgFYIwFS0QbCkS", - "tmC60rQHvqUIOJvQaSrcFvRtuRI5Z1gizIzQ6FXkRd5Yq30XaCEW8ms2EiTGlGka1wbx1j1Ceociu4n0", - "kALOlOBRpIWCUiROlHS7qKvFOEM4SSIagOgpbaq9eCA73Q5Lo0g/rIwwX20S0SmFF1qRhsrCIrlvkeKI", - "MEVEtsPbkKYkFps6zsntXY1cLrbqcsajcJQyRaN6r+/1z5amxZ6oRDMShYhPJl1EJwgj3Yr+2fB4mew7", - "g52D3mCvNzh4t/3scPDicLD/X51uZ8JFjFXnsBNiRXp6ldusDQhtSVngXx1WZZFYH0iCBGZ1sgOrtIBj", - "EvCYIN300pHvffnIG4+b43xR9YvIvpifrJ6l7leOjHbcHWGpMiEE60XVYoQ9Y3pHYyIVjhMth/QYCsRs", - "kkKuweo6OMovJfD2FxGYkRs1shTyzsfHH+QmIYE+EbmTJpmCoduz7J2JrAfhcUGw1APWapI+TH/vpEym", - "iT66SThKIqx0u1qnAjYYxVRK/Wn2Q0ilkSPdjmPyEeNqJFLGzIuMqGsuropv2lZGNOl0OzMsR/Npkna6", - "y46tMlNDFyTCiYT27IqLERGCi45RjRejCRdukfSZm5NwSVM1CsnsiPVQqNPtlAiQiXM3FzfubFW9g4Ne", - "gJeEsSqMGQCTqQ+82FZ9uNnQlgt2c4oYJdotM7Ify7IECCmeMi4VDWQrMQ/Kg17emIce0XmSNYdoSJii", - "E0qE1asJEvp8iAlyjSDdCKIMpbKyDzLVf0Tm2lYbzfdGKkjqRKkYNsXFK+gm+YlYOJWz5c92ygomLc/d", - "azjNMYU9eULm1BwtZd3NLs0oFHROhEd8ZwqAEYXmPbSh97oWIYwzslmiFJvTkOI24iCEMY2oh3vOj0+R", - "eYxOT9DGjNyUO9l5Nn7eaW6S4djDCz+nMWY9vSH0sFz78G6x7V/3vCYKj+N0NBU8Teotn745O3uP4CFi", - "oOEWW3y+49NUk4COcBgKIqV//u5hcWyDwWBwiHcOB4P+wDfKOWEhF40kNY/9JN0ehGRJk61IatuvkfS3", - "D6cnp0fomIuEC7DZVm6cInmK8yqyTXlVfPz/MqVRWOf6sf6ZNNCJCEml0vLqpXkNCWJMOXQ945KgAAcz", - "gsbGaAHzHNprw/JZx+708o3ghMqEg5xHH85ybQ5MQ3JDglSV+v0BhVTbwoFVqgqTazGigOs10uevT7MA", - "CiD7jja1lVOdbq0EBILgFd3pN1p1Vt//qWGvUSybWnevaDEf0yiikgSchbLYB2XqYK95MoVdbI7NWlev", - "9M8oJlLiKUEb4JYCc8NIeK1tTTCNSLjZTsNumsw/+bhwrpX2HLBBD4+D7Z1dr0CL8ZSMQjq1fsUqD+rf", - "tX6s21EI3vZPBDSMdvOALgWZ1Pv7Cc4T6ESQCRFEc/wXdpcIPicMW5PqL9Bv5//Zyh2uW9bbugXEPM9f", - "/9zt/JGSlIwSLqkZYU1M2CeajYDUCL7wjxkeLVvrAkdJhcXy/QFv3MFOzJXNlbSxrh+tb+Hpyk/e6Xeq", - "Ah3EUabgFKRAo9x+pTUtj8rCmbIPKi5gPkURZcYM0vLbrAUoe4uE/Bjx6WbnzuiQkb+++fW4byG8zA8N", - "reln3cwqiPi0SM0ZwUKNSYmYDeeqbSgfXSP5z0vbp3KAYklGyyXIOWWMhOBztxvbvKl1a6/tA7voiqrR", - "XJ/Avj0Hw/qFKmTfaGwq4sHVhEZkNMNyZp2UYUiNw/W8NBOPClm6zMDgJHANgmoDRvXFz0c7+wfIduCh", - "ofX+6hfqMyl8rZu36oXCYoyjyMsbzey2/hld5xA/B+QO36azJ+NAx5hG0nXsalrjPZUz8xfIbj0qOPu0", - "GNDsFem/a4zb7dz0dKu9ORZAdd18YVj/4Xoq/PYy77Tw63nWf+HHt3YohZ9+cqMq/HacD9BRxZhJXhc4", - "TNqvVXLKVI8yWAKtgFvPkzk6cjer9WQgvYyZhgcn+xcpVsc1lQptvP3peHd398Vme+WKyquRpJ/IaDpe", - "qjfbQVtFWX+G9Gf6qJzSKR4vFJF9dIwZ4wqNCQpmmE1JiPBE6a/sWEsG+P5g1SVMo5JERJOKpD7t4RfP", - "b26wenFAr+WLT/FYTP+5673/BO+iljfNHjhsV1O/igRmnkW83UEd4xtt+Rsu8V1+npnrKpS9ZNdAVofQ", - "R7lBP6E3muYKbZco7SWu3/rLrrAYZ72U0T9SWO0kwou6DagIjnvAET7yGrlhNpBcvX2k3j/mGzfTa0yV", - "3jnZnamdchfxKNTnzoQKqTrtb8PWUgyIuCsVqbTL/BpTnSOqBGw81LNxNu6ViE5IsAgiUhfuTnwnIrXu", - "1pDoMwf+zByIbcW4G0pJErsfz7MuSj+f5P2Vfn9lOv/cNZKO2IeNYRPLRdnxUrn1JqZKawCpNPd2xhuM", - "7HX/HUit/GYYRxERPZkmSUThAsWJsa5xmk4JI0JzhXHFKaRNHhpW4jryndfb9u29e9/ca22IGt+aFTUO", - "zMYFXTGHacS1ZrVAbiK576+PTsu06yIMD+BKpEzl7NKq4J9DG6Q/7XfRsJMEtDcYDAY9vNMbDHqDYafs", - "YYv2eubmIcFKEaEH+P//jnufjnr/Nei9+Jj/Oer3Pv79Lz5KtnUaOvXCznPDSZkucoMtehKrA13lZby1", - "gCsO/2PjUp9qa2HdlT4+rbsPzFxDHlwR0ad8K6JjgcVii00puzmMsCJSlWe+/F3virhbq/otIxZTUrjv", - "wBJxuYVFMPt9a44FxUx9NGNEw05EWXqzhePwYG/Y2eyiGKtAK63oBEaEej3XjhVA7gqjdKfSR79xhbA9", - "2rXYPYSX6v1OBY5jLDSPz3FEQ2BvI8l6koYEYRYiyuCZfiMlEgmiUsHQ3mDgnowKg9KmNpduKsMOuqZq", - "hnS3CKaFuEBYxAd75qoxIAm4MXmIK/e9BVLU94reKH/fyv7acH/2P/598//85U6ZFRhtCZeyqebjNfm0", - "4jAHObMR8WsiAm0oR0TPVna1rUyV7MIyhGBjIs6ixQ8oMFqz8QFxgQgLLaXhvTI7x4seTmiPmqEaveFX", - "wqZq1jk82PUSd8P+0fv4N/dTA2FFGhGPFHrLU9DD4HExgs2NIdO/lq2Io24agTcupuzUfLZdV9O+bIXd", - "RJattPHANy61PiWy69UVA6mHMmrTLfapAXMihN6KRq4dn52gjYheESvQtKmIhulgsBvAC/Ansb8EPI4x", - "C81vm2WdxTpjPOrK7x0SzDj4k6KIrxM5BuopaCc4WupuWUYaL7WPs3brzpmfuVS9GDOsjcd8AGgs+BXR", - "AzXxJJRIdEUW2lRYoKlutDenEoKZCJujOTY3Vv0hewf3O/CKeyQhLoTOCYp5cGWiHGccNGUjFrvoekYj", - "owoKgqNcWsaYsiHTAronA55oLY3Z12Bq6JKw+SWKcQLbHAsCe1zLfiIojugnE60KESokpFpGDxmBjYES", - "rPd8EHARQjAZRwQHswIV/irRpfErXULzl5Rptr40G7MSl/ln5837dy/fvP/tZPTm/NVvR6ejX179p/7Z", - "fNQ5/P3PjolKzizYlwQLItBf/oT5fjZeSHCOdI5SNeOCfjI3fRCLJRUo/jihfZ4Qhmk/4HGn2/lb8Z8f", - "P390fjMTAjHX28AzsM9eRdEoOx6RdOJukiWyt5MuLkaTTIuo1+fvt7T6lGAp1UzwdDorbwyru621JcDI", - "oHw0TqT3ku8KnW69QVqzRBHVGzTTJLcHg7OXW3LY0f/Yd//Y7KMTs2th+FoGcWEVXDnT7JMFOB+fv0c4", - "inhgr7omTbGMriufgCdMiUWiLe+Vwil/tS6jer386RqiaGtM2ZbUy9AL1qM78M2tPb6v2JwKzmLCFAJ1", - "aRwRWd4rv705eTV69duHzqE+CMI0sDfa52/evuscdnYHg0HHx6Cag1bIwNfn703EHGwbgiM1GwUzElyt", - "+vBnePcYXoUdp5IonYKNW1+9o4w0KCYxF+aSxH6DNmZlJcVseQTrOuzsvn5p+HL7NbCkW08bmpS1Yhqu", - "RJO9fuljtNkiIWJOpe8m9efsmWOaelB8aVtYc9zxe+5KtE6MIOJp2Ct02e1MqCABBCHrf/1BYm2kzT+V", - "o6E83/kvOFsZJyusDhwllJElZsdXol1ec3EVcRyCY+EulUsbx+cJIDcPyuubWUOOJWp5GWPMwmsaqtko", - "5NdMD9kjku0TlL2cyeUbPRMc/e9//8+Hs9yG3n49TqyQ3t7Z/0IhXRHLumnvLVk2kTTxT+N94p/Eh7P/", - "/e//cTN53EkYHeZW+qBd/1emhWpouc24MX70hoDE7ODPI2u4dZbA58jx3sroQ5+M53MiIrwoCF47ps72", - "AKRfZVSCQi4Rst8VvI7LxbBuzekHr6sOnJ2BX9B+d1Z85c4KQUD0jpIsy23ZBnlr3s7tSA/TeXjupT4L", - "7LnfhtMyRtveObN/7tRZzs9x8oomIzCoRniaRW0sS4y7uKKJtdLgC7NNo8gI+jAFu27MueoPmQl813sT", - "OIfckAB4QSqs0NH5qUTXNIrAuwuHRl110DZfIWMCXpdK/69IWReNU6UNOa4Isia1jcDTY4GXxwSlDLsw", - "24pZZSdYj1oGslwRwUg0MmaTbEkZ8xGyHzUSB6Y6wdIm6giVJmV6nfxydoE2ThYMxzRAv5hWz3iYRgRd", - "mKDlzTL1ukOWCIh+1p1odqS2Xz5BPFU9PukpQYgbYgyNZf5xGwM6f33+3kYRy83+kL0lmrCEhTbd0WkU", - "NhUu5OyvWiKTsNxssf8K0ZsixSXDiZzxtpvrwr6e7672fqZuZx4kaXlJd7qNqXBzKlSKI32WliwNb9yw", - "SeX1WJQmU7ho2dpzLU8dVOXYyrbOONMy5PV6kwY9PjWjCbf2qRW8PDXvmnNB/NlusCvaP2VuIEt9irkX", - "4gv6ujCN1HICzM9dN7NbUOk0o0nFE3k35DmSBa9NqxRcc7Kb40+ijUuc0L7l437A48suuvxb6Qe9953p", - "qPXHa2SoAfKE6Z+K7Vf9VSs9SWslvRYXB8vbr8eRbEygQPNtpARm0qS+zHBC+uhnEOJIkVirJpBxJlGW", - "MYIYv/4BcaO0uk+HTA9Nog0tKy05Mn+ipFNG2XRTm3H6YMJhaJyOk1SlQr83pzKnZpl1nGOvphma0dn4", - "c8gTpyyI0pCgS+f8uywrRXXXYN3kt77CmgVrSAKWKxjzaitOle5eTzhTNHmqTD6KnXo5V6jigFwVEmnH", - "kkXn3GL9LzJxUYUDmHtMWD05e0ELHuOC67rJQ2wVFb/3+oosYMmdpxrXfNVFJ7XflSyI5NGc2GO36OYe", - "A+ABN4pT7uE2vmrrntbbv5rq73PcrloKTa/W5C+bgh6gA6l6brI5x1jrzqWaOimkJ2f662rlXxIgPpiW", - "hwjUscuusYUJWB6IaWaJUEgFCVStecqmQwZR4Jf2l75t7VJvcq2j3Al8BGRjg9JeXFpUWFmn9kEzemo8", - "pkqRsFvWDa4ISeTqSWn12t5peC5eBLkW1Akyl6vYUj0jbMJFQGJrJHyZY+BVobElQSVtm6gHZRv6Fsbs", - "stSxjdWBtTfrAR74UvJ6FcMlrFhtJgi43OUljqJLtGFf2kSC/BMSfO1aMc5yZn93fO5YIAtZ+XDW1Ryp", - "pcDlTKlkpP9HjvQuvqw2Zr91OzzH13g+APtqb2/Xrqp1qpoBV5ot+0+9gc3NS+PU78ZLV80XepQ2UryN", - "Kn+cf5I72a8oC9s28It+d3XElLM07tsBmwjSS5OpwJC5d5fu11tfqQM1myX4CjSj5dFwqVQ8LqbxbiwP", - "gisTa86jXogVXicGzgy3Hu4WL0xTxhbz+j2aYgsvqgGFpdhbL6bJl8Y3uLH4lqUpL9hYkCQcKe5Jd3WH", - "xukJwAjYd1slQEIas+Kj+YTy5VnjNnathEJijiNr1+omeklArTsBdJxgZnLEDBFAafxwVrzW7Q9ZD47f", - "Q3SSdZA1mzWJQbfEobkZ2+CiMAiTH47Gi02E0YezPnqXjfavEmmDZU5covQMSzQmhKEUrhbgNOyZs7g4", - "gFTCoamqn1vficmp3oTba26f9TPkJfDSZDhSEOY4ppX5GKwTWCgbLoBZ0QvWymu1LMPgLZlSqcSyLIMS", - "qsR+b7Dd295/tz04HOj//6/2aQh3nzbua+uoLFts4GhR+hy/Pz3Zsc7SzVsnFtx5YrlflJ3kEa9oI5VE", - "9JyYBF+3J861EE7aEMd66/DUe4o2zVPolr1rKPFOv3kfGfO+mH6bdLd+TntVYK5MnCxMrm7JLxKwO/Nd", - "UtDgbGRyQD0phF2IVnkpCL4CJJD6uR3jKZEjc575Q11SaeKvyI31bgjO1USae/Gy13N779ne892DveeD", - "gScnu87wPKAjiD1vNYA3x6cowossPWkDLjRDNI74uMzo+7sHz58NXmzvtB2HucJrR4fM8HJfoQ1Lkb87", - "tEb3pDSonZ1nB7u7u4ODg529VqOy/uJWg3K+5ZJK8mz32d728529VlTwKfSvXI58VYH3AbYcGYwz/a+e", - "TEhAJzRAkFuC9AdoI4YjjGS3VeU9Ocahg7Xxnx0K00gujYgxndk3jaMtTiNFk4iYZ7AgrXzRMPMTaMmL", - "E8hYBiO0XksWWWBlBIibS/YKqDMhGafTqU3byUh3ZnCSCsoTJVF4aHboSjkHq5kP7GMTH9g5tOSGX7Xp", - "1IvInERFJjBHlwHsEQRlfGIWrTQrd/lLWZJ6WaKRlD+lAnRR0yjCY54qc81ocZ/yTiBjAWyPiRbX7ezc", - "n7i4WhlarE/iDN5qpVfoCBzpE+uqgVMcZymlNsm4oPRl14Hm0tQ+l+it+cJ4iPKfk7SM7dmFnqwniSFB", - "pOIgSa3D0DbTVrv06y3gLHXxA6a/XHY+UGxTb2LCQe7WwhZTArBuaqXGojnFBGhcwOutMxX0hysdKS3o", - "zsj1QxAdcm16mm17kuHkfii+LNgw8zXkL8EpLGhI+gh2F0Q9OYSPyk67UDxJSJj5f/pDZkP9s5+kuUHR", - "Hxo6qBmhAnFBp7TccdnBdp9Ri+uwouOmW7Nj8cOmKCSZXfj4Nr3JEp+YIEADXlRMUrWL0Ol2LjLAOyuJ", - "yqR5m4EG1iiSR+HWhvj6/P26sYeJ4BPqQ12FWAj71FpmLirv173BRW/7P0yEreY3UNEoM/ETMQ8r+HT2", - "/XYnz+vz9+dNY8oQ41BxdLU5ZREvyyB+HUXspZK9lbQWjGN/fbBkneS69wufLjsROCbjdDIhYhR7nGs/", - "6efIvJCl7p69LOuzWm9uazWflxYHzOYJDiibbramvschV5lGt0DNj/7lemsRyJqSt/VSZShlJm+7j37L", - "MPrQ6/P3EuVRSh5PXSU/uimV4ny2kDTAkWnRAOxQVnSwAXO21pDP8w+tK9KjJ/uhHd1GQBvzaZLCNrx4", - "2zt982ErDsm8WxoTRBbNeET0uDcL0mLuMnLzvI+SkJg3eToMY8i2G6hAq2wHtyZSYb96qKO4wtFIRtwX", - "rPFOP0TwEG18+Mkk5OkRdFFSWkr9e4EKJf4+8O4YAMNo6PYCOqy6TEsb3Gs7lmsCGPdKYXqlTn1bxWRB", - "1HWcOq4qvyovNL9ajeVpGmnu99glalSc2laRRCafA0E+h7t/RuZT47W2rhFJEiywItHCaBbZ0VfGZiD1", - "q0RyQ4I1MkVe6dc/GzyeVJCRmgkiZzwqX0DvduuYzhKCIOfEwtiZORUc74qjGIsrOBidIo1SZihQjqHd", - "XYWRMFMqWWNSP797d26sa0XEHEfVKGxZu1o9IRFeoDFR14QwNxUsEUaveYYXWE2bkg1QYkKNEiIoL9Ow", - "s+vp98IEZqKpwAFB5iuHCG+XRMKp2ZaUthcPTm8QECkb1nd72fraTydp1G6NfcPaXllAIVhngd8dnztI", - "rAxy3JF5p07lcyJ6Zss57PHlS7sjl4Ozua4YZ6TemeBjAmhtNvS9mMPlAksAjU5/XsqbKggHWUxYsv3A", - "LjCk6pp9/rGVslfd7r6L9Biz0AfdbiKeTXLqNI31gughixTujmhook5MvqxRy4uB24LgkDIiZSWzL0hF", - "1Ol2ehM7q8OtrYgHOJpxqQ73drefby2P31sauGnjVEYhXWbfuWgWE+/gEuAMhDpMuswSWzhJWnjADB1X", - "nA8gnuqBYgDXrs+2gobnIocHg1qS4g0OlAOYBJdY6coTF7dtolmyNB9oMCtxsv/iRXF/Drx30HmFIcf+", - "WzXe1zMzwWSaR4y7oUJHw+SfvBoVFz4UQS6UTUEaExdtlp2HLpbLXqqUOns+eF6cZatCLiBsKtvc7jvP", - "VM3b5bREUiC33b+2AYiLKescbksvB0LWdFnBU1oi1lhqOUF5Qtha9Nzf291Zj55tJ3LqsiYrcskHinB8", - "dmJ0ooAzhSkjAsVEYVu1qiBkwJekpQzAVGESQ57C5IfloqUhfqGIcnBbjL27uv1ugNt8a+I3QxRjRida", - "INs3iz3LGd7ZPzg0oMAhmeztH/T7/XVzv1/lyd6tlmLL5LcW0sD7cvZl63APKd5t5vJn5/zo3c9akKVS", - "mENrS44pOyz8O/tn/gD+MP8cU+ZNDW+FI00nNfzocjyYNvjN74eFsklO72lVa8TvDIawUMCj8AItKTzV", - "uo3huPtDVMp43AzgNsmKS5L3tLryBircNdDm1pDQebEEVYCCLqb5tICFpp+W3287dxe8Y/uE8kc5Ynb9", - "ZvtWmOdyKSxsDTUwISwDgo0i81fA2VxvVx8qbOmIdM9agArCMWLRA7Muiz9mvRd+PC4OpPC7AyUs/GTh", - "YT+uGZKyVCH9R10PXS2FnDq6YjP7fY/ZqdAWitsiMXnO50c/C28TtlXu/c303//4/+T5s39u//Hrhw//", - "OX/97ye/0f/8EJ2/+SLQheUgXI+KpHVn4FmmvEQRQastK51hFXh8dNr8a6CwfWI8DiqYAXgwGpPDIeuh", - "X6kiAkeHaNipZJ0NO2iDgKUEX2l1Vzdlk2c39cfn5kZRf/ynU4M/V9sIbZassAuSgR/IdBzyGFO2OWRD", - "ZttCbiIS7AL9V4gCnChw51CGtP27QGMB9TLtjU/eeRf9iZPk8+aQ2YIfSugZJFioDNbR9QBMYUdlwlDt", - "6yTLgzeXLkOWndYZCpW59utnyj+Ee1STePxEWW6/Wcvp+cCH1wWJBHohIwCLRtkFGpXA6FmGA3o+2Kzb", - "cytsjIyHlrAf7IR6IV3HlC32kmFg6NoI7pFzUa4Iz9CyyewRBBak4vDfC+QaymmRLbG5NzBpJdJcO6tI", - "FhJKNjveuliwui0nZO5d4bOoRSL6K5Nx9O7XC6SIiF0O6EagyTmhgZ4fRJRSKVPNihSjo+OzV5v9FpWA", - "gbbZ+Jes47tshtV8YXsP23S9nJu7OCZddHoCGV92h+ZqLURq/8QFioyAyff1IXpvsX9LTSETKGpWMlrk", - "l7nmBBh2Nl2LSVVSHKK3mTaNs6GUqg+X74fzfQnN2lgeE0Zea71bq+spnCZtRRsEjWOV5R3qE7dZFLR3", - "3zhYEb3nK86Gtfd28X690ZVQWPu7xmi8e3Vndz11x9VqS2ZY+rh7VrwggpeW1HOlFa+3aI6GKPW7skZp", - "1h1UiSVZcQPzua+65H5ve/vd9t76npB14fXKMCcFCKQMYa89NN59QMzVvQI3VI0a42SRfmyjYp2J+eEM", - "zbBkf1XwsGJobu8+a1XTSvfaNsK0GFvKJ2ZImZRymClZZKRBj7miUWQCjiWdMhyhF2jj4vT1L6e//rqJ", - "eujNm7PqUiz7wrc+LZD2nKh4ff4eLhmxHLkgrea8JJzn9pEbKpWsQ8q0inX8EmQ/82m72gRukqaNvETB", - "cnjAn0sQfl6MoM07xPVzAao1Mj4EYt9jZv58fWiBS/H9vhSkzxov94TR1yjcffh2ZTlvfr5btL17Gc7K", - "MurFs96lZd4a3q7boZ6UtCOpRTAJ0el5XgIh93C65itzerHT3z543t8eDPrbg3Y1b4IlfZ8dHbfvfLBj", - "VItDPD4MwkMy+QJHuGVso4zj6BovJBo6c2nYMfZZwTArbFtrUrW6tK+jCN4ONLCq0DTAAoJi56IhpC2e", - "2aTctMg5qkLgxGlk0lqLBTfL2uIMSyQTg/dmoKwzXXbIYIBdizsDRwWNCcJBINLcn+EqHxnNN00s3w+Z", - "IDLhTJra8330C1lIFFO4Wcm6h3gqibLQ9nDINoRLg8jyHRKcShLqHyDGuOtiWfXQqALIbv3BkMlZCkWz", - "N/vomDOZxkRYVw8aU3BDbyKZGuMOxgvUWGiBKWlIxJDp1zyIcn9mivrhwWAwGGTFvDuHu/rfAx833euN", - "St8iJppEPsBuYhY7EUChRMpQykIi0FsuiVIYEcMO1bibNW9jvhAq0X3eTo+yn+cKlH9jrkJWbAeZ+KU4", - "dcsqiF6Ua4e2NlO/oOp/q7wCp6jajAL71Wida1GCAp5GobZ9xvq0M64pEloPmiQqL8sKB+R7dsX0Hi1N", - "3cYzKo7+SIlYoA9nZ6W7VEEmtupki4mDlGhYB56stQw7K7wFK0dzS/TCh0AsrGoqBQ3xzvEJi9cWLrPR", - "cGiL64vcYvRGo1NmlkbzyZI5VRzPIZmP0tRniOhHDs/g/fvTkxJzYHyw/Xzw/EXv+Xj7oLcXDrZ7eHv3", - "oLezjweT3eDZbkPd5/bZKLdPMPGanb5Cvi7MdOTCXX1RiE3BxpVz3wZQXlMW8uvS0eKNaCv2bqPlVnVf", - "j4VtPQRvBD3UlISWGqTEGRygJNBtmwjSSpnXBnfawbvB9gp32kp5AYNrkL/vRMoCAzQGkjhzVMeFARcX", - "qzzO24lTGJCLVF9FrWLn7Yk2ONx/cbj/pURz0darxlhlpwdc3KYQDodWWQnndilFBaeMQ3frWH3D+I5t", - "9Hen28kC1OFvOGgrwY/Z41ZZF00btusXI8vkd0Py4WnJFoCrZQNaFR5qLSDLWxunCmU5rVq9OI54GqKC", - "Q8tg+MBtz2nBLtDNwOWL9XcZMDYTPa3tB0D/BOxpyrQghlsu3YjNVDxEr+FdeIRjYzLZQRgE9OIFDw4X", - "5oJb7y/XtTFglg/5wtou8I02ZJD+F0xbk8H6PZc3YTSfQ/Qbh28yS4rxqgPVvA4mTP31qrN1w6YQulRz", - "6MyqcYfop0x1y5Q/q+xtSGL/HFmBlSM8bJbybO2KdzS35CtXSCHtdgxFO92OIxSkmtaTTt/nXF/bf0VW", - "9EV+EByZUrdZUl+qaGQBTWEmVCoaSBvtrRe3Sb+wxRlIODKGSVMQmckUs8ZL9pFTXz6coQ2ALvs7st5h", - "/a/NLOCsdNbtvNh7cfBs58VBK4CSfICr1c5jyGOsD26lDhok6cj6HZqmfnz+3vgVAmOxw3WDnXshHzwR", - "XIsePXPXYLHzF/0XRVyWkKfjqHB9ZUGcID3JLJgXeiiTRQ2BS3/QaE4nE/bHp+Bq55+Cxts3B3Jn7PXS", - "Zh35XVqnxSvsmv+XjHum0IIfOgMYSshGdJm3RMIM0AVRCPinh3AApkOWfmhZzmHQWIp7GWtvd3f3+bP9", - "nVZ8ZUdX2DgjcHB5DmU7gsIWgzfRxtuLC7RVYDjTpsvJBixYZs1K/z5DtizfoKyQ9rcHuz4uaTi4c66x", - "bc/jRpJ/sKaZnZQlOmRRZmZbbZd7qb27O3i2t/98v902tq7XkbhZLmFcjoEhj4UsLq78BmiT747OEWTw", - "TXBQ9pts7+zu7R88e77WqNRaowK4bQOTu8bAnj872N/b3dluB5PkC22wAGClDVuWXZ5N52EKz2p4SFEX", - "vd2m08KnThkGe0uCCNP4KHAx0ZXTx8DhjoR5LV+ENgeD9fbXDq4W37ZyHGXuIBNRb1QDLlDKMhD2/up7", - "zdtdUzaLaXMerBbjvsB5psll8TxM1ZVb0C4RZE55Ku+gIa5MVtsk4lys9W2ThfKWyDRS5i6RSvTh7K8g", - "RDRzIalIUjaaLPstQT255eTW2sAlnvBzdROxWq1Gm6VfNuFuwzbtLkt5L23/5kL7WlSlbHVM4TGOghTq", - "DOBsPfWsACYEknaTJFqY6Nso4pyhYIYZ3DgIW9WETRFGMx6FfW9EpH4ymnhjEfg1iriBRb0iJLEQ/GYQ", - "+jOts9A5QRuFpGFkWKlSKm0/NlLFgqyXuXE/9td8wtKXTpIlq2p6YsULiJ3mk5KPMeJTCVaggrjifhUo", - "OsHChAtjZkpKzGNjPJZRlnb0ae8ZYkV6+45Qc3TyibVorY4BqaCGkjgQXEpEIjqF8gUfzioZhkuyUrI8", - "w9VxguXBtmBdczvoObvgTJOtK8/4DkRPxP2XHInAw5DVsyQCz3kjY8xSAOUvMDK5Sagw7NEuym7GpRpl", - "yDFrDlaqEQCup4Lk8FJZXmzmAHLveM9FJ9puQy4bznqrr2tc5W+qaYDNMtVLUT+1uhkP+ti4jp2zFK4n", - "x/+pgr2sg+6UI3RTCa3SArAQ2mBclcRSAWV6s03Eid9G1f3UzFNbUOzXvcFFW+Cl5ThL51jNTtmEe7Lz", - "17iGdK5nGwSZEBFTKDmAQsIoCZ3xmN1HWt8WpHRGkqAwJZZyRiEV2BIcm+0NGfbMOcUom1ZkfbXDNv5g", - "M4bleOzQr32xTeyQ9Ke8vRMp0MpE+0mE8+S3VqGTVI7891f1hgWZphEWqAoutmTIchFHlF21aV0u4jGP", - "aID0B9VL5gmPIn490o/kjzCXzVaz0x+M8tyNyqWxGZzN3DELUuk3n8KPepablbxBcL1sme+3ANOhTSiW", - "NwD5JxoRi7/1ntGbAqOXAYv3dgZN+awNjZYyWevYbetKbsuyvh3vYNWOsgK2nltKE0pcuSstOyJX3vRB", - "rPqy7N26KwZtuOguBwhdpmsBmLmVJ6RduHw1jtGNZkuSoNz73vP9ZwctkbG/yNdp0Aru2rM5j5d4NBtW", - "6qyN2+z5/vMXL3b39l/srOWgciGvDevTFPZaXJ9KneqK02wfwsMGaw3KBL36h9QQ+FoeUKnm9K0H9HnJ", - "1m0KLsj3ZtMlZ1RcSXfPUvaAtvMxLtGWjkoqV14nG22QyYSAUTkydOvlg6mkPbYaQ4ATHFC18DhM8LUJ", - "xMteqSD7tfGmlQfrIalt26JAackl03GeGbLhOkd/M671Ci88bw2wL9Nxkxv/TbVX48TPfUDFK6IWNzR5", - "DdC6uyCbzzWWpVgz/XcAIaEu+L0eCGzeWI4yVo3ShEtAW0eiEEnhQ6esnH/2o+LyV5az4PYtKclVii87", - "Qpu34Fo2tOdE9pjQwer0nop8sAfg7b4ajYulL5bWFinVychP3fX7bZHBVMeFzU6w9fsr5HKs82EVAw34", - "0Y7Bkjxvu1tiiQZuKoTpeswRHpFeFudgY3iRTI1/Ve95C6vpSS4JrvhkUsb22m/GggQQRohDd71gpUic", - "qC4iN2Cmk7AGJOhKpu/LYQdxgYad7XjYqTgBvZkdMb4Z2Q7KCdiDZeCMWdGn6iClm8E44sGVqeqgBCWy", - "jwYoJphJlDLY/BUf5fZgua+t20kKa5NBIRJzQ1wTWzCmMZnhOQUEXuuhmpbiWMgNVRLibaCdQxRysG/L", - "Ja3sDPVrJu/iMJ80HDqYLWzDukH9HmcuICh/F8ylCRTSYp+I4F2bUagl9ps3Z11z/wORG2ZgpfAQN1Ez", - "Ai0gsy4qcLL57/7wq3FERjDuKjxpXKdjMT8O/NSCSKKkxSvM2aHCBCjgKVNV3NK4XQhnOeK9fiSlDGIl", - "7O0Z4FDY3m0hzJAEtthw3b1UYvRbMHcl7NJS2hd3uetjYdgU4Jnze97fWvd6dQASYUGK1ehMO8WwOONz", - "HUnFbfmCbFePyE1ASFgFOPK/0jbU0H7pDTX8FdtE/qxQnH0bwsXqs+vfX+w5jLWJ2sWQSMZZD1KI3ZLa", - "dF8DhWITysuMVoJaLKTVjnxwUr4X2iSDkZvltP6N3CjAgwzTSJO3iXWtqLKH0SqK3zrpomlDc7G6ruo9", - "lMkw4Xq3KpRhI/0epFaG/fle6mPUluOCKPfuheWb5rKoJSTrkkfQBUi6V8pXlIZ3usie6Gg73qzw3N7M", - "7waxqGMt80UYjskoEWRCb5Zwi3nBWMLlFOt855RK50q0EeMbtPcMBTMsZGXsjE5nKlqU7y/3PAgHX1Q1", - "RhBFmFqjyHC+mu7DerCAXc5i6z5t+KKAR+AvfkzC0TJgwOPsNXcdm+AFuG0afazPdvcGg92dwa2QAe+q", - "JnOhnaYUhMJ39p6kFNVTbCFL+KoX7roWFPLGMjJJJQiODyFQOcEBQRGZAHRNVjBx5WFR63r54K0GZZPj", - "M/53C2XXzV1hlDHBs64sqKKbRscFQJXxEIrP68Negm+TiZmgBnTjyVHY7Q0O3m3vHu4fHG5v3weaX0ak", - "pujYZ5+2r59FO3iyFz1fPPtje/ZsuhPveg2veyj/Xc75rFYDt3NIiKhWZKtWMpQkooz0ZBZRvjqtY4ks", - "MEEaK/f/eo59M4OlysJFeZJFnQGrnDglznog8A87+qW3E9Xhn54sH/atQrSrA/EzWHUowE/tBgOYu9tf", - "Wl03ZS3PnfeFF1ufPEvTBladPb4sT9ja3lVuoLiPn0uCsbTDlp3Y9VPN4x2dckHVLF5+PGSvZcCIEGf2", - "SaqwDDbRR6dTBvUXiz9nYQVFM0l/3Ol2ok975T1jf28PO2KBADMGtEtdVANaXLtDec/lVIBXctNCmMg/", - "bY3rMf+43dt+AcFv0ae9Hwe9F330j0IQXtdQq0i+bfd26ddBGxoWi5o4MPztF2tFqDl6LuOgX6ivJEd+", - "EFuIQMvjefE7d1a4jKTSAuePa2tcgRFo1Di/VLWzp9moqCWFJMILX8nygitWVuzDIpOhMZlSJtt4ZncH", - "mWt2Px52+ujIAmuCtZrXNi01D1UtC3xC45iEVCuVxrhvjvjcaeltqxoP66Etu6886lnfr5+9WJ1DuipA", - "fdUx2f+ChKUvMnfbmbjL0pvBc+ZsUgAwgRe7iE4QZpWSS7bAsM00hMwRgM05dCgxOctaGSBzxc95Qrpo", - "yhXKcwxbetRS1uz5y8ZPbsCjuiSp2DDEzp1kjGfoJXSZ+Do9QYngYRrkCTYRDDpPiRZpBR9miVa/OoTp", - "Ph0akLk24QKtdmg0eTDaeSCb1rvifdQM27zU24PVS30vXpBuJ03C1TLMvNROgq0FoLoiZcPjkymTvaIJ", - "FibzsYVEf1ukYN3INd7iQKtEaeKuUDRP1TnJc6EClwg+LMETEhF9TNUbQTwK86hSKnMpulqkbh88nzVd", - "YsKdU30gvxCSaFsFACKgvxizhXdg1Xq6aGPgiqRJc6XVMwDsllrlwT1bqYk1LlX70sQVr7ZJKC9Wgs6Q", - "RO+2LrH9cmXV+Pvwwz2mkvbGXi5UMOMcomGGwer6NxkzgDnJosp5vee7gPexxTtrGTfh4FWzZopu5qPe", - "fxm3Mhr1D7d+/Pv/2/v4N697uWI3SyJ6IZlAKNEVWfQAxB9pG71fRoEDAGKtTE8tqxAcg9MouCLGSRXj", - "m+J49weZ0Fj8huPaFCAGK6Ys+/fKCf39L80RTAUyvgc5uZJlvxif+z5KQinujqONmIipK1rtAu83+0MG", - "sfxXZCFRocKCVWkco/5VZp9oFR2cljhCl0YN7BM2v0RjCiVr5JBpqxYHAUm0NWGR5qmpu8hB+giCo2I7", - "ttKDS5SzV44mYoCgD2c1CME379+9fPP+t5PRm/NXvx2djn559Z8QxHHdMz2EPc17e/sHttpikZLbniX+", - "AjTjL0Lx87GbwQLz8BckpUAJS4/CTCWklLqAgsLLaIPEiVq48k0ut2VzPWyyo6xBbzjbHUPLD17cRSWd", - "90tL58x51NMadQMysNeBaWjhDceGpkyYe6fJsT31VHy/sN7EKZ1ijy/bW2z2LireuAGtBI2rrX9jvQp/", - "cPxJFTbZSANDqgrMb8UularXHDsfa0VqlJf1LEdkpMyml9BCwFY5lyRmastWpvKltIYcwDuXJRTlu8wh", - "FvXgo9V5MktV+cLMCiNpXpszp7FWdOolBDrXpLmeEUEKCwEf5HC0a5LMJnu0SJQ29WMSIvJASJcpAmXu", - "BYXskczZ4EiQJQTVPbDL4YbP8E3WA3jvsazdccE8cuD/7dcvARr2ratKSyeuCRhGxZ7wA6GWuWgZTRxX", - "1RejyFX1eZv3vRvPyqol0q9pb1WYM++jxJo+fvwHpuonLsACaU5Lvnc8VbBuQiIAl6WKltoKapTGJBxl", - "tbmb9r8rx21ykrPK53ltKmdtYWBiLeRW1w9yibP5GOqU1uQgQSqoWlxADV8TIUywIOIoNRvelQK2P+cd", - "Q6mnz5/BTznxZCG8JowIGqCj81PYjzFmoKSjD2eF8iymUk8NQw3UyzfHp9bCdTB8YLFQBazngvmOzk87", - "3c6cCGPldQb9nf4ANnNCGE5o57Cz29/uDzqmeDNMcQvKUhIB/7AphpmtdBpaTeile0l/KXBMFHzxuydZ", - "D4LZ7Oug9eJpwW5JMBXWcEkiSCI0DEP11wCv6w7UQ3Mq2wrMrd10Ui1sSgVJ3tjF/QhKJewdmObOYGDB", - "RpU9fiEhxEShb/3ThiTm/bbS6iyJPGizNcvC6ZYZ6T93O3uD7bXGtGwosHd9Hb9nOFUzLugnAgbh/pqE", - "uFWnp8zkeiGDGmZDboo7DhipuNd+/6jXTKZxjMXCEaxIrYTLJsWYSITduwYDUUkUaFEBmPh99IYRW4sV", - "K4RNOKxImYT4C/uh5tDyLjBtu0XOcARe8nBxZyQs9eHM4s9lcaa3y+caP98d72RsXF9I+8ihXhqufQAG", - "eolDV4Xs0XbK3uDF/Xd6zNkkooFCvYyBbZArlRBlEgGGp4MH4AL9kXKFURYj/oS2tNVZxxm7dfOjaOtP", - "Gn422zsiPs/rORExZibi3ryzYtPXtrPxgufbeemp5hgf4LbhpHJ58uagAkWuvEWLx1ZVGawfR3sNlZOJ", - "sNMLH5Hx9x5gh9vJZkXBHnPLQcUplErylLaTvdUZ50qIV5d7TdTXwvODhzyyLLDvN7iLngoDvyaZhpev", - "Vu1Q2EpEyowB7NUA3+ZZcPa7v5aVv3f5k0JgBrjSddMAMa3MVR4OF33kaGqMfrUAxGxBYJ5h/Vg518P7", - "WnbYzkPsMJhxdjnx/Zj6fkwt2+WGW9wUYGMWdnkLH8RaHohvz/+wtvfhu++hve+hleeBkWvrXfgnH/eR", - "DYKEirxyxtMoRGOCDIiOC3dQWPSnnxAWwYzOyZDZ24I4jRRNsIBghhiFWGFzbdvomFjqlsia29LN9Vzo", - "W07gKjiCJCOohjQK6ZT4Jp0HvVHGSIj0J7ZqnP3EVz7T7H2vgz1rMD8a0fWMS7jakFQqKHqTneaQMyuN", - "dQzN9ofsncVi0wSE+F0naySJAFFuif+HM4SHzH7wgxMhLvZI4jiXXFgQzVJQDIKEZlnq6VN6pCMZcB+A", - "yzvCMFM9mZCATmhgp3VFFjaE0Ntgq1oIesBunB/OshwBtLPpBwGD2rF+/LyT7BmynFS+v2EQdxtEaZhf", - "cjlcGizGOIq8YNnTiI9xNDL0uSKeO8HX8IYlSrHMrrtNYjwkpmRqslAzzszf6ThlKjV/jwW/lkQMO5t9", - "Wwrf0pqE3VxBRNdQXCWGcvQQqQV9bpkhbv15RRaf+0N2FMaUOY6AT3AkOSI38B3UnAAgBiO9GvjB7Cb/", - "PfhxKhWP7e6BLB7Hd2aYPFVJqmwSgyTK1u+H17VKmsoZCYdMcfSnIFMqlVh83voz7/EzXBYTHGo+Kbxi", - "pgS6ddOo5Qjr2Y/gVc91OwECDDv6IB129N9TgZmCMYFiKEG9nhaXdCMDMNabdLNK4QAzlPDEgD8DU82w", - "ZrlSGwAAgKMIKdhK7lutuMNKNszH4rnF40YwN4O+VdlGlKGzl4XNNNh77t9PkgSC+CJK/v3izW8ITmW9", - "Bua1PELIZBEwrTCgMIWrUyfTXuFghsxFFRT4GXZoOOxk17nhJow1lTYlvteDO8Uf9dB+NN10afhjv6+b", - "MteVh+j3P00rh3ovJfFI8SvChp3PXVR4MKVqlo6zZx/9BG3CxLooCQK0YY65TZAkmAJ8SeHEN0ckZiHi", - "9hSIFgijXAIVA1fGlGGxWJa75iG9pSCfmOC5AjH+HEKw3LBzOHThcsNOd9ghbA6/2Zi6YeeznwL21rK5", - "mgycZ9nlZsZEB4PB5mqwSktfz51li4uBO7YBG62irBSWXsE/UpJ+c/cD/9L2Z3b1g5nuPMe7MYa/c74/", - "wQuIgsZetEQ9VxAVtRuzgERO7V7t6Hn4ywO9WAGJoodm0Mdiz+x6zNZMfGL3YbBY+TZa6r5/ZI4bPNSh", - "UnLbPw7/Pjn/ucd7bn3nZO5Cnf1Q4oBzYk1pZF5GWKILGFPvQhvfr+DXvv2vs/0AqO8y4tPLQ2O6o4hP", - "UUSZDUEvBCpr9cDSEj4yUCfZdxb5xNVx2TCaxP/+9//AoCib/u9//4+2K8xfsN23DGYX1H28nBEs1Jhg", - "dXmIfiEk6eGIzombDFRmI3MiFmh3YH3+8KhYzdtqaXLIhuwtUalghVB9U1JF2gbtVYGeD2UpkRYqRr9I", - "Jxbv3cQ2evw2bi8bUj7oju56MPZgBoUJ6FPR8QAAlFFT/NJaoh2/y9TMueQ0rYZp1oL1VssXRW6U4d6e", - "GeCaAgZI7Nt38MBOGm1cXLza7COwtgxXAKY/2A55M9aM6H+XSatlkpEoZYECVDayyeAnLXf6n9h32nn9", - "bYvfktvflnlZw+9vnD8Am+hW4PsdQIs7AD/d3H2Azyl/4gDC7i9Y0HTxSLGCjvfqNDdPCiR7DGcA2nBA", - "DOBQ5QKdH58iHIaCSLn5r+0q0DM1XJofHYgzgP1/jFtrOxaoCR6TzFQrM8hTEQdv7agRdvOqVs8qnm9b", - "pWIQjSddVhciP/Lu//SodLrOMZJX+Mp57ftJsjJOj8qA628L3NILcAKEdOpLtk+LXLTKIWUiALMjZ6m6", - "ZMXz6YnbkA/nmrJdp6x6NjyAUDypCMRHFITlLM1iTbynxM3vs1V0YKhLPFdfF2sOHk4Lemgvlo/Nn5Ib", - "K6yQTUtBgyfQeIC+JsqgCHTucaFtD56JXxDhdrUrYQqzzqZlPkUGDgEmBFfzy23fU/NKO9PXtPctWb5A", - "nnU0Fkvy7ypKC2M3p9UyA9cswX3at9DDWubt3d14WwbzEBnCbsbOYy0UCdEGlgsWbH6/9L5zjjYhUbkR", - "K1BWeBklEVYQHQlALJmdpce28wB63VsbR4UEVsTGDT3FZLzzNIrc1cycCIXeHJ8aEVA8rLb+hEiy1UaI", - "EwtLz633b3/tERZwCB3Mwt782p59csemiOGsUoLdw/PzE0wyo+7gbVLFvmD9TYQnMkGpfcr/beeniI4F", - "Fot/2/kJRwll5N92jyKsiFSb98Ysg4c6Qx7aNHjCzKctA1omGogmNgXowBWqdPZWS23avf9NKdRm0mup", - "1Bldv2vVbbTqIrmWKtZ2Ke5VtTZ9PNLdUcZsPmrDo+84Ew/gjrQcWcCZKN3P5EgTMy4VPHp6SYc20pNm", - "HFc8Nlr61fMNufT4cKx7etIFQkLtUEA2tzk9D+Rld+N4cOXW9vvwLvajeEynKU9lMV0oxiqYEWlT6SJS", - "FsBPTe3Oj+dGxfsr5tLBQx4dD65Xf+f7e9L4qwtqhLe5Klul87u32ur89n2t8xuYQZtuaOHXu640x2ZD", - "9KMDGmzLxiU8xnpUpm9cPlsEvdeGSm4uILAgDofs/2j743dFcPzxR5fXlA4GOwfwO2Hzjz+61CZ25liF", - "MKgKDimuR7+dwP3kFBAaodhSnkVZHYepzgqs5+Cl/+UMpPyKtr2F5Ljwu4XUykIqkGu5hWTX4n5NpDJE", - "/YPbSI7ffAS3QL/fppX0FV88PLgFJ9PJhAaUMAD6h2xRWYu0M5bc95uRW2YJMnvTVwjTKWkirc3ITGqt", - "0NDz2qIPHqJ1mhdTeWjr0ZUxfZrZDjyxdQGtvZZrC80G29fGD4OHPb0e3lB7yixmLKI66RKtdHvKdZhC", - "NXGqILw0B/iB+F0kjFmTtdhHx1let0yThAslTbEbsBBMOcyZthB8hXHKtW58xW2goAslsjtkUO5UPzb4", - "FFtXZGFK2VDOsqo12UxtRRhfFl25lNCjbqO7V0L9dZJaKaEPvI1t5bvHU0IfTXQ8iLp3WiooupFtDLC4", - "xyTbyTxL06SfKJtuPqlYYiOssrkV4Mg8qtYWThV3dfC3ZtwgE/nB2c4jHAA2m37NwAbZvF+DE1ZsCpJ5", - "BY8iIgwcVJIqVzdryLLBUVaoC2yLVFzq5kcpUzS67JpoGsjplwizhcVEGbJSZ1gpEidasFmUHxihIIkZ", - "caVgmB405amEt7pI8lKXCEfXeCGHTJBJRAI7NyiuKEhgkNOiqI9+5pBIjfAUU2Zze/WbptzWX+WQXdIw", - "IiObB32JqERyxoUijIQo5nMiy/0SLCJKBEziGGvKSRTjBQASGWw2Qx+eEAP6U8q25vrfmIUUilHpnrMp", - "Hw4ZRjuDAYoJZhJRSMiVeEL0V7YNBIMoDegHhNHe4IX9qrJuAJrpyL+h94sQZM4DPI4WiGguhhNRbcIC", - "ZvsL6jrq5ZtQIc16Ze5FW/WntLBUuvqUYRelLIDyianQ/+ICpcwer7pFATnmME97CUeoyMqO2YT4MQmw", - "pifj5X4AiowHQSp8h6Ne6kJhvH9FJbMwvQsglU86aTog2FMhrDnjagZ7msNW2vyhgatypvo2DhnvJuEC", - "YVTg69yhADWk2RRtAHTXZV5yi7mqjZebP7i9o7evFQRu+xvwrKdyPgET8cmktAFXH01mAy9LXKiz8Le6", - "T49drcWiiAspnjIuFQ2cMKxWA/5uPLY2HpdT1svNEy6uirpVmX9/4uKqrfV14UrcPykjrDjDr/AeQA8P", - "wFcf/zoAnNHGUNFM8+AGWpW/sl0KShdV0sUZcxRxNtW7KHeKP7jXvmLRBVEKarkz5ZwTRBshI/ujKdeo", - "J2OL4YGHP7CtPrYs0r0/wF3Qb1whGicRiQmUc+wZZtOLnWnVptoylWiW1dFbT1bqXVVMyjW2oDTX/12n", - "DgFfuQXbAO29vlxeoRrx6WogrqxzhzrlQeIaMlMNmrjS0Zcok8FaoTWw1+h6RoMZoHKB3arbN6BdOEku", - "M0DSzUP0GjZyEZcVOt8wYNea1ySPiAHbmsfx5WG9YOGHszP4yABymdKEl4fIFSnMzg+p3yqibOlZRFgq", - "9JvFDtvIjHFY0UuFtb2ZzW/T4m/lgLFD5sPiYuTaNkgn6LIAy3XZgMvl5O2vfPpoyli3GebbzEVxZE1H", - "4E3Cwk5TjAWN/Ihc24OBD322JTqYGcY9g4PVBvMrn2YQ4yVWxknSln3tMIGL53G8hIfRRi5BkFQhT9Xf", - "pQqJEPCx5e4m5kYbOLDlZfCVZlRmpJLb2JvAft5IIoP56yWVFqqdboewNO4c/m7/NY/jTrdjx1PACl5D", - "uV+BslZtsB7xolemAKX2XS1fByStLOwLKGmVk8Oa080a+Vvzwjd/s+h8do/IhqAfVJy4X5MKWhhv2eHD", - "OJIMJ3LG1dPCZbKuporW1uyqcbPs6eGFqauB0SaE48J+euG+/Aqs31WRHW7MyE33wUM86iN4ypmwsjab", - "CRdVMJ9VsR9fPSPd3ZLUptqGQ77z5vp+vlaMmfiq8bulCU1NJJwqHmNFA6jHEcw4lwW2H5MZnlNur0rd", - "nVXGmeDcMHamDaG/1Kx6aR3Bl1aRP7ROK4SLj2wfffjcBt77v3CP8i9+KtjlmcTvOuUbMKuhYLCgZIIS", - "nEqi9ao0JsiU4rcFWAgOZijAiUoFgdpSBMWU0TiNC64GbTiJOY4QlehyO77sonGqUITFFOwi89CE0wsS", - "8DgmLCTgIRuyGcFzqo06gSKsCAsWPUmgJuWc5JX+tZFvo3BMTStBNAdSzrooJgqHWGFQNS71jh+ZLJ7L", - "rEylMawZucm5IRwykbIfDM62bvbSDfQSEanwOKJylpUzC3BIWOAFsb74usXY3XuDL4iqTvSR4nJuJUsf", - "M1Cn6PV0w/k6YnieWDAyF3YZ24j5JUqvbDYiy+kPjo3+Nbe0maub4yNd8WQkXraLv467nYzpvpr7nce/", - "wOEChanprrArgc2/1VuZTKAUw50gtdIs422vZrK6TRmZ15J5W3+6P09v4U37SiRht9Gwb6oQkk/6axC5", - "lqq3krmP5Ea0vqSCV+wRRbCLqXo09YmLgpR7Ku5OK7DN1szkdlE6KYHB+uLsu9iuim0bcnBbse18s7VL", - "9YIgp6wHUZp+CW7duI2i2roO/kVzQSqzK4jMRxeR+d3Bg4nF00wQGtGY4EXEcfgthOkuucEJuBAG/wEQ", - "JZ4S/mjBa1gM0AffXDeTEF2XW/nh7GyzSUoItVRGCPWEJUS5Rn8Q+4roz4kQNHTFwY/PTmzALJVIpKyP", - "3sQUKnZfEZLkOSUA5NHX83NIGPUyxyXIi26HMCUWCadMrRxF/ur9DObzrYojP7CctFDR3y+kW19Ig2f/", - "6YkzkDKQNWEmsNwyVVg1hgK60DjKTO1zrZfhMU9161oGaTLp9ZzCKTihEZELqUhs4gInaQTbDcoO2KqU", - "9juzyl2IitU7xySsJUTEVErKmRwym62REKH71p/r9gshTt4LAYUz+XpuhOTXET6nB2MixrBqohpgFkFN", - "+M5hZwsnyVaIFW4I0bLD+4Ih/QTxcEgu4jGPaIAiyq4k2ojolTFP0FyiSP+xuTSgbgTf3XXNzdvvLE3p", - "Uzbh3rJkhmczZv6m8qqsWHMXk09OrL0mxc3i5A8stF+syZVyTRAc9RSNSYZcg1JFI/rJiDrdCJWKBibp", - "J4cs+HCWoxYM2RlRQr+DIbksikignMNmKxE82Bqmg8FukFCAP9slMDgQeM2PY+jx+Py9SQQlMReL7pDp", - "f0DD747Oze3uBFtvQmGgjKhrLq7Q6dabFSHGF0Cmf+EYPTPBpdgB3gX/fiW4PiJI4x6SDVuUJ8tMJZ58", - "80GkVoP77ld4mn4FgGTKZrMxFTgApVjOUhXya+b3Icx5lMb6H+aP01XAXgoHsw/w6lej7ZrhrOzGTfBJ", - "bEo7p5CYsomPculhCPZUY1Y14dwUQIkpRQN6T4Ej9S1y992774t0/AqvOy1FXUnSr2ZvPfTJZ8fgMC6K", - "9Hgq29xwmpuJ4su9T9eYNnufXkY8uJIWDKXoNtR2GwCM6x9zQGh7RQhqAuRmIgsihMhNQgUgv1UckAZz", - "RyKMFBExZTjagjmbRgDa2nmx8JxTSJEOIgpJajQE1KII0OmuZ4QhPRtwVLkGCje60paWKr5TvIxUHI1J", - "wGPi4L43fabbPzBVP3FRxu7+WuTiuwL99Xz0VPU8V8CVN/f4RfDlZ/gGQqXD1F4ouxFtvOb5j8YV1EWw", - "NsPO7kAOO1007OzEw45egWMMLlSs0D6KKUsVkX10YvxbkAR7MECSBJyF0qGOOw/e7kA2pcQatmzIrzyA", - "7x5S7bFcBaR8azvxiQf9HtLfQ9IO2ihuOLsnwy5suhDxVBl3v91X9q2QKHCPbD74XW1hj3y37dtI8n/Y", - "7VuSUbDKWlwWlt5I9gz7eaXXzSVqzAw4nHUaBDjBAVWLLsJRxIPce5DK7Haglw1lLAi+0jZUf8jeZqjT", - "NrkCHZ+/7zqnGQqpvDItWL9YH72ZEyHTcTY4BNLAePBgMUg4ZIqjAEdBGmm+JZMJCSAvAsCkZYNfLRvK", - "fRaCzjvxIl8XIozSJ1dww88TsHo5W8gKx22Zpd4SJIgwjZuhGK3qC5e/4PYd60a5PoYnkb3eCgSXEtmm", - "eiSiUzqO7GWN7KN3WuXAMRmyJMKMEYFSaSKU9NB7iSBSpibZRjcAkGWGo7ooh1lJBFfWTRxxLqTx7GoO", - "/3CGpCLJEjZ7a1o+gznfU/0B07jt6ZEMhsoYmo8l+wrSC2I4xRBc85E+ph8hLMgM6LHrFDyVjf9O0OmU", - "CL0rsBGy5mrUbGtHTrPpS9kjjcV3LrK32hXfyVotRIgXoqeXwmSMcuTBsLPeDayn8yvaiKRiH62X0fGL", - "/qhl3+XMAf8g7KMvnOW3UtP0ohCw3bZkT87hT616TmHkpa1aSnpYDXHQOsvhPrMOWmMZPBqEwVNGLsCl", - "VIYmiIKvjxEGD5tx99DlKZ42b5WQB0oV+xrSr1Zji34VHHg/oKKPnHF6C1DRryoHClAfHy8X1btRHyun", - "qeQHdJW/vnlc0PtKZTLgoACN0ZTKZKSeDSRYaih9sO+0M5Nsi9+SBm/vntfQ3x3Zv1v9LUyGArH8LjuT", - "b+2wYEicqIW7XOSTygWgpJ8gbcMHJpHFENwfhsMtrtfvjj0cnzZern+bhTof5f7eFlKhEp2eeCpgPjG8", - "l+KeKx0sW/rU6WERzOicNDvdyzvYkigRpJfwBC5XQkMwSw93liks+tNPyDZv8a/sv6ASDwCXkhCFVJBA", - "RQtTFUlLBNPHXyUSXFsC8JyLhc+ZXty5PwkeH9nZrDgP7Z6yzrD8zjde9EKscG/upM0SF9oX3LS7u20t", - "8BBl6PVLtEFulDB4v2iiLR9EJxlJTelTCTy5WRzw9qDBs0k/kdF03GaUS5Cb31hkbBSkUvHYrf3pCdqA", - "ShBTwvRaaFV/AppsIvichqbCeU7UOY8MVbcbCLqu31UrFVkZD2dcmME9ig7T5kCafqJJWSyY0IXOYWdM", - "GYbBrcRILu8pk1Cl+8OU2SJjbo3cKL4fYdby23DGjuZEqEpkiag4N3B7m9+Puad8zBUDU92ZVjrt2pVY", - "bher2jKE9D5AeLM45od1W3/4esIrqXySkZXWdT7PDNImt/nXxYKDhzsfHtpd/uEJh+O/Js74LrjKoQHd", - "oo9hfuUBjlBI5iTiCVRfNu92up1URJ3Dzkyp5HBrK9LvzbhUh88Hzwedzx8//98AAAD//+tl1PJTuAEA", + "H4sIAAAAAAAC/+y963IbObI/+CoI7pxoaYakqKttdXT81y253TrdsnUs27Nnml4KrAJJtKqAagBFie7w", + "fpwHmEecJ9lAAqgriizKulhjxzkxLbOqcEkkEpmJzF/+2Ql4nHBGmJKdwz87MpiRGMOfz5XCwew9j9KY", + "vCF/pEQq/XMieEKEogReinnK1CjBaqb/FRIZCJooylnnsHOG1QxdzYggaA6tIDnjaRSiMUHwHQk73Q65", + "xnESkc5hZytmaivECne6HbVI9E9SCcqmnU/djiA45CxamG4mOI1U53CCI0m6lW5PddMIS6Q/6cE3WXtj", + "ziOCWecTtPhHSgUJO4e/FafxIXuZj38ngdKdP08VP1eYhePFGY9osKhP9lfK0mvoDeFU8RgrGiBpvkEJ", + "fITGWJIQcYZwoOicIMrGPGUhent0hgLOGAl0Y3LI+FgSMSchmggeIzUjaMalgneUwMElUngckf6QdbqV", + "9SBMPwlXU+nvM6JmRHgGSyWyraAJF0jNqESU6acB6RcXTImU1Cnb7dAwIiNFY8JTVSfUz/wKRZxNYVqu", + "XRSnUqEZnhP0kQiO/khxRCcLyqbNRBqTCRcE/bxISIwZSiIcEImoQpQp7mZjaJTz2H7sYy46ZVyQUUik", + "ogzr9kcJF2ZHlEf/Gv7AESq8C0OD95GaYeW4nHGFLglJyhPFV/iyTMbfdna6zwaDwYduhyoSm22Fr2mc", + "xp3Dg/393f1uJ6bM/Hs7Gz1likyJ0MO3v2Ah8KIwHclTEZBRQEOxbCZBRAlT6Ojk+M0NJ9DZHvTh/7ae", + "drqd7Wc7/e2Dp/Dv7YNOcVo1wpdH/mn51jtXWKWyLoPMbhpZRhkVmKQ+61dpPCYC8QkKUiEIU9ECwZYi", + "YQumK0174FuKgLMJnabCbUHfliuRc4YlwswIjV5FXuSNtdp3gRZiIb9iI0FiTJmmcW0Qb9wjpHcosptI", + "DyngTAkeRVooKEXiREm3i7pajDOEkySiAYie0qbaiwey0+2wNIr0w8oI89UmEZ1SeKEVaagsLJL7FimO", + "CFNEZDu8DWlKYrGp45zc3tXI5WKrLmc8CkcpUzSq9/pO/2xpWuyJSjQjUYj4ZNJFdIIw0q3onw2Pl8m+", + "M9g56A32eoODt9tPDgfPDgf7/+h0OxMuYqw6h50QK9LTq9xmbUBoS8oC/+qwKovE+kASJDCrkx1YpQUc", + "k4DHBOmml4587/NH3njcHOWLql9E9sX8ZPUsdb9yZLTj7ghLlQkhWC+qFiPsGdNbGhOpcJxoOaTHUCBm", + "kxRyDVbXwVF+KYG3P4vAjFyrkaWQdz4+/iDXCQn0icidNMkUDN2eZe9MZN0LjwuCpR6wVpP0YfpbJ2Uy", + "TfTRTcJREmGl29U6FbDBKKZS6k+zH0IqjRzpdhyTjxhXI5EyZl5kRF1xcVl807Yyokmn25lhOZpPk7TT", + "XXZslZkauiARTiS0Z1dcjIgQXHSMarwYTbhwi6TP3JyES5qqUUhmR6yHQp1up0SATJy7ubhxZ6vqHRz0", + "ArwkjFVhzACYTH3gxbbqw82Gtlywm1PEKNFumZH9WJYlQEjxlHGpaCBbiXlQHvTyxjz0iM7jrDlEQ8IU", + "nVAirF5NkNDnQ0yQawTpRhBlKJWVfZCp/iMy17baaL43UkFSJ0rFsCkuXkE3yU/EwqmcLX+2U1YwaXnu", + "XsNpjinsyWMyp+ZoKetudmlGoaBzIjziO1MAjCg076ENvde1CGGckc0SpdichhS3EQchjGlEPdxzdnSC", + "zGN0cow2ZuS63MnOk/HTTnOTDMceXvg5jTHr6Q2hh+Xah3eLbf+65zVReByno6ngaVJv+eT16ek7BA8R", + "Aw232OLTHZ+mmgR0hMNQECn983cPi2MbDAaDQ7xzOBj0B75RzgkLuWgkqXnsJ+n2ICRLmmxFUtt+jaSv", + "3p8cnzxHR1wkXIDNtnLjFMlTnFeRbcqr4uP/H1MahXWuH+ufSQOdiJBUKi2vfjSvIUGMKYeuZlwSFOBg", + "RtDYGC1gnkN7bVg+69idXr4RHFOZcJDz6P1prs2BaUiuSZCqUr/fo5BqWziwSlVhci1GFHC9Rvr89WkW", + "QAFk39GmtnKq042VgEAQvKI7/Uarzur7PzXsNYplU+vuFS3mYxpFVJKAs1AW+6BMHew1T6awi82xWevq", + "hf4ZxURKPCVoA9xSYG4YCa+1rQmmEQk322nYTZP5nY8L51ppzwEb9PA42N7Z9Qq0GE/JKKRT61es8qD+", + "XevHuh2F4G3/REDDaDcP6FKQSb2/n+A8gU4EmRBBNMd/ZneJ4HPCsDWp/gL9dv6vrdzhumW9rVtAzLP8", + "9U/dzh8pScko4ZKaEdbEhH2i2QhIjeAL/5jh0bK1LnCUVFgs3x/wxi3sxFzZXEkb6/rR+haervzkrX6n", + "KtBBHGUKTkEKNMrtF1rT8qgsnCn7oOIC5lMUUWbMIC2/zVqAsrdIyA8Rn252bo0OGfnrm1+P+wbCy/zQ", + "0Jp+1s2sgohPi9ScESzUmJSI2XCu2oby0TWS/6y0fSoHKJZktFyCnFHGSAg+d7uxzZtat/baPrCLLqka", + "zfUJ7NtzMKxfqEL2jcamIh5cTmhERjMsZ9ZJGYbUOFzPSjPxqJClywwMTgLXIKg2YFSf//x8Z/8A2Q48", + "NLTeX/1CfSaFr3XzVr1QWIxxFHl5o5nd1j+j6xzi54Dc4dt09mQc6BjTSLqOXU1rvKdyZv4C2a1HBWef", + "FgOavSL9d41xu53rnm61N8cCqK6bLwzrf1xPhd9+zDst/HqW9V/48Y0dSuGnn9yoCr8d5QN0VDFmktcF", + "DpP2a5WcMtWjDJZAK+DW82SOjtzNaj0ZSC9jpuHByf5ZitVRTaVCG29+Otrd3X222V65ovJyJOlHMpqO", + "l+rNdtBWUdafIf2ZPiqndIrHC0VkHx1hxrhCY4KCGWZTEiI8UforO9aSAb4/WHUJ06gkEdGkIqmPe/jZ", + "0+trrJ4d0Cv57GM8FtPfd733n+Bd1PKm2QOH7WrqV5HAzLOINzuoY3ytLX/DJb7Lz1NzXYWyl+wayOoQ", + "+ig36Cf0WtNcoe0Spb3E9Vt/2RUW46yXMvpHCqudRHhRtwEVwXEPOMJHXiM3zAaSq7eP1PvHfONmeoWp", + "0jsnuzO1U+4iHoX63JlQIVWn/W3YWooBEbelIpV2mV9jqnNElYCNh3o2zsa9EtEJCRZBROrC3YnvRKTW", + "3RoSfebAn5kDsa0Yd0MpSWL341nWRenn47y/0u8vTOefukbSEfuwMWxiuSg7Wiq3XsdUaQ0glebezniD", + "kb3uvwWpld8M4ygioifTJIkoXKA4MdY1TtMpYURorjCuOIW0yUPDSlxHvvN62769d+ebe60NUeNbs6LG", + "gdm4oCvmMI241qwWyE0k9/310UmZdl2E4QFciZSpnF1aFfxzaIP0p/0uGnaSgPYGg8Ggh3d6g0FvMOyU", + "PWzRXs/cPCRYKSL0AP/f33Dv4/PePwa9Zx/yP0f93oe//cVHybZOQ6de2HluOCnTRW6wRU9idaCrvIw3", + "FnDF4X9oXOoTbS2su9JHJ3X3gZlryINLIvqUb0V0LLBYbLEpZdeHEVZEqvLMl7/rXRF3a1W/ZcRiSgr3", + "HVgiLrewCGa/bc2xoJipD2aMaNiJKEuvt3AcHuwNO5tdFGMVaKUVHcOIUK/n2rECyF1hlO5U+ugVVwjb", + "o12L3UN4qd7vVOA4xkLz+BxHNAT2NpKsJ2lIEGYhogye6TdSIpEgKhUM7Q0G7smoMChtanPppjLsoCuq", + "Zkh3i2BaiAuERXywZ64aA5KAG5OHuHLfWyBFfa/ojfK3reyvDfdn/8PfNv/PX26VWYHRlnApm2o+XpNP", + "Kw5zkDMbEb8iItCGckT0bGVX28pUyS4sQwg2JuIsWnyPAqM1Gx8QF4iw0FIa3iuzc7zo4YT2qBmq0Rt+", + "JWyqZp3Dg10vcTfsH70Pf3U/NRBWpBHxSKE3PAU9DB4XI9jcGDL9a9mKOOqmEXjjYspOzGfbdTXt81bY", + "TWTZShsPfONS61Miu15dMZB6KKM23WKfGjAnQuitaOTa0ekx2ojoJbECTZuKaJgOBrsBvAB/EvtLwOMY", + "s9D8tlnWWawzxqOu/NYhwYyDPymK+DqRY6CegnaCo6XulmWk8VL7KGu37pz5mUvVizHD2njMB4DGgl8S", + "PVATT0KJRJdkoU2FBZrqRntzKiGYibA5mmNzY9UfsrdwvwOvuEcS4kLonKCYB5cmynHGQVM2YrGLrmY0", + "MqqgIDjKpWWMKRsyLaB7MuCJ1tKYfQ2mhi4Im1+gGCewzbEgsMe17CeC4oh+NNGqEKFCQqpl9JAR2Bgo", + "wXrPBwEXIQSTcURwMCtQ4TuJLoxf6QKav6BMs/WF2ZiVuMw/O6/fvf3x9btXx6PXZy9ePT8Z/fLif/XP", + "5qPO4W9/dkxUcmbB/kiwIAL95U+Y7yfjhQTnSOd5qmZc0I/mpg9isaQCxR8ntM8TwjDtBzzudDt/Lf7z", + "w6cPzm9mQiDmeht4BvbJqygaZccjko7dTbJE9nbSxcVokmkR9fLs3ZZWnxIspZoJnk5n5Y1hdbe1tgQY", + "GZSPxon0XvJdopOt10hrliiieoNmmuT2YHD645YcdvQ/9t0/Nvvo2OxaGL6WQVxYBVfONPtkAc5HZ+8Q", + "jiIe2KuuSVMso+vKJ+AJU2KRaMt7pXDKX63LqF4vf7qGKNoaU7Yl9TL0gvXoDnxzY4/vCzangrOYMIVA", + "XRpHRJb3yqvXxy9GL1697xzqgyBMA3ujffb6zdvOYWd3MBh0fAyqOWiFDHx59s5EzMG2IThSs1EwI8Hl", + "qg9/hneP4FXYcSqJ0inYuPXVe56RBsUk5sJckthv0MasrKSYLY9gXYed3Zc/Gr7cfgks6dbThiZlrZiG", + "K9FkL3/0MdpskRAxp9J3k/pz9swxTT0ovrQtrDnu+D13JVonRhDxNOwVuux2JlSQAIKQ9b/+ILE20uYf", + "y9FQnu/8F5ytjJMVVgeOEsrIErPjC9Eur7i4jDgOwbFwm8qljePzBJCbB+X1zawhxxK1vIwxZuEVDdVs", + "FPIrpofsEcn2CcpezuTytZ4Jjv79z3+9P81t6O2X48QK6e2d/c8U0hWxrJv23pJlE0kT/zTeJf5JvD/9", + "9z//5WbysJMwOsyN9EG7/i9MC9XQcptxY/zoDQGJ2cGfR9Zw6yyBz5HjvZXRhz4Zz+dERHhRELx2TJ3t", + "AUi/yqgEhVwiZL8reB2Xi2HdmtMPXlYdODsDv6D95qz4wp0VgoDoHSVZltuyDfLGvJ3bkR6m8/Dcj/os", + "sOd+G07LGG1759T+uVNnOT/HyUuajMCgGuFpFrWxLDHu/JIm1kqDL8w2jSIj6MMU7Lox56o/ZCbwXe9N", + "4BxyTQLgBamwQs/PTiS6olEE3l04NOqqg7b5ChkT8LpU+n9FyrponCptyHFFkDWpbQSeHgu8PCYoZdiF", + "2VbMKjvBetQykOWSCEaikTGbZEvKmI+Q/aiRODDVCZY2UUeoNCnT6/iX03O0cbxgOKYB+sW0esrDNCLo", + "3AQtb5ap1x2yRED0s+5EsyO1/fIJ4qnq8UlPCULcEGNoLPOP2xjQ+cuzdzaKWG72h+wN0YQlLLTpjk6j", + "sKlwIWffaYlMwnKzxf4rRG+KFJcMJ3LG226uc/t6vrva+5m6nXmQpOUl3ek2psLNqVApjvRZWrI0vHHD", + "JpXXY1GaTOGiZWvPtTx1UJVjK9s640zLkNfrTRr0+NSMJtzap1bw8tS8a84F8We7wa5o/4S5gSz1KeZe", + "iM/o69w0UssJMD933cxuQKWTjCYVT+TtkOe5LHhtWqXgmpPdHH8SbVzghPYtH/cDHl900cVfSz/ove9M", + "R60/XiFDDZAnTP9UbL/qr1rpSVor6bW4OFjefD2ey8YECjTfRkpgJk3qywwnpI9+BiGOFIm1agIZZxJl", + "GSOI8avvETdKq/t0yPTQJNrQstKSI/MnSjpllE03tRmnDyYchsbpOElVKvR7cypzapZZxzn2apqhGZ2N", + "P4c8ccqCKA0JunDOv4uyUlR3DdZNfusrrFmwhiRguYIxr7biVOnu9YQzRZOnyuSj2KmXc4UqDshVIZF2", + "LFl0zg3W/zwTF1U4gLnHhNWTsxe04DEuuK6bPMRWUfF7ry/JApbceapxzVdddFL7XcmCSB7NiT12i27u", + "MQAecKM45R5u46u27mm9/aup/j7H7aql0PRqTf6yKegBOpCq5yabc4y17lyqqZNCenKmv65W/iUB4oNp", + "eYhAHbvoGluYgOWBmGaWCIVUkEDVmqdsOmQQBX5hf+nb1i70Jtc6yq3AR0A2NijtxaVFhZV1ah80o6fG", + "Y6oUCbtl3eCSkESunpRWr+2dhufiRZArQZ0gc7mKLdUzwiZcBCS2RsLnOQZeFBpbElTStol6ULahb2HM", + "Lksd21gdWHuzHuCBLyWvVzFcworVZoKAy11e4Ci6QBv2pU0kyO+Q4GvXinGWM/vbozPHAlnIyvvTruZI", + "LQUuZkolI/0/cqR38UW1Mfut2+E5vsbTAdhXe3u7dlWtU9UMuNJs2X/qDWxuXpqzVM4aL1wrl43LeEW3", + "c1R4vdlL64BMjKfWnUBJKmdoQ+Gp3jo2gl2fE5uf68GlTJIgFWT1Xn8O6pGNooYxRZiy3s9v354hQaZU", + "t1mMQS9mx4JbpyHdxhq2ecCMm67iZTfezu7e/sGTp8/64aXok0D0U9kjWKredh/H+CNn+EpqtWsrXuAk", + "OZxvr5TzZgmy8TWLfGeFNbMCj7VIdgkDbSy6o/yT/K7lkrKwbQO/6HdXB845g/Ou/fCJIL00mQoMCZy3", + "6YW/cWQFULN5VVeAWi0Pikyl4nExm3tjeSxkmVhzHvVCrPA6oZBmuPWox3hhmjImudf91RRiel6NKy2F", + "YHuhbT43zMWNxbcsTenhxpFAwpHinqxnpzucHAOahH23VR4sZLMrPppPKF8OHmBDGEtgNEYrse4N3UQv", + "Caj1KoGqG8xMqqAhAtgO70+Lt/v9IeuBFnaIjrMOsmazJjGYGDg0F6QbXBQGYWAC0HixiTB6f9pHb7PR", + "fieRtlvnxOXLz7BEY0IYSuGGCZSinlHJigNIJehOqvq5daGZ1PpNCGLg9lk/A+ACZ10GJwbRrmNamY+B", + "vIGFsmeJPvByZ2gr5+WyRJM35jxalmxSAhfZ7w22e9v7b7cHhwP9//9on41y++gBvrael2WLjR8uSp+j", + "dyfHO9Znvnnj/JJbxxfwi7LjPPAZbaSSiJ4Tk3Dl4Ql3LkQVN4Qz3zhK+Y6CjvNMymXvGkq81W/eBXCC", + "L7XD5l6uD21QFZgr82cLk6s7dBYJuB/yXVJQ5G2AekA9maRdCFr6URB8CYAw9XNba3VyZM4zf8RTKk0Y", + "Hrm2Ti7BuZpIo3SXnd/be0/2nu4e7D0dDDyp+XWG5wEdQQpCqwG8PjpBEV5kWWobcK8donHEx2VG3989", + "ePpk8Gx7p+04zE1uOzpk9rf7Cm1YivzNgXa6J6VB7ew8Odjd3R0cHOzstRqVvTZoNSh3xVBSSZ7sPtnb", + "frqz14oKPrvuhYNKqCrwPtye5wbqTv+rJxMS0AkNEKQYIf0B2ojhCCPZpWV5T45x6NCN/GeHwjSSSwOj", + "TGf2TeNvjdNI0SQi5hksSKsrCZj5MbTkhYtkLEOTWq8lCzCxMhDIzSV7BdSZkIzT6dRmb2WkOzVwWQXl", + "iZIoPESZybZUzsFq5gP70MQHdg4tueFXbTr1IjInUZEJzNFlcJsEQRmfmEUrzcrFAFCWpF6WaCTlT6kA", + "XdQ0ivCYp8rcNlv4r7yTE+M5gNvXlIXt3B0/cXG5MsJcn8QZyllLh8HEeuzgFMdZZrHNNS8ofdmtsLk7", + "t88lemO+MI7C/OckLUO8dqEn61BkSBCpOEhS6ze2zbTVLv16C/jMXRiJ6S+XnfcU4tabmKig27WwxZQA", + "up9aqbFoTjFxOufweuuEFf3hSkdKC7ozcnUfRIeUq55m255kOLkbii+LOc18DflLcAoLGpI+gt0FwW8O", + "6KWy084VTxISZv6f/pDZjI/sJ2ku0vSHhg5qRqhAXNApLXdc9rPeZfDqOqzouOnG7Fj8sCkYTWb3fr5N", + "b8ACJiYW1GBYFXOV7SJ0up3zDPfQSqIyad5k2JE1iuTB2LUhvjx7t24IaiL4hPrAdyEkxj61lpkLzvx1", + "b3De2/4fE2it+Q1UNMpMGE3MwwpMoX2/3cnz8uzdWdOYMuBAVBxdbU5Z4NMypGdHEXu3aC+nrQXj2F8f", + "LFknue79zKfLTgSOyTidTIgYxR7n2k/6OTIvZBncpz+W9VmtN7e1ms9KiwNm8wQHlE03W1Pf45CrTKNb", + "oOYH/3K9sUB0TTn8eqkysDqTvt9HrzKoRvTy7J1EebCax1NXSZNvyqg5my0kDXBkWjQ4S5QVHWzAnK01", + "5LP8Q+uK9OjJfoRPtxHQxnyapLANz9/0Tl6/34pDMu+WxgQBZjMeET3uzYK0mLvE7Dz9pyQk5k2eDsMY", + "su0GKtAq28GtiVTYrx7qKK5wNJIR98XsvNUPETxEG+9/MnmZegRdlJSWUv9eoEKJvw+8OwYwURq6PYcO", + "qy7T0gb32o7l0hDGvVKYXqlT31YxyTB1HacOr8svywvNL1dDuppGmvs9cvk6Fae2VSSRSetBkNbjwhCQ", + "+dR4ra1rRJIEC6xItDCaRXb0lSE6SP1GmVyTYI2EoRf69U8GlikVZKRmgsgZj8pxCLvdOrS3hFjYObFo", + "hmZOBce74ijG4hIORqdIo5QZCpRDqXdXQWXMlErWmNTPb9+eGetaETHHUTUYX9Zu2I9JhBdoTNQVIcxN", + "BUuE0UuewUZWs+dkA6KcUKOECMrLNOzsevo9N/G5aCpwQJD5yhUGsEsi4dRsS0rbiweuOQiIlA3ru71s", + "fe2nkzRqt8a+YW2vrKMRrLPAb4/OHDJahjzvyLxTp/IZET2z5RwE/fKl3ZHLMfpcV4wzUu9M8DEB0D6b", + "AVFM5XPxRQBKqD8vpc8VhIMs5q3ZfmAXGFJ1zT7/0ErZq25330V6jFnoQ/A3ge8mR3maxnpB9JBFCndH", + "NDTBRyZt2qjlxfh9QXBIGZGykuAZpCLqdDu9iZ3V4dZWxAMczbhUh3u720+3lodxLo3fteFKo5Aus+9c", + "UJMJe3F5kAZJHyZdZoktnCQtPGCGjivOBxBP9XhBQO3XZ1tBw3MB5INBLVf1GgfK4YyCS6x05YmL2zbR", + "LFmaDzSYVbrZf/asuD8H3jvovNCUY/+tGu9DXArEFGoeMe6GCh0Nk3/0alRc+MAkuVA2E21MXNBhdh66", + "kD57qVLq7OngaXGWrer5gLCpbHO77zxTNW+Xs1NJgdx2/9oGIDyqrHO4Lb0cD1vTZQVPaYlYY6nlBOUJ", + "YWvRc39vd2c9eradyIkLy6rIJR82xtHpsdGJAs4UpowIFBOFbfGygpABX5KWMoBWhkkM6SqT75eLlob4", + "hSLYxU2hFm/r9rsBdfWNCeMNUYwZnWiBbN8s9ixneGf/4NBgQ4dksrd/0O/314UAeJHn/Ldaii0TJFdA", + "A+jL2eetwx1k+reZy5+ds+dvf9aCLJXCHFpbckzZYeHf2T/zB/CH+eeYMi9CQCs4cTqpwYiX48G0wW9+", + "PyxUz3J6T6uSM35nMEQHAyyJF2+rFCV5d8BaGY+bAdwkZ3VJDqdWV15DocMG2twYGTyvmaEKiODFbK8W", + "6OD04/L7befugndsn1AFKwdOr99s3wj6Xi5FB66BRyaEZXjAUWT+Cjib6+3qAwcuHZHuWQtsSThGLIhk", + "1mXxx6z3wo9HxYEUfnfYlIWfLErwhzVDUpYqpH+v66GrpZBTR1dsZr/vMTsV2iKyW0Aubyz2A5+FNwnb", + "Kvf+evrff/w/8uzJ79t//Pr+/f/OX/738Sv6v++js9efhb2xHIvtQQHVbg1DzVQZKQKptWWlU6wCj49O", + "m38NFLZPjMdBBTPAkEZjcjhkPfQrVUTg6BANO5Xkw2EHbRCwlOArre7qpmwO9ab++MzcKOqP/3Rq8Kdq", + "G6FNlhZ2QTIMDJmOQx5jyjaHbMhsW8hNRIJdoP8KUYATBe4cypC2fxdoLKBsqr3xyTvvoj9xknzaHDJb", + "90UJPYMEC5Whe7oegCnsqEwYqn2dZHAI5tJlyLLTOgMjM9d+/Uz5h3CPai6XnyjL7TdrOT0d+GDbIJ9E", + "L2QEmOEou0CjEhg9S3RBTwebdXtuhY2R8dAS9oOdUK+n7JiyxV4yDAxdG8E9ci7KFeEZWjaZPYLAglQc", + "/nuOXEM5LbIlNvcGJrtImmtnFclCXtHmigSQFhMy967wWdQCj+CFSTx7++s5UkTELhV4I9DknNBAzw8i", + "SqmUqWZFitHzo9MXm/0WBaGBtksTRMqD9qSN23vYpuvl3NzFMemik2NI/LM7NFdrIVL7Jy5QZARMvq8P", + "0TsLAV1qCplAUbOS0SK/zDUnwLCz6VpMqpLiEL3JtGmcDaVUhLp8P5zvS2jWxvKYMPJa691aeVfhNGkr", + "2iBoHKss/VSfuM2ioL37xqHL6D1fcTasvbeL9+uNroTC2t82VOftqzu766k7rmRfMsPSx92z4gURvLSk", + "rC+teL1FczREqd+VpWqz7qBYMMlqXJjPfUVG93vb22+399b3hKyLslhGuykgYWVAi+0REu8CabDuFbim", + "atQYJ4v0YxsV60zM96dohiX7TsHDiqG5vfukVWkz3WvbCNNibCmfmCFlUspB52SRkQZE6JJGkQk4lnTK", + "cISeoY3zk5e/nPz66ybqodevT6tLsewL3/q0AFx0ouLl2Tu4ZMRy5IK0mvOScJ7bR66pVLKOLNQq1vFz", + "AB7Np+1KVLhJmjbyShXLUSJ/LiE5eqGiNm8R3tEFqNbIeB/AjQ+Z+fPlgUYuhXn8XKxGa7zcEVRjo3D3", + "wRxW0pz3m+T8zUEX72Q4K6vpF896l5Z5Y5TDbod6UtKeSy2CSYhOzvJKGLmH0zVfmdOznf72wdP+9mDQ", + "3x60K30ULOn79PlR+84HO0a1OMTjwyA8JJPPcIRbxjbKOI6u8EKioTOXhh1jnxUMs8K2tSZVq0v7Opjk", + "zbAjqwpNAzokKHYuGkLaGqpNyk2LnKMqElKcRiattVh3tawtzrBEMjGwfwbRPNNlhwwG2LXwQ3BU0Jgg", + "HAQizf0ZrgCW0XzTxPL9kAkiE860NqytGvQLWUgUU7hZybqHeCqJstD2cMg2hEuDyPIdEpxKEuofIMa4", + "62JZ9dCoAuR2/cGQyVkKtdM3++iIM5nGRFhXDxpTcENvIpka4w7GC9RYaIEpaUjEkOnXPMCCf2aK+uHB", + "YDAYZDXdO4e7+t8DHzfd6Y1K3wJnmkQ+gPBiFkITsMFEylDKQiLQGy6JUhgRww7VuJs1b2M+EzHTfd5O", + "j7Kf5wqUf2OuAthsh5z5uXCFywrJnpdLyLY2U/f/8VnVZklbRdVmFNivRutcixIU8DQKte0z1qedcU2R", + "0HrQJFF5dV44IN+xS6b3aGnqNp5RcfRHSsQCvT89Ld2lCjKxxUdbTBykRMM68GStZdhZ4S1YOZobglje", + "B3BlVVMpaIi3DlNZvLZwmY2GQ1tcX+QWozcanTKzNJpPlsyp4ngOyXyUpj5DRD9yeAbv3p0cl5gD44Pt", + "p4Onz3pPx9sHvb1wsN3D27sHvZ19PJjsBk92G8p/t89GuXmCidfs9NVzdmGmIxfu6otCbAo2rpz7NoDy", + "irKQX5WOFm9EW7F3Gy23qvt6LGzrIXgj6KG0KLTUICVO4QAlgW7bRJBWqv02uNMO3g62V7jTVsoLGFyD", + "/H0rUhYYvDmQxJmjOi4MuLhY5XHeTJzCgFyk+ipqFTtvT7TB4f6zw/3PJZqLtl41xio73ePiNoVwONDS", + "Sji3SykqOGUcyF/H6hvGd2yjvzvdThagDn/DQVsJfswet8q6aNqwXb8YWSa/G5IPT0q2AFwtG9Cq8FBr", + "AVne2jhVKMtp1erFUcTTEBUcWgbDB257Tgp2gW4GLl+sv8tg8pnoaW0/AAgsQJBTpgUx3HLpRmym4iF6", + "Ce/CIxwbk8kOwgDhFy94cLgwF9x6f7mujQGzfMjn1naBb7Qhg/S/YNqaDNbvubwJo/kcolccvsksKcar", + "DlTzOpgw9derztYNm0LoUs2hM6vGHaKfMtUtU/6ssrchif1zZAVWjvCwWcqztSve0dySr1whhbTbMRTt", + "dDuOUJBqWk86fZdzfW3/FVnRF/lBcGQqHmdJfamikcW1hZlQqWggbbS3Xtwm/cLW6CDhyBgmTUFkJlPM", + "Gi/ZR059eX+KNgC67G/Ieof1vzazgLPSWbfzbO/ZwZOdZwetAEryAa5WO48gj7E+uJU6aJCkI+t3aJr6", + "0dk741cIjMUO1w127oV88ERwLXr0zF2Dxc6f9Z8VcVlCno6jwvWVBXEyMJCwYF7ooUwWNQQu/UGjOZ1M", + "2B8fg8ud3wWNt68P5M54uwFv0nTkd2mdFK+wa/5fMu6Zeht+6AxgKCEb0WXeEAkzQOdEIeCfHsIBmA5Z", + "+qFlOYdBYynuZay93d3dp0/2d1rxlR1dYeOMwMHlOZTtCApbDN5EG2/Oz9FWgeFMmy4nGyCBmTUr/fsM", + "2eqMg7JC2t8e7Pq4pOHgzrnGtj2PG0n+3ppmdlKW6JBFmZlttV3upfbu7uDJ3v7T/Xbb2LpeR+J6uYRx", + "OQaGPBa5urjyG6BNvn1+hiCDb4KDst/EoZKuNSq11qgAdd2gJa8xsKdPDvb3dne228Ek+UIbLABYacOW", + "ZZdn03mYwrMaHlLURW+36bTwqVOGwd6QIMI0fh64mOjK6WNQkUfCvJYvQpuDwXr7awdXi29bOY4yd5CJ", + "qDeqARcoZRkWf3/1vebNrimbxbQ5D1aLcV/gPNPksngepvjODWiXCDKnPJW30BBXJqttEnEu1vq2yUJ5", + "Q2QaKXOXSCV6f/odCBHNXEgqkpSNJst+S1BPbji5tTZwiSf8XN1ErFar0Wbpl02427BNu8tS3kvbvxFc", + "KNSiKmWrYwqPcBSkUG4CZ+upZwUwIZC0myTRwkTfRhHnDAUzzODGQdjiNmyKMJrxKOx7IyL1k9HEG4vA", + "r1DEDSzqJSGJrcRgBqE/0zoLnRO0UUgaRoaVKhXz9mMjVSzWfpkb92N/6S8sfekkWbKqpidWvIDYaT4p", + "+RgjPpVgBSqIK+5XgaITLEy4MGamssg8NsZjGWVpR5/2niFWpLfvCDVHJ59Yi9bqGJAKaiiJA8GlRCSi", + "U6hi8f60kmG4JCslyzNcHSdYHmwL1jW3g56zC8402boAke9A9ETcf86RCDwMWT1LIvCcNzLGLIXaDAVG", + "JtcJFYY92kXZzbhUoww5Zs3BSjUCwPVUkBxeKsuLzRxA7h3vuehE203IZcNZb/R1jav8TTUNsFmmeinq", + "p1Y340EfG9exc5bC9eT4P1Wwl3XQnXKEbiqhVVoAFkIbjKuSWCqgTG+2iTjx26i6n5p5auvK/bo3OG8L", + "vLQcZ+kMq9kJm3BPdv4a15DO9WyDIBMiYgolB1BIGCWhMx6z+0jr24KUzkgSFKbEUs4opAJbgmOzvSHD", + "njmnGGXTiqyvdtjGH2zGsByPHfq1L7aJHZL+lLe3IgVamWg/iXCe/NYqdJLKkf/+qt6wINM0wgJVwcWW", + "DFku4oiyyzaty0U85hENkP6gesk84VHEr0b6kfwB5rLZanb6g1FT8Y5zMzibuWMWpNJvPoUf9Cw3K3mD", + "4HrZMt9vAaZDm1AsbwDyTzQiFn/rHaPXBUYvAxbv7Qya8lkbGi1lstax29aV3JZlvTs+lZ5EtKVajis7", + "QkKLIm3UniSVsyzUoGJXmmebnVYOCwdp6C4Eb3a/U05W+DxkgSMjzCu4AmhMINcE5uaNVWwjM730MgJ0", + "s808vbDtqZyh3/m47DttG33rKQa0wfL0d0Em3oBz4IWlvmvzRktGqfPFOsnnIK01FWz+ua+vPA99nYTv", + "VQWX8gCpJhn2plZ7aEYssTMKmDpELXD1XQBHltRse22f3VytEtVU12JRqHYm0ZgLAVCwWfQ/kFvTuYss", + "2oVUi4gcmgjEAEcREeCpsK1FfEqZjajLbs+DiBKmvpPo/9vqm1a2TAR1/3fJ2eaQCRoSiTDYqg49NsvZ", + "0noWwSHwGp1DsUTDCsaB30evyJyIIUuIkFQqU/gh4tMpCb9H2MwAHJsiTazjEyMbJgd7Uha6GDJBlAB7", + "3BnoakbiPvp7wfrtFrr/TiJ+xTICDFmRnto2TaWrGFeBscFSXnERLlka9wpUsoCLYaT4JWFFiZs14zVv", + "TUMj81U9rM+UtYSnCDz+WPZoFv1RKjqJqkUll3edSiL8Om82u+yVVqE3hV1YsLXMbgcIBigmZv+Cn3LY", + "hdroHOzo86zOvyeKx6TaVGKJyhd1KyNhIJdrGbpF/aoCbbjoZ1cwoax3FAoXtDp426WTVeP83Wi2JAnK", + "ve893X9y0LJyxGfdBRo0n9u++ZvHS278GlbqtM210tP9p8+e7e7tP9tZ6wLHpYQ0rE9TWkhxfdAGuVZ6", + "L0X//ue/3p9WLpX2IXx6sNagTFKIf0gNiSHlAb0//fc//+VGdeMB+eRAHTG4ISygMQgoKq6ki0Mo3xC2", + "u4Nb4k14XnJJ4EzMoA0ymRBwuo4M3Xr5YCqwAO3UapzggKqFR87iKxOonr1SQb5tc9tUHqxPhzZtW5RE", + "LblkOs4zJzdc5+iv5uq5wgtPWxegkem46Zr7dbVXc8md35EUQyhaRDDkpdLr7vRsPldYlmKx9d8B6AYu", + "OayeKGPeWI7CWc1igCAZW2epEGnoQ2+uaJL2o+LyV5azcC1aciJVKf5hyT5s3oJr+Zg9J7LHxRysTn+t", + "yAd7AN7sq9G4WBpqae2tUh2p/NRdv98WGb513PTsBFu/v0Ku4zofVjFCgR/tGCzJ87a7JZZo4KZCGovH", + "Xccj0sviAJ3yLlNz/6j3vIWd9iRfBpd8MiljX+43YyUDSDHkablesFIkTlQXkWvnIakC7aKIXhI07OzL", + "YUcr68POdjzsVC7JvJmPMb4e2Q7KACWDZeDFWVHE6iClm8E44sGlqXoExXT7aIBigplEKYPNX7nD2x4s", + "v4vqdpLC2mRQwcREUNXEFoxpTGZ4TgGh3t7gTEtxnuSaKgnxqNDOIQo5+H/LJR/tDPVrJi/xMJ80HDqY", + "LWzDukH9HmcuYDZ/F3wFEyg0yT4Swbs2415L7NevT7smPgIiG83ASuGTbqJmBFpAZl1U4Nbz3/3hyeOI", + "jGDcVfjuuE7HYv442MaCSKKkxfPN2aHCBCjgKVNVXO+4nZ1VzgirH0kpg1hC63ABnCbbu60XHpIAdqSs", + "78Uyo9+AuStpCZbSvryEXR8Lw6aAmyv/zfQbe/1cHYAx5AvVWk07xbBxcyc5korb8j7Zrh6R64CQsAoA", + "6H+lbSi+/dIbiv8rtkA3WSFV+zaEU9dn17+73CwYaxO1iykDjLMeQGy4JbVwGAYqzAKulBmtBEVcgJ0Y", + "+eAWfS+0SZYm18tp/YpcK8BLDtNIk7eJda2osofRKorfOCmxaUNzsbru+B2UkTLh7DcqJGUj4e+llpT9", + "+U7qR9WW45wo9+655ZvmsuGlSg+lGzOXQOBeKYfwGN7pInuio+14s8JzezO/G8SicrbMp2Q4JqNEkAm9", + "XsIt5gVjCZchSPKdUyotL9FGjK/R3hMUzLCQlbEzOp2paFGO79nzIAB9VlU1QRRhao0i/Plqug/rwXR2", + "OYut+7Th8wJeT62mgdVBR8uAc4/yyzwbrpTgBbhtGu8gn+zuDQa7O4MbIee6Ya1BrqP8E1uSrNxOU4pe", + "4TsbR1CKei22kCVE1wtbXgkKedUZmaQSBMeHkMiT4ICgiEwA2i0rKLz6brLa9fLBWw3Kgsdk/O8Wyq6b", + "u+Iv18zIurKgw24aHXc9WcYLKj5fcaXaIGaCGhCcJ4dvtzc4eLu9e7h/cLi9fRdotxmRmrJHnnzcvnoS", + "7eDJXvR08eSP7dmT6U686zW8LqkpDdKGV3/R7zYG8eSnYhl3qCTS0IadQ0JEtWJptdKvJBFlpCezjKvV", + "aY9LZIG53l+5/9dz7JsZLFUWzsuTLOoMWOXEKXHWPYFj2dEvvZ2oDv/kePmwb5TCVB2In8GqQwF+ajcY", + "wKTf/tzq8ylree68K7zY+uRZmla36uzx3Z7D1vaucgPFffxcEoylHbbsxK6fah7v6JQLqmbx8uMhey0D", + "DoY47I9ShWUwpj46mTKoT1z8OQu7K5pJ+uNOtxN93CvvGft7e1guC5SbMaBd6qIa0CIsDcpfL6cCvJKb", + "FsJExmtrXI/5h+3e9jMIDo8+7v0w6D2rXtMDtYrk23Zvl34dtKFhseiXKxaz/WytCG5Hz2Uc9Av1lazK", + "D2ILoWt5PC8O684Kl7FbWuD8cW2NKzA7jRrn56p29jQbFbWkkETY4+0tuWJlxT4sMhkakyllso1ndneQ", + "uWb342Gnj55b4GmwVvPa36XmoepzgU9oHJOQaqXSGPfNGRE7Lb1tVeNhvWoE7iuPetb362fPVmMsrErg", + "WnVM9j8jofezzN12Ju4y+A/wnDmbFAC+4MUuohOEWaUkoS3AbzPxIbMSAtUOHYpazrJWBshc8XOekC6a", + "coXyHPyWHrWUNXv+svGTa/CoLgHdMAyxcyuIKhm6F10mvk6OUSJ4mAZ5AmoEg84hQ0RawU9botWvDvG9", + "S4cGZHZPuECrHRpNHox2Hsim9a54HzXDNi/19mD1Ut+JF6TbSZNwtQwzL7WTYGsBjK9IafT4ZMpkr2iC", + "hcl8aCHR3xQpWDdyjbc40CpRmrgrFM1TdU7yXKjAJYIv2veYREQfU/VGEI/CPOuCylyKrhap2wdPZ02X", + "mHDnVB/IL4Qk2lYBACXoL8Zs4R1Ytd482hi4IqLSXGn1TIESS63y4J6s1MQal6p96f6KV9sArlwWHNwZ", + "0vbt1u23X7rAtkaH8V344R5SSXttLxcqmKoO8TeLUnb9m4xSwGRmUeW83vNdwPvY4q21jJtwYqtZpUU3", + "8/PeP4xbGY36h1s//O3/7n34q9e9XLGbJRG9kEwglOiSLHpQ5AZpG71fRkkFgH6tTE8tqxAcg9MouCTG", + "SRXj6+J49weZ0Fi8wnFtChCDFVOW/XvlhP72l+YIpgIZ34GcXMmyn12/4i5KJirujqONmIiptgupSxKD", + "3Oghg1y3S7KQqFCByKo0jlG/k9knhTB7dGHUwD5h8ws0plDSTQ6ZtmpxEJBEWxO2Egs1dYk5SB9BcFRs", + "x1ZCcnHq9srRRAwQ9P60BrH7+t3bH1+/e3U8en324tXzk9EvL/4XgjiueqaHsKd5b2//wFYjLlJy27PE", + "n4H2/1kotz52M1iZHv6CpE0o8exRmKkEyAUXUFB4GW2QOFELV97Q5X5urofd+Txr0BvOdsulVwbPbqPS", + "3LulpeXmPOppjboBOd/rwDS08IZjQ1MmzL3T5Niejj1qo/UmTukUe3zZ3mLst1ERzg1oZQJObf0b6zn5", + "g+OPq2UFjDQwpKrA4FfsUql6zbHzsVakRnnZ63JERsps+iUtBGyVcy1jprZs5UYf5EPIAdx6WcJtvssc", + "ol8PPlqdR7pUlS/MrDCS5rU5dRprRadeQqAzTZqrGRGksBDwQQ7XvibJbLJHCyARU18tISIPhHSZIloR", + "gitNiTYyZ4MjQZYwW/fALofjP8XXWQ/gvceydscF88gL42y//BGg09+4qu104pqAYVTsCT9QeJmLltHE", + "cVV9MYpcVZ+3ed+78aysWiL9mvZWhTnzPkqs6ePHv2OqfuICLJBm2I47xxsH6yYkAnDLqmjiraC4aUzC", + "EU/V8v1vyzZbzI7QGRF57UZnbWFg4qCUiNskCxywRD6GOqU1OUiQCqoW51Dj3kQIQxbc89RseFcq3/6c", + "dwylED99Aj/lxJOF8JIwImiAnp+dwH6MMQMlHb0/LZQvM5XsahijoF6+PjqxFq6DqQWLhSpgPRfM9/zs", + "pNPtzIkwVl5n0N/tD2AzJ4ThhHYOO7v97f6gA4r8DKa4BWWbbSKvTV/NbKWT0GpCP7qX9JcCx0TBF795", + "ktkhmM2+DlovnhbslgRTYQ2XJIIke8MwVH8N8PPuQD00p3LXkL21mw5yUCGlgiSv7eJ+AKUS9g5Mc2cw", + "sGDcyh6/kBBiotC3frchiXm/rbQ6SyIPGnvNsnC6ZUb6T93O3mB7rTEtGwrsXV/H7xi2iZsEDML9NQlx", + "o05PmMn1slmxNuSmuOOAkYp77bcPes1kGsdYLBzBitRKuGxSjIlE2L1rMIKVRIEWFVAzpo9eM2JrlWOF", + "sAmHFSmTEH9hP9QcWt4Fpm23yBnOzo88XNwaCUt9OLP4U1mc6e3yqcbPt8c7GRvXF9I+cqjQhmvvgYF+", + "xKHL+H6wnbI3eHb3nR5xNolooFAvY2Ab5EolRJlEgHHt4HO4QH+kXGGUxYg/oi1tddZxxm7d/Cja+pOG", + "n8z2jojP83pGRIyZibg376zY9LXtbLzg+XZeeqo5xodyFHBSORwZc1CBIlfeosVjq6oM1o+jPU8Cvu3T", + "TC98QMbfu4cdbiebFc18yC1nUA5SSR7TdrK3OuNcCfHqci+J+lJ4fnCfR5YFvv8Kd9FjYeCXJNPw8tWq", + "HQpbiUiZMYC9GuCbPAvOfvddWfl7mz8pBGaAK103DSUYlLnKw+GijxxNjdGvFoDYIwjMM6wfK2d6eF/K", + "Dtu5jx0GM84uJ74dU9+OqWW73HCLmwJszMIub+GDWMsD8fX5H9b2PnzzPbT3PbTyPDByZb0Lv/NxH9kg", + "SKhYL2c8jUI0JsiA6LhwB4VFf/oRYRHM6JwMmb0tiNNI0QSQyLiIUYgVNte2jY6JpW6JrLkt3VzPhb7l", + "BK6CI0gyArC5URNKYh70RhkjIdKfWFy7HJ2uVl7a7H2vgz1rMD8a0dWMS7jaAAA3pgqnOeTMSmMdQ7P9", + "IXtrsUo1ASF+18kaSSJAXF3i/+EM4SGzH3zvRIiLPZI4ziUXFgDyRg1EolmWevqUHulIBtwH4PKWMMxU", + "TyYkoBMa2GldkoUNIfQ22KpWkB6wG+f70yxHAO34URYN6p4fX/Y4e4YsJ5XvbxjE3QZRGuaXXA6XBosx", + "jiJvMYlpxMc4Ghn6XBLPneBLeMMSpViG3t0mMR4SU1I8WagZZ+bvdJwylZq/x4JfSSKGnc3+kEHsv6W1", + "A+kzPHAFxcfihOt9Jnhs+twyQ9z685IsPvWH7HkYU+Y4Aj7BkeSIXMN3UJMJgBiM9GrgB7Ob/PfgR6lU", + "PC5CcDq+M8PkqUpSZZMYJFFdH4jkkCmO/nTQfp+2/sx7/FTESiy8YqYEunXTqOUI69mP4FXPdTsBAgw7", + "+iAddvTfU4GZMuiKGTYhmhaXdCMD+NebdLNK4QAzlPDEFEcAppphzXKlNgAAAEcRUrCV3LdacYeVbJiP", + "xXOLx41gbgZ9q7KNKEOnPxY202DvqX8/SRII4oso+e/z168QnMp6DcxreYSQySJgWmFAYQpXp06mvcDB", + "DJmLKiiAN+zQcNjJrnPDTRhrKm1KfK8Hd4o/6KH9YLrp0vCHfl83Za4rD9Fvf5pWDvVeSmIDAznsfOqi", + "woMpVbN0nD374CdoEybWeUkQoA1zzG2CJMEU4EsKJ745IjELEbenQLRAGOUSqBi4MqYMi8Wy3DUP6S0F", + "+cQEzxWI8ecQguWGncOhC5cbdrrDDmFz+M3G1A07n/wUsLeWzdXW4DzLLjczJjoYDDZXgzlb+nruLFtc", + "DNyyDdhoFWWlIvUKWujNr+t+4D/a/syufjDTned4N8bwd873R3gBUdDYi5ao5wqionZjFpDIqd2rHT33", + "f3mgFysgUXTfDPpQ7Jldj2V48o+KHWGx8m201H3/wBw3uK9DpeS2fxj+fXT+c4/33PrOydyFOvtLbQDO", + "iTWlkXkZYYnOYUy9c218v4Bf+/a/zvYDoL6LiE8vDo3pjiI+RRFlNgS9EKis1QNLS/jIQJ1k31nkE1fn", + "bMNoEv/+579gUJRN//3Pf1k873//81+w3bds7QRobkawUGOC1cUh+oWQpIcjOiduMoBjTuZELNDuwPr8", + "4REqlGe3WpocsiF7Q1QqWCFU35Qck7ZBe1Wg50NZSqSFitEv0omth2JiGz1+G7eXDSnvdUd3PRh7MIPC", + "BPSp6HgAAMqoKQ5tLdGO32Vq5lxymlbDNGvBeqvliyLXynBvzwxwTQEDJPbtO3hgJ402zs9fbPYRWFuG", + "K6DmDdgOeTPWjOh/k0mrZZKRKGWBAlQ2ssngJy13+h/bd9p5/W2LX5Pb35ZBW8Pvb5w/AJvoVuDbHUCL", + "OwA/3dx9gM8pf+wAwu4uWNB08UCxgo736jQ3TwokewhnANpwQAzgUOUCnR2dIByGgki5+Z/tKtAzNVya", + "Hx2IM4D9f4hbazsWLixElTXVygzyWMTBGztqhN28qtUli+fbVqkYRONJl9WFyI+8uz89Kp2uc4zkFTBz", + "Xvt2kqyM06My4PrbArf0ApwAIZ36ku3TIhetckiZCMDsyFmqLlnxfHLsNuT9uaZs1ymrng33IBSPKwLx", + "AQVhOUuzWDP2MXHzu2wVHRjqEs/Vl8Wag/vTgu7bi+Vj88fkxgorZNNS0OAJNB6gL4kyKAKdO1xo24Nn", + "4udEuF3tSnzDrLNpmU+RgUOACcHV/HLb98S80s70Ne19TZYvkGcdjcWS/JuK0sLYzWm1zMA9scVI786+", + "hR7WMm9v78bbMpiHyBB2M3Yea6FIiDawXLBg89ul961ztAmJyo1Y4eZNQpREWEF0JACxZHaWHtvOPeh1", + "WcFSgRWxcUOPMRnvLI0idzUzJ0Kh10cnRgQUD6utPyGSbLUR4sTC0nPr3Ztfe4QFHEIHs7A3v7Znn9yy", + "KWI4q5Rgd//8/AiTzKg7eJtUsc9Yf1sf2gSl9in/r52fIjoWWCz+a+cnHCWUkf/afR5hRaTavDNmGdzX", + "GXLfpsEjZj5tGdAy0UA0sSlAB65QpbO3WmrT7v2vSqE2k15Lpc7o+k2rbqNVF8m1VLG2S3GnqrXp44Hu", + "jjJm81EbHn3DmbgHd6TlyALOROl+JkeamHGp4NHjSzq0kZ4047jisdHSr55vyKXHh2Pdk+MuEBJqhwKy", + "uc3puScvuxvHvSu3tt/7d7E/j8d0mvJUFtOFYqyCGZE2lS4iZQH82NTu/HhuVLy/YC4d3OfRce969Te+", + "vyONv7qgRnibq7JVOr97q63Ob9/XOr+BGbTphhZ+vetKc2w2RD86oMG2bFzCY6xHZfrG5bNF0DttqOTm", + "AgIL4nDI/o+2P35TBMcffnB5TelgsHMAvxM2//CDS21ip45VCIOq4JDi+vzVMdxPTgGhEYot5VmU1XGY", + "6qzAeg5e+j/OQMqvaNtbSI4Lv1lIrSykArmWW0h2Le7WRCpD1N+7jeT4zUdwC/T7dVpJX/DFw71bcDKd", + "TGhACQOgf8gWlbVIO2PJfbsZuWGWILM3fYUwnZIm0tqMzKTWCg09ry167yFaJ3kxlfu2Hl0Z08eZ7cAT", + "WxfQ2mu5ttBssH1p/DC439Pr/g21x8xixiKqky7RSrenXIcpVBOnCsJLc4AfiN9Fwpg1WYt9dJTldcs0", + "SbhQ0hS7AQvBlMOcaQvBVxinXOvGV9wGCrpQIrtDBuVO9WODT7F1SRamlA3lLKtak83UVoTxZdGVSwk9", + "6Da6fSXUXyeplRJ6z9vYVr57OCX0wUTHvah7J6WCohvZxgCLe0yyncyzNE36kbLp5qOKJTbCKptbAY7M", + "o2pt4VRxVwd/a8YNMpEfnO0swgFgs+nXDGyQzfs1OGHFpiCZV/AoIsLAQSWpcnWzhiwbHGWFusC2SMWF", + "bn6UMkWji66JpoGcfokwW1hMlCErdYaVInGiBZtF+YERCpKYEVcKhulBU55KeKuLJC91iXB0hRdyyASZ", + "RCSwc4PiioIEBjktivroZw6J1AhPMWU2t1e/acptfSeH7IKGERnZPOgLRCWSMy4UYSREMZ8TWe6XYBFR", + "ImASR1hTTqIYLwCQyGCzGfrwhBjQn1K2Ndf/xiykUIxK95xN+XDIMNoZDFBMMJOIQkKuxBOiv7JtIBhE", + "aUDfI4z2Bs/sV5V1A9BMR/4NvV+EIHMe4HG0QERzMZyIahMWMNtfUNdRL9+ECmnWK3Mv2qo/pYWl0tWn", + "DLsoZQGUT0yF/hcXKGX2eNUtCsgxh3naSzhCRVZ2zCbEj0mANT0ZL/cDUGQ8CFLhOxz1UhcK4/0nKpmF", + "6Z0DqXzSSdMBwZ4KYc0ZVzPY0xy20ub3DVyVM9XXcch4NwkXCKMCX+cOBaghzaZoA6C7LvKSW8xVbbzY", + "/N7tHb19rSBw29+AZz2W8wmYiE8mpQ24+mgyG3hZ4kKdhb/WfXrkai0WRVxI8ZRxqWjghGG1GvA347G1", + "8bicsl5unnBxWdStyvz7ExeXba2vc1fi/lEZYcUZfoH3AHp4AL768NcB4Iw2hopmmns30Kr8le1SULqo", + "ki7OmKOIs6neRblT/N699hWLLohSUMudKeecINoIGdkfTblGPRlbDA88/IFt9aFlke79Hu6CXnGFaJxE", + "JCZQzrFnmE0vdqZVm2rLVKJZVkdvPVmpd1UxKdfYgtJc/3edOgR85RZsA7T3+nJ5hWrEp6uBuLLOHeqU", + "B4lryEw1aOJKR1+gTAZrhdbAXqOrGQ1mgMoFdqtu34B24SS5yABJNw/RS9jIRVxW6HzDgF1rXpM8IgZs", + "ax7HF4f1goXvT0/hIwPIZUoTXhwiV6QwOz+kfquIsqVnEWGp0CuLHbaRGeOwohcKa3szm9+mxd/KAWOH", + "zIfFxciVbZBO0EUBluuiAZfLydtf+fTBlLFuM8y3mYviyJqOwJuEhZ2mGAsa+RG5tgcDH/psS3QwM4w7", + "BgerDeZXPs0gxkusjJOkLfvaYQIXz+N4CQ+jjVyCIKlCnqq/SRUSIeBjy91NzI02cGDLy+BLzajMSCW3", + "sTeB/byRRAbz10sqLVQ73Q5hadw5/M3+ax7HnW7HjqeAFbyGcr8CZa3aYD3iRa9MAUrtm1q+DkhaWdgX", + "UNIqJ4c1p5s18jfmha/+ZtH57B6QDUE/qDhxvyQVtDDessOHcSQZTuSMq8eFy2RdTRWtrdlV42bZ08ML", + "U1cDo00Ix7n99Nx9+QVYv6siO9yYkZvuvYd41EfwmDNhZW02Ey6qYD6rYj++eEa6vSWpTbUNh3zjzfX9", + "fK0YM/FV43dLE5qaSDhVPMaKBlCPI5hxLgtsPyYzPKfcXpW6O6uMM8G5YexMG0J/oVn1wjqCL6wif2id", + "VggXH9k++vC5Dbz3f+Ee5V/8VLDLM4nfdco3YFZDwWBByQQlOJVE61VpTJApxW8LsBAczFCAE5UKArWl", + "CIopo3EaF1wN2nAScxwhKtHFdnzRReNUoQiLKdhF5qEJpxck4HFMWEjAQzZkM4LnVBt1AkVYERYsepJA", + "Tco5ySv9ayPfRuGYmlaCaA6knHVRTBQOscKgalzoHT8yWTwXWZlKY1gzcp1zQzhkImXfG5xt3eyFG+gF", + "IlLhcUTlLCtnFuCQsMALYn3+ZYux2/cGnxNVnegDxeXcSJY+ZKBO0evphvNlxPA8smBkLuwythHzS5Re", + "2WxEltMfHBv9Z25pM1c3xwe64slIvGwXfxl3OxnTfTH3Ow9/gcMFClPTXWFXApt/rbcymUAphjtBaqVZ", + "xptezWR1mzIyryXztv50f57cwJv2hUjCbqNh31QhJJ/0lyByLVVvJHMfyI1ofUkFr9gDimAXU/Vg6hMX", + "BSn3WNydVmCbrZnJ7aJ0UgKD9cXZN7FdFds25OCmYtv5ZmuX6gVBTlkPojT9Ety6cRtFtXUd/IfmglRm", + "VxCZDy4i87uDexOLJ5kgNKIxwYuI4/BrCNNdcoMTcCEM/gMgSjwm/NGC17AYoA++uW4mIbout/L96elm", + "k5QQaqmMEOoRS4hCUoz+LPYV0Z8TIWjoioMfnR7bgFkqkUhZH72OKVTsviQkyXNKAMijr+fnkDDqZY5L", + "kBfdDmFKLBJOmVo5ivzVuxnMpxsVR75nOWmhor9dSLe+kAbP/uMTZyBlIGvCTGC5ZaqwagwFdKFxlJna", + "51ovw2Oe6ta1DNJk0us5hVNwQiMiF1KR2MQFTtIIthuUHbBVKe13ZpW7EBWrd45JWEuIiKmUlDM5ZDZb", + "IyFC960/1+0XQpy8FwIKZ/L1zAjJLyN8Tg/GRIxh1UQ1wCyCmvCdw84WTpKtECvcEKJlh/cZQ/oJ4uGQ", + "XMRjHtEARZRdSrQR0UtjnqC5RJH+Y3NpQN0Ivrvtmps331ma0idswr1lyQzPZsz8VeVVWbHmLiYfnVh7", + "SYqbxckfWGi/WJMr5ZogOOopGpMMuQalikb0oxF1uhEqFQ1M0k8OWfD+NEctGLJTooR+B0NyWRSRQDmH", + "zVYieLA1TAeD3SChAH+2S2BwIPCaH8fQ49HZO5MISmIuFt0h0/+Aht8+PzO3uxNsvQmFgTKirri4RCdb", + "r1eEGJ8Dmf6DY/TMBJdiB3gX/NuV4PqIII17SDZsUZ4sM5V48tUHkVoN7ptf4XH6FQCSKZvNxlTgAJRi", + "OUtVyK+Y34cw51Ea63+YP05WAXspHMzew6tfjLZrhrOyGzfBR7Ep7ZxCYsomPsilhyHYY41Z1YRzUwAl", + "phQN6D0Fnquvkbtv331fpOMXeN1pKepKkn4xe+u+Tz47BodxUaTHY9nmhtPcTBRf7n26wrTZ+/RjxINL", + "acFQim5DbbcBwLj+MQeEtleEoCZAbiayIEKIXCdUAPJbxQFpMHckwkgREVOGoy2Ys2kEoK2dFwvPOYUU", + "6SCikKRGQ0AtigCd7mpGGNKzAUeVa6BwoyttaaniO8XLSMXRmAQ8Jg7ue9Nnuv0dU/UTF2Xs7i9FLr4t", + "0F/PR09Vz3MFXHlzj58FX36KryFUOkzthbIb0cZLnv9oXEFdBGsz7OwO5LDTRcPOTjzs6BU4wuBCxQrt", + "o5iyVBHZR8fGvwVJsAcDJEnAWSgd6rjz4O0OZFNKrGHLhvzKA/juPtUey1VAyje2E5940O8h/T0k7aCN", + "4oazezLswqYLEU+VcffbfWXfCokC98jmvd/VFvbIN9u+jST/u92+JRkFq6zFZWHpjWRPUjkjzS63X00l", + "n1SNAc3aXPvob9DvfCy7iJEr4w0XUvVrck9/fWY6uA+kfd3VOij7du7fIPZbQOzntPLDJZoAS30kO+4w", + "mInk2iDCYpf2bnkILAnAbuABjtDro5MhC7QoMuB+gsQcpJMFBDen8PO/n6MXR2+66BgqPaKf0/FmHx1l", + "YLHgzB2yMReCXzlnru7ERXzAeZ8IPqehPh5YiBjRVEmIkFQqEn6PuJoRcUUlGTLQU4Bs30nEr1g+nKDc", + "JUolCfvoNYsWrqS2uSwaMqMSGikaYIbGZvuQ0KcnGCICG99l1Lru4IGqI5st6rnicUzzrTDB/RXxvJ+L", + "LBPKYUrCjVOA42SQB4DDxeMKN5IzlMkvn5QqnqgZtH9TIq/d5UttAOiyKTj7C/KKL93VJbD2/9TdBTN9", + "tHdBSWmdNBNnJUFWXsa6/N2ZwQy2d0kBTnBA1aKLcKRP+OxSKZVZ0Egv01DHguDLkF+x/pC9yYqR2Jxb", + "dHT2ruvuUlFI5WXXntxwXdpHr+dEyHScDQ7BRjMHM9CchEOmOApwFKSRPojJZEICSJeFGiOy4bo1G0rn", + "DvdO3om3IEoh8Dx9dHXY/DwBq5ezRZXjtsxSbwkSRJjGzQjdVoWBmECIBhjrRjlDlE0iG/UUCC4lsk31", + "SESndBzZGB7ZR2+1hodjMmRJhBkjAjQ5NF7A0HuJIFKmJgdbNwBItoajuihH30sEVzZ6IOJcSHPhrzn8", + "/SmSiiRL2OyNafkU5nxHWp9p3Pb0QH7kyhiavRX2FaQXxHCKIbjmozRyMYb3Gi1uBvTQWuJj2fhvBZ1O", + "tU0lODZC1kTMmW3tyGk2fSmpuLEm43n2VruajFmrhcTBQlLdUvS0UQ5IHXbWC8zzdH5JGwH27KP1En1/", + "0R+17LucUOofhH30mbP8Wkrdnxfy+Nr6mHIOf2wen8LIS1u1lAu7GvmqdfLrXSajtoa4ejBkq8cMaIVL", + "Ga5NBu+XxwiD+wViuO+qZY+bt0qAVCXbtCErfzXk/BfBgXeDNf/AQCQ3wJr/olLjAQz84SBKvBv1oVLd", + "S9fDriDsVw8Xf1cZ7gYzHhDTmjLcjdSz8aVLDaX39p12ZpJt8WvS4G1I4hr6uyP7N6u/hclQINaqW2LN", + "8CRO1MLFnPFJJS5M0o+k33BFmoWW3t0l6Q2iLm+PPRyfNsZcfp3XpA8S1mnr61GJTo49hdEfGQxgcc+V", + "DpYtfer0sAhmdE6ane7lHWxJlAjSS3gClyuhIZilhzvLFBb96Udkm7ewqPZfUKAR8OxJiEIqSKCihSmW", + "qSWC6eM7iQTXlgA852LRHD9htshPgsfP7WxWnId2T1lnWB4KGC96IVa4N3fSZokL7TMCMF3IoxZ4iDL0", + "8ke0Qa6VMGUg0ERbPohOMpKaivgSeHKzOODtQYNnk34ko+m4zSiXFPR4bQumoCCVisdu7U+O0QYUCJsS", + "ptdCq/oT0GRdnE1pjJ05jwxVtxsIuq7fVSsVWXU3Z1yYwT2IDtPmQJp+pElZLJiI1s5hZ0wZhsGtLJ1R", + "3lMmz173hymztWfdGrlRfDvCrOW34YwdzYlQrNISUXFuUJg3vx1zj/mYK+YruTOtdNq58Jzlzut2KUwt", + "M4vuojZDlt52v27r919O1g2VjzLhxrrO55lB2uQ2/7JYcHB/58N9u8vfP+IszZfEGd8FVzk0oFv0Mcyv", + "EHYdkjmJeBJD0XB4t9PtpCLqHHZmSiWHW1sQnj3jUh3uPXuy2/n04dP/HwAA///SB8fhkcsBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 1b52e3e4..baef2cad 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -19,6 +19,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor/cloudhypervisor" "github.com/kernel/hypeman/lib/hypervisor/firecracker" "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/imagepush" "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/logger" @@ -245,6 +246,14 @@ func ProvideRegistry(p *paths.Paths, imageManager images.Manager) (*registry.Reg return registry.New(p, imageManager) } +// ProvidePushManager provides the manager for outbound image pushes to remote +// registries. Credentials default to the server's Docker keychain; API callers +// can instead lend per-request credentials, which the manager borrows for the +// duration of a single push without persisting them. +func ProvidePushManager(p *paths.Paths, cfg *config.Config, imageManager images.Manager) (imagepush.Manager, error) { + return imagepush.NewManager(p, imageManager, nil, cfg.Limits.MaxConcurrentPushes) +} + // ProvideResourceManager provides the resource manager for capacity tracking func ProvideResourceManager(ctx context.Context, cfg *config.Config, p *paths.Paths, imageManager images.Manager, instanceManager instances.Manager, volumeManager volumes.Manager) (*resources.Manager, error) { mgr := resources.NewManager(cfg, p) diff --git a/lib/scopes/scopes.go b/lib/scopes/scopes.go index 0240c9c4..84c7c592 100644 --- a/lib/scopes/scopes.go +++ b/lib/scopes/scopes.go @@ -233,6 +233,11 @@ var RouteScopes = map[string]Scope{ "DELETE /images/{name}": ImageDelete, "GET /images/{name}": ImageRead, + // Pushes (outbound image pushes to remote registries) + "POST /pushes": ImageWrite, + "GET /pushes": ImageRead, + "GET /pushes/{id}": ImageRead, + // Ingresses "GET /ingresses": IngressRead, "POST /ingresses": IngressWrite, diff --git a/openapi.yaml b/openapi.yaml index cd796174..6cf2d95c 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1780,7 +1780,101 @@ components: status: $ref: "#/components/schemas/BuildStatus" description: New build status (only for type=status) - + + PushStatus: + type: string + enum: [queued, pushing, pushed, failed] + + PushCredentials: + type: object + description: | + Registry credentials borrowed for this push only, docker-style: the + caller's registry login (e.g. from the client's ~/.docker/config.json) + rides along with the request instead of living on the server. Never + persisted or logged; a push interrupted by a restart fails instead of + retrying without them. When omitted, the server's own registry + credentials are used. + properties: + username: + type: string + description: Registry username + password: + type: string + description: Registry password or access token + format: password + registry_token: + type: string + description: Bearer token sent as-is in the Authorization header + format: password + + CreatePushRequest: + type: object + required: + - image + - target + properties: + image: + type: string + description: Hypeman image name to push (tag or digest form) + example: "docker.io/library/alpine:latest" + target: + type: string + description: Full remote reference to push to + example: "123456789.dkr.ecr.us-east-1.amazonaws.com/myapp:v1" + insecure: + type: boolean + description: Allow pushing to plain-HTTP registries + default: false + credentials: + $ref: "#/components/schemas/PushCredentials" + + Push: + type: object + required: + - id + - image + - digest + - target + - status + - created_at + properties: + id: + type: string + description: Push job identifier + image: + type: string + description: Hypeman image name (normalized ref) + digest: + type: string + description: Cached manifest digest being pushed + target: + type: string + description: Remote reference the image is pushed to + status: + $ref: "#/components/schemas/PushStatus" + queue_position: + type: integer + description: Position in the push queue (only when status is queued) + nullable: true + error: + type: string + description: Error message (only when status is failed) + nullable: true + layers: + type: integer + description: Number of layers pushed (only when status is pushed) + bytes: + type: integer + format: int64 + description: Total compressed layer bytes pushed (only when status is pushed) + created_at: + type: string + format: date-time + completed_at: + type: string + format: date-time + nullable: true + Build: type: object required: [id, status, created_at] @@ -4630,3 +4724,124 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" + /pushes: + get: + summary: List pushes + description: Lists outbound image push jobs, newest first. + operationId: listPushes + security: + - bearerAuth: [] + responses: + 200: + description: List of pushes + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Push" + 401: + description: Unauthorized + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + post: + summary: Push an image to a remote registry + description: | + Creates a push job that exports a hypeman image from the local OCI + cache to a remote registry (e.g. AWS ECR, Docker Hub). Credentials are + borrowed from the request when provided and never persisted; otherwise + the server's own registry credentials are used. Only images in the + ready state can be pushed. + operationId: createPush + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CreatePushRequest" + responses: + 202: + description: Push job created + content: + application/json: + schema: + $ref: "#/components/schemas/Push" + 400: + description: Bad request + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 401: + description: Unauthorized + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 404: + description: Image not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 409: + description: Image exists but is not ready + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + + /pushes/{id}: + get: + summary: Get push details + operationId: getPush + security: + - bearerAuth: [] + parameters: + - name: id + in: path + required: true + schema: + type: string + description: Push ID + responses: + 200: + description: Push details + content: + application/json: + schema: + $ref: "#/components/schemas/Push" + 401: + description: Unauthorized + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 404: + description: Push not found + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + 500: + description: Internal server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" From 2076b555f1f251d8c41486c2a3e96b941a28f1a0 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:14:51 +0000 Subject: [PATCH 16/21] Fix gofmt alignment for push manager wiring --- cmd/api/api/api.go | 2 +- cmd/api/wire.go | 2 +- lib/providers/providers.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/api/api/api.go b/cmd/api/api/api.go index 62e4a647..8a2c72db 100644 --- a/cmd/api/api/api.go +++ b/cmd/api/api/api.go @@ -7,8 +7,8 @@ import ( "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" - "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/imagepush" + "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/network" diff --git a/cmd/api/wire.go b/cmd/api/wire.go index cc4870a5..133bf41f 100644 --- a/cmd/api/wire.go +++ b/cmd/api/wire.go @@ -14,8 +14,8 @@ import ( "github.com/kernel/hypeman/lib/builds" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guestmemory" - "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/imagepush" + "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/network" diff --git a/lib/providers/providers.go b/lib/providers/providers.go index baef2cad..b67feac6 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -18,8 +18,8 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/hypervisor/cloudhypervisor" "github.com/kernel/hypeman/lib/hypervisor/firecracker" - "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/imagepush" + "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/ingress" "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/logger" From b38c3b629571d1490ba746009b8c7876012eaf8b Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:43:23 +0000 Subject: [PATCH 17/21] Fix push status enum varnames and empty-credentials fallback - Pin x-enum-varnames on PushStatus so generated constants carry the PushStatus prefix like the other enums, avoiding bare Failed/Queued names in the oapi package. - Treat an empty credentials object the same as absent credentials so the server's default credential resolution stays in effect instead of pushing with an empty auth config. --- cmd/api/api/pushes.go | 8 +- cmd/api/api/pushes_test.go | 20 ++ lib/oapi/oapi.go | 665 +++++++++++++++++++------------------ openapi.yaml | 1 + 4 files changed, 360 insertions(+), 334 deletions(-) diff --git a/cmd/api/api/pushes.go b/cmd/api/api/pushes.go index f1cff277..1a080369 100644 --- a/cmd/api/api/pushes.go +++ b/cmd/api/api/pushes.go @@ -99,8 +99,9 @@ func (s *ApiService) ListPushes(ctx context.Context, request oapi.ListPushesRequ } // pushCredentialsToAuthn maps API credentials to the go-containerregistry -// auth config. Returns nil when absent so the push falls back to the -// server's default credential resolution. +// auth config. Returns nil when absent or empty so the push falls back to +// the server's default credential resolution — an empty credentials object +// must not mask the keychain. func pushCredentialsToAuthn(creds *oapi.PushCredentials) *authn.AuthConfig { if creds == nil { return nil @@ -115,6 +116,9 @@ func pushCredentialsToAuthn(creds *oapi.PushCredentials) *authn.AuthConfig { if creds.RegistryToken != nil { cfg.RegistryToken = *creds.RegistryToken } + if cfg.Username == "" && cfg.Password == "" && cfg.RegistryToken == "" { + return nil + } return cfg } diff --git a/cmd/api/api/pushes_test.go b/cmd/api/api/pushes_test.go index 69de7dec..ec4daca8 100644 --- a/cmd/api/api/pushes_test.go +++ b/cmd/api/api/pushes_test.go @@ -111,6 +111,26 @@ func TestCreatePush_NoCredentialsStaysNil(t *testing.T) { require.Nil(t, fake.createdReq.Credentials) } +func TestCreatePush_EmptyCredentialsFallsBackToDefault(t *testing.T) { + t.Parallel() + + fake := &fakePushManager{push: &imagepush.Push{ID: "push-1", Status: imagepush.StatusQueued}} + svc := &ApiService{PushManager: fake} + + // An empty credentials object must behave like no credentials at all: + // the server's default resolution stays in charge. + resp, err := svc.CreatePush(context.Background(), oapi.CreatePushRequestObject{ + Body: &oapi.CreatePushRequest{ + Image: "alpine:latest", + Target: "registry.example.com/app:v1", + Credentials: &oapi.PushCredentials{}, + }, + }) + require.NoError(t, err) + require.IsType(t, oapi.CreatePush202JSONResponse{}, resp) + require.Nil(t, fake.createdReq.Credentials) +} + func TestCreatePush_ErrorStatusMapping(t *testing.T) { t.Parallel() diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 576fa723..81ea612b 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -195,10 +195,10 @@ const ( // Defines values for PushStatus. const ( - Failed PushStatus = "failed" - Pushed PushStatus = "pushed" - Pushing PushStatus = "pushing" - Queued PushStatus = "queued" + PushStatusFailed PushStatus = "failed" + PushStatusPushed PushStatus = "pushed" + PushStatusPushing PushStatus = "pushing" + PushStatusQueued PushStatus = "queued" ) // Defines values for RestartPolicyPolicy. @@ -18451,341 +18451,342 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+y963IbObI/+CoI7pxoaYakqKttdXT81y253TrdsnUs27Nnml4KrAJJtKqAagBFie7w", - "fpwHmEecJ9lAAqgriizKulhjxzkxLbOqcEkkEpmJzF/+2Ql4nHBGmJKdwz87MpiRGMOfz5XCwew9j9KY", - "vCF/pEQq/XMieEKEogReinnK1CjBaqb/FRIZCJooylnnsHOG1QxdzYggaA6tIDnjaRSiMUHwHQk73Q65", - "xnESkc5hZytmaivECne6HbVI9E9SCcqmnU/djiA45CxamG4mOI1U53CCI0m6lW5PddMIS6Q/6cE3WXtj", - "ziOCWecTtPhHSgUJO4e/FafxIXuZj38ngdKdP08VP1eYhePFGY9osKhP9lfK0mvoDeFU8RgrGiBpvkEJ", - "fITGWJIQcYZwoOicIMrGPGUhent0hgLOGAl0Y3LI+FgSMSchmggeIzUjaMalgneUwMElUngckf6QdbqV", - "9SBMPwlXU+nvM6JmRHgGSyWyraAJF0jNqESU6acB6RcXTImU1Cnb7dAwIiNFY8JTVSfUz/wKRZxNYVqu", - "XRSnUqEZnhP0kQiO/khxRCcLyqbNRBqTCRcE/bxISIwZSiIcEImoQpQp7mZjaJTz2H7sYy46ZVyQUUik", - "ogzr9kcJF2ZHlEf/Gv7AESq8C0OD95GaYeW4nHGFLglJyhPFV/iyTMbfdna6zwaDwYduhyoSm22Fr2mc", - "xp3Dg/393f1uJ6bM/Hs7Gz1likyJ0MO3v2Ah8KIwHclTEZBRQEOxbCZBRAlT6Ojk+M0NJ9DZHvTh/7ae", - "drqd7Wc7/e2Dp/Dv7YNOcVo1wpdH/mn51jtXWKWyLoPMbhpZRhkVmKQ+61dpPCYC8QkKUiEIU9ECwZYi", - "YQumK0174FuKgLMJnabCbUHfliuRc4YlwswIjV5FXuSNtdp3gRZiIb9iI0FiTJmmcW0Qb9wjpHcosptI", + "H4sIAAAAAAAC/+y9/3IbN7I/+ioo3j0VaZekqJ+2lUp9ryI5jk4iW8eyvfds6EuBMyCJaAaYABhKdMr3", + "z32AfcR9kltoAPMTQw5lS7LWrnNqI3Nm8KPRaHQ3uj/9ZyfgccIZYUp2Dv/syGBGYgx/HimFg9k7HqUx", + "eU3+SIlU+udE8IQIRQm8FPOUqVGC1Uz/KyQyEDRRlLPOYeccqxm6nhFB0BxaQXLG0yhEY4LgOxJ2uh1y", + "g+MkIp3DzlbM1FaIFe50O2qR6J+kEpRNOx+7HUFwyFm0MN1McBqpzuEER5J0K92e6aYRlkh/0oNvsvbG", + "nEcEs85HaPGPlAoSdg5/K07jffYyH/9OAqU7P0oVv1CYhePFOY9osKhP9lfK0hvoDeFU8RgrGiBpvkEJ", + "fITGWJIQcYZwoOicIMrGPGUhenN8jgLOGAl0Y3LI+FgSMSchmggeIzUjaMalgneUwMEVUngckf6QdbqV", + "9SBMPwlXU+nvM6JmRHgGSyWyraAJF0jNqESU6acB6RcXTImU1Cnb7dAwIiNFY8JTVSfUz/waRZxNYVqu", + "XRSnUqEZnhP0gQiO/khxRCcLyqbNRBqTCRcE/bxISIwZSiIcEImoQpQp7mZjaJTz2H7sYy46ZVyQUUik", + "ogzr9kcJF2ZHlEf/Cv7AESq8C0OD95GaYeW4nHGFrghJyhPF1/iqTMbfdna6zwaDwftuhyoSm22Fb2ic", + "xp3Dg/393f1uJ6bM/Hs7Gz1likyJ0MO3v2Ah8KIwHclTEZBRQEOxbCZBRAlT6Pj05PUtJ9DZHvTh/7ae", + "drqd7Wc7/e2Dp/Dv7YNOcVo1wpdH/nH51rtQWKWyLoPMbhpZRhkVmKQ+65dpPCYC8QkKUiEIU9ECwZYi", + "YQumK0174FuKgLMJnabCbUHfliuRc4YlwswIjV5FXuSNtdp3gRZiIb9mI0FiTJmmcW0Qr90jpHcosptI", "DyngTAkeRVooKEXiREm3i7pajDOEkySiAYie0qbaiwey0+2wNIr0w8oI89UmEZ1SeKEVaagsLJL7FimO", - "CFNEZDu8DWlKYrGp45zc3tXI5WKrLmc8CkcpUzSq9/pO/2xpWuyJSjQjUYj4ZNJFdIIw0q3onw2Pl8m+", - "M9g56A32eoODt9tPDgfPDgf7/+h0OxMuYqw6h50QK9LTq9xmbUBoS8oC/+qwKovE+kASJDCrkx1YpQUc", - "k4DHBOmml4587/NH3njcHOWLql9E9sX8ZPUsdb9yZLTj7ghLlQkhWC+qFiPsGdNbGhOpcJxoOaTHUCBm", - "kxRyDVbXwVF+KYG3P4vAjFyrkaWQdz4+/iDXCQn0icidNMkUDN2eZe9MZN0LjwuCpR6wVpP0YfpbJ2Uy", - "TfTRTcJREmGl29U6FbDBKKZS6k+zH0IqjRzpdhyTjxhXI5EyZl5kRF1xcVl807Yyokmn25lhOZpPk7TT", + "CFNEZDu8DWlKYrGp45zc3tXI5WKrLmc8CkcpUzSq9/pW/2xpWuyJSjQjUYj4ZNJFdIIw0q3onw2Pl8m+", + "M9g56A32eoODN9tPDgfPDgf7/+h0OxMuYqw6h50QK9LTq9xmbUBoS8oC/+qwKovE+kASJDCrkx1YpQUc", + "k4DHBOmml45879NH3njcHOeLql9E9sX8ZPUsdb9yZLTj7ghLlQkhWC+qFiPsGdMbGhOpcJxoOaTHUCBm", + "kxRyDVbXwVF+KYG3P4nAjNyokaWQdz4+/iA3CQn0icidNMkUDN2eZe9MZN0LjwuCpR6wVpP0YfpbJ2Uy", + "TfTRTcJREmGl29U6FbDBKKZS6k+zH0IqjRzpdhyTjxhXI5EyZl5kRF1zcVV807Yyokmn25lhOZpPk7TT", "XXZslZkauiARTiS0Z1dcjIgQXHSMarwYTbhwi6TP3JyES5qqUUhmR6yHQp1up0SATJy7ubhxZ6vqHRz0", - "ArwkjFVhzACYTH3gxbbqw82Gtlywm1PEKNFumZH9WJYlQEjxlHGpaCBbiXlQHvTyxjz0iM7jrDlEQ8IU", + "ArwkjFVhzACYTH3gxbbqw82Gtlywm1PEKNFumZH9WJYlQEjxlHGpaCBbiXlQHvTyxjz0iM6TrDlEQ8IU", "nVAirF5NkNDnQ0yQawTpRhBlKJWVfZCp/iMy17baaL43UkFSJ0rFsCkuXkE3yU/EwqmcLX+2U1YwaXnu", - "XsNpjinsyWMyp+ZoKetudmlGoaBzIjziO1MAjCg076ENvde1CGGckc0SpdichhS3EQchjGlEPdxzdnSC", - "zGN0cow2ZuS63MnOk/HTTnOTDMceXvg5jTHr6Q2hh+Xah3eLbf+65zVReByno6ngaVJv+eT16ek7BA8R", - "Aw232OLTHZ+mmgR0hMNQECn983cPi2MbDAaDQ7xzOBj0B75RzgkLuWgkqXnsJ+n2ICRLmmxFUtt+jaSv", - "3p8cnzxHR1wkXIDNtnLjFMlTnFeRbcqr4uP/H1MahXWuH+ufSQOdiJBUKi2vfjSvIUGMKYeuZlwSFOBg", - "RtDYGC1gnkN7bVg+69idXr4RHFOZcJDz6P1prs2BaUiuSZCqUr/fo5BqWziwSlVhci1GFHC9Rvr89WkW", - "QAFk39GmtnKq042VgEAQvKI7/Uarzur7PzXsNYplU+vuFS3mYxpFVJKAs1AW+6BMHew1T6awi82xWevq", - "hf4ZxURKPCVoA9xSYG4YCa+1rQmmEQk322nYTZP5nY8L51ppzwEb9PA42N7Z9Qq0GE/JKKRT61es8qD+", - "XevHuh2F4G3/REDDaDcP6FKQSb2/n+A8gU4EmRBBNMd/ZneJ4HPCsDWp/gL9dv6vrdzhumW9rVtAzLP8", - "9U/dzh8pScko4ZKaEdbEhH2i2QhIjeAL/5jh0bK1LnCUVFgs3x/wxi3sxFzZXEkb6/rR+haervzkrX6n", - "KtBBHGUKTkEKNMrtF1rT8qgsnCn7oOIC5lMUUWbMIC2/zVqAsrdIyA8Rn252bo0OGfnrm1+P+wbCy/zQ", - "0Jp+1s2sgohPi9ScESzUmJSI2XCu2oby0TWS/6y0fSoHKJZktFyCnFHGSAg+d7uxzZtat/baPrCLLqka", - "zfUJ7NtzMKxfqEL2jcamIh5cTmhERjMsZ9ZJGYbUOFzPSjPxqJClywwMTgLXIKg2YFSf//x8Z/8A2Q48", + "XsNpjinsyRMyp+ZoKetudmlGoaBzIjziO1MAjCg076ENvde1CGGckc0SpdichhS3EQchjGlEPdxzfnyK", + "zGN0eoI2ZuSm3MnOk/HTTnOTDMceXvg5jTHr6Q2hh+Xah3eLbf+65zVReByno6ngaVJv+fTV2dlbBA8R", + "Aw232OLTHZ+mmgR0hMNQECn983cPi2MbDAaDQ7xzOBj0B75RzgkLuWgkqXnsJ+n2ICRLmmxFUtt+jaQv", + "352enB6hYy4SLsBmW7lxiuQpzqvINuVV8fH/jymNwjrXj/XPpIFOREgqlZZXP5rXkCDGlEPXMy4JCnAw", + "I2hsjBYwz6G9NiyfdexOL98ITqhMOMh59O4s1+bANCQ3JEhVqd/vUUi1LRxYpaowuRYjCrheI33++jQL", + "oACy72hTWznV6dZKQCAIXtGdfqNVZ/X9nxr2GsWyqXX3ihbzMY0iKknAWSiLfVCmDvaaJ1PYxebYrHX1", + "XP+MYiIlnhK0AW4pMDeMhNfa1gTTiISb7TTspsn8zseFc62054ANengcbO/segVajKdkFNKp9StWeVD/", + "rvVj3Y5C8LZ/IqBhtJsHdCnIpN7fT3CeQCeCTIggmuM/sbtE8Dlh2JpUf4F+O//XVu5w3bLe1i0g5nn+", + "+sdu54+UpGSUcEnNCGtiwj7RbASkRvCFf8zwaNlaFzhKKiyW7w944zPsxFzZXEkb6/rR+haervzkjX6n", + "KtBBHGUKTkEKNMrt51rT8qgsnCn7oOIC5lMUUWbMIC2/zVqAsrdIyA8Rn252PhsdMvLXN78e9y2El/mh", + "oTX9rJtZBRGfFqk5I1ioMSkRs+FctQ3lo2sk/3lp+1QOUCzJaLkEOaeMkRB87nZjmze1bu21fWAXXVE1", + "musT2LfnYFi/UIXsG41NRTy4mtCIjGZYzqyTMgypcbiel2biUSFLlxkYnASuQVBtwKi++PloZ/8A2Q48", "NLTeX/1CfSaFr3XzVr1QWIxxFHl5o5nd1j+j6xzi54Dc4dt09mQc6BjTSLqOXU1rvKdyZv4C2a1HBWef", - "FgOavSL9d41xu53rnm61N8cCqK6bLwzrf1xPhd9+zDst/HqW9V/48Y0dSuGnn9yoCr8d5QN0VDFmktcF", - "DpP2a5WcMtWjDJZAK+DW82SOjtzNaj0ZSC9jpuHByf5ZitVRTaVCG29+Otrd3X222V65ovJyJOlHMpqO", - "l+rNdtBWUdafIf2ZPiqndIrHC0VkHx1hxrhCY4KCGWZTEiI8UforO9aSAb4/WHUJ06gkEdGkIqmPe/jZ", - "0+trrJ4d0Cv57GM8FtPfd733n+Bd1PKm2QOH7WrqV5HAzLOINzuoY3ytLX/DJb7Lz1NzXYWyl+wayOoQ", - "+ig36Cf0WtNcoe0Spb3E9Vt/2RUW46yXMvpHCqudRHhRtwEVwXEPOMJHXiM3zAaSq7eP1PvHfONmeoWp", - "0jsnuzO1U+4iHoX63JlQIVWn/W3YWooBEbelIpV2mV9jqnNElYCNh3o2zsa9EtEJCRZBROrC3YnvRKTW", - "3RoSfebAn5kDsa0Yd0MpSWL341nWRenn47y/0u8vTOefukbSEfuwMWxiuSg7Wiq3XsdUaQ0glebezniD", - "kb3uvwWpld8M4ygioifTJIkoXKA4MdY1TtMpYURorjCuOIW0yUPDSlxHvvN62769d+ebe60NUeNbs6LG", - "gdm4oCvmMI241qwWyE0k9/310UmZdl2E4QFciZSpnF1aFfxzaIP0p/0uGnaSgPYGg8Ggh3d6g0FvMOyU", - "PWzRXs/cPCRYKSL0AP/f33Dv4/PePwa9Zx/yP0f93oe//cVHybZOQ6de2HluOCnTRW6wRU9idaCrvIw3", - "FnDF4X9oXOoTbS2su9JHJ3X3gZlryINLIvqUb0V0LLBYbLEpZdeHEVZEqvLMl7/rXRF3a1W/ZcRiSgr3", - "HVgiLrewCGa/bc2xoJipD2aMaNiJKEuvt3AcHuwNO5tdFGMVaKUVHcOIUK/n2rECyF1hlO5U+ugVVwjb", - "o12L3UN4qd7vVOA4xkLz+BxHNAT2NpKsJ2lIEGYhogye6TdSIpEgKhUM7Q0G7smoMChtanPppjLsoCuq", - "Zkh3i2BaiAuERXywZ64aA5KAG5OHuHLfWyBFfa/ojfK3reyvDfdn/8PfNv/PX26VWYHRlnApm2o+XpNP", - "Kw5zkDMbEb8iItCGckT0bGVX28pUyS4sQwg2JuIsWnyPAqM1Gx8QF4iw0FIa3iuzc7zo4YT2qBmq0Rt+", - "JWyqZp3Dg10vcTfsH70Pf3U/NRBWpBHxSKE3PAU9DB4XI9jcGDL9a9mKOOqmEXjjYspOzGfbdTXt81bY", - "TWTZShsPfONS61Miu15dMZB6KKM23WKfGjAnQuitaOTa0ekx2ojoJbECTZuKaJgOBrsBvAB/EvtLwOMY", - "s9D8tlnWWawzxqOu/NYhwYyDPymK+DqRY6CegnaCo6XulmWk8VL7KGu37pz5mUvVizHD2njMB4DGgl8S", - "PVATT0KJRJdkoU2FBZrqRntzKiGYibA5mmNzY9UfsrdwvwOvuEcS4kLonKCYB5cmynHGQVM2YrGLrmY0", - "MqqgIDjKpWWMKRsyLaB7MuCJ1tKYfQ2mhi4Im1+gGCewzbEgsMe17CeC4oh+NNGqEKFCQqpl9JAR2Bgo", - "wXrPBwEXIQSTcURwMCtQ4TuJLoxf6QKav6BMs/WF2ZiVuMw/O6/fvf3x9btXx6PXZy9ePT8Z/fLif/XP", - "5qPO4W9/dkxUcmbB/kiwIAL95U+Y7yfjhQTnSOd5qmZc0I/mpg9isaQCxR8ntM8TwjDtBzzudDt/Lf7z", - "w6cPzm9mQiDmeht4BvbJqygaZccjko7dTbJE9nbSxcVokmkR9fLs3ZZWnxIspZoJnk5n5Y1hdbe1tgQY", - "GZSPxon0XvJdopOt10hrliiieoNmmuT2YHD645YcdvQ/9t0/Nvvo2OxaGL6WQVxYBVfONPtkAc5HZ+8Q", - "jiIe2KuuSVMso+vKJ+AJU2KRaMt7pXDKX63LqF4vf7qGKNoaU7Yl9TL0gvXoDnxzY4/vCzangrOYMIVA", - "XRpHRJb3yqvXxy9GL1697xzqgyBMA3ujffb6zdvOYWd3MBh0fAyqOWiFDHx59s5EzMG2IThSs1EwI8Hl", - "qg9/hneP4FXYcSqJ0inYuPXVe56RBsUk5sJckthv0MasrKSYLY9gXYed3Zc/Gr7cfgks6dbThiZlrZiG", - "K9FkL3/0MdpskRAxp9J3k/pz9swxTT0ovrQtrDnu+D13JVonRhDxNOwVuux2JlSQAIKQ9b/+ILE20uYf", - "y9FQnu/8F5ytjJMVVgeOEsrIErPjC9Eur7i4jDgOwbFwm8qljePzBJCbB+X1zawhxxK1vIwxZuEVDdVs", - "FPIrpofsEcn2CcpezuTytZ4Jjv79z3+9P81t6O2X48QK6e2d/c8U0hWxrJv23pJlE0kT/zTeJf5JvD/9", - "9z//5WbysJMwOsyN9EG7/i9MC9XQcptxY/zoDQGJ2cGfR9Zw6yyBz5HjvZXRhz4Zz+dERHhRELx2TJ3t", - "AUi/yqgEhVwiZL8reB2Xi2HdmtMPXlYdODsDv6D95qz4wp0VgoDoHSVZltuyDfLGvJ3bkR6m8/Dcj/os", - "sOd+G07LGG1759T+uVNnOT/HyUuajMCgGuFpFrWxLDHu/JIm1kqDL8w2jSIj6MMU7Lox56o/ZCbwXe9N", - "4BxyTQLgBamwQs/PTiS6olEE3l04NOqqg7b5ChkT8LpU+n9FyrponCptyHFFkDWpbQSeHgu8PCYoZdiF", - "2VbMKjvBetQykOWSCEaikTGbZEvKmI+Q/aiRODDVCZY2UUeoNCnT6/iX03O0cbxgOKYB+sW0esrDNCLo", - "3AQtb5ap1x2yRED0s+5EsyO1/fIJ4qnq8UlPCULcEGNoLPOP2xjQ+cuzdzaKWG72h+wN0YQlLLTpjk6j", - "sKlwIWffaYlMwnKzxf4rRG+KFJcMJ3LG226uc/t6vrva+5m6nXmQpOUl3ek2psLNqVApjvRZWrI0vHHD", - "JpXXY1GaTOGiZWvPtTx1UJVjK9s640zLkNfrTRr0+NSMJtzap1bw8tS8a84F8We7wa5o/4S5gSz1KeZe", - "iM/o69w0UssJMD933cxuQKWTjCYVT+TtkOe5LHhtWqXgmpPdHH8SbVzghPYtH/cDHl900cVfSz/ove9M", - "R60/XiFDDZAnTP9UbL/qr1rpSVor6bW4OFjefD2ey8YECjTfRkpgJk3qywwnpI9+BiGOFIm1agIZZxJl", - "GSOI8avvETdKq/t0yPTQJNrQstKSI/MnSjpllE03tRmnDyYchsbpOElVKvR7cypzapZZxzn2apqhGZ2N", - "P4c8ccqCKA0JunDOv4uyUlR3DdZNfusrrFmwhiRguYIxr7biVOnu9YQzRZOnyuSj2KmXc4UqDshVIZF2", - "LFl0zg3W/zwTF1U4gLnHhNWTsxe04DEuuK6bPMRWUfF7ry/JApbceapxzVdddFL7XcmCSB7NiT12i27u", - "MQAecKM45R5u46u27mm9/aup/j7H7aql0PRqTf6yKegBOpCq5yabc4y17lyqqZNCenKmv65W/iUB4oNp", - "eYhAHbvoGluYgOWBmGaWCIVUkEDVmqdsOmQQBX5hf+nb1i70Jtc6yq3AR0A2NijtxaVFhZV1ah80o6fG", - "Y6oUCbtl3eCSkESunpRWr+2dhufiRZArQZ0gc7mKLdUzwiZcBCS2RsLnOQZeFBpbElTStol6ULahb2HM", - "Lksd21gdWHuzHuCBLyWvVzFcworVZoKAy11e4Ci6QBv2pU0kyO+Q4GvXinGWM/vbozPHAlnIyvvTruZI", - "LQUuZkolI/0/cqR38UW1Mfut2+E5vsbTAdhXe3u7dlWtU9UMuNJs2X/qDWxuXpqzVM4aL1wrl43LeEW3", - "c1R4vdlL64BMjKfWnUBJKmdoQ+Gp3jo2gl2fE5uf68GlTJIgFWT1Xn8O6pGNooYxRZiy3s9v354hQaZU", - "t1mMQS9mx4JbpyHdxhq2ecCMm67iZTfezu7e/sGTp8/64aXok0D0U9kjWKredh/H+CNn+EpqtWsrXuAk", - "OZxvr5TzZgmy8TWLfGeFNbMCj7VIdgkDbSy6o/yT/K7lkrKwbQO/6HdXB845g/Ou/fCJIL00mQoMCZy3", - "6YW/cWQFULN5VVeAWi0Pikyl4nExm3tjeSxkmVhzHvVCrPA6oZBmuPWox3hhmjImudf91RRiel6NKy2F", - "YHuhbT43zMWNxbcsTenhxpFAwpHinqxnpzucHAOahH23VR4sZLMrPppPKF8OHmBDGEtgNEYrse4N3UQv", - "Caj1KoGqG8xMqqAhAtgO70+Lt/v9IeuBFnaIjrMOsmazJjGYGDg0F6QbXBQGYWAC0HixiTB6f9pHb7PR", - "fieRtlvnxOXLz7BEY0IYSuGGCZSinlHJigNIJehOqvq5daGZ1PpNCGLg9lk/A+ACZ10GJwbRrmNamY+B", - "vIGFsmeJPvByZ2gr5+WyRJM35jxalmxSAhfZ7w22e9v7b7cHhwP9//9on41y++gBvrael2WLjR8uSp+j", - "dyfHO9Znvnnj/JJbxxfwi7LjPPAZbaSSiJ4Tk3Dl4Ql3LkQVN4Qz3zhK+Y6CjvNMymXvGkq81W/eBXCC", - "L7XD5l6uD21QFZgr82cLk6s7dBYJuB/yXVJQ5G2AekA9maRdCFr6URB8CYAw9XNba3VyZM4zf8RTKk0Y", - "Hrm2Ti7BuZpIo3SXnd/be0/2nu4e7D0dDDyp+XWG5wEdQQpCqwG8PjpBEV5kWWobcK8donHEx2VG3989", + "FgOavSL9d41xu52bnm61N8cCqK6bLwzrf1xPhd9+zDst/Hqe9V/48bUdSuGnn9yoCr8d5wN0VDFmktcF", + "DpP2a5WcMtWjDJZAK+DW82SOjtzNaj0ZSC9jpuHByf5JitVxTaVCG69/Ot7d3X222V65ovJqJOkHMpqO", + "l+rNdtBWUdafIf2ZPiqndIrHC0VkHx1jxrhCY4KCGWZTEiI8UforO9aSAb4/WHUJ06gkEdGkIqkPe/jZ", + "05sbrJ4d0Gv57EM8FtPfd733n+Bd1PKm2QOH7WrqV5HAzLOItzuoY3yjLX/DJb7LzzNzXYWyl+wayOoQ", + "+ig36Cf0RtNcoe0Spb3E9Vt/2RUW46yXMvpHCqudRHhRtwEVwXEPOMJHXiM3zAaSq7eP1PvHfONmeo2p", + "0jsnuzO1U+4iHoX63JlQIVWn/W3YWooBEZ9LRSrtMr/GVOeIKgEbD/VsnI17JaITEiyCiNSFuxPfiUit", + "uzUk+syBPzMHYlsx7oZSksTux/Osi9LPJ3l/pd+fm84/do2kI/ZhY9jEclF2vFRuvYqp0hpAKs29nfEG", + "I3vd/xmkVn4zjKOIiJ5MkySicIHixFjXOE2nhBGhucK44hTSJg8NK3Ed+c7rbfv23p1v7rU2RI1vzYoa", + "B2bjgq6YwzTiWrNaIDeR3PfXR6dl2nURhgdwJVKmcnZpVfDPoQ3Sn/a7aNhJAtobDAaDHt7pDQa9wbBT", + "9rBFez1z85BgpYjQA/x/f8O9D0e9fwx6z97nf476vfd/+4uPkm2dhk69sPPccFKmi9xgi57E6kBXeRlv", + "LeCKw3/fuNSn2lpYd6WPT+vuAzPXkAdXRPQp34roWGCx2GJTym4OI6yIVOWZL3/XuyLu1qp+y4jFlBTu", + "O7BEXG5hEcx+25pjQTFT780Y0bATUZbebOE4PNgbdja7KMYq0EorOoERoV7PtWMFkLvCKN2p9NFLrhC2", + "R7sWu4fwUr3fqcBxjIXm8TmOaAjsbSRZT9KQIMxCRBk802+kRCJBVCoY2hsM3JNRYVDa1ObSTWXYQddU", + "zZDuFsG0EBcIi/hgz1w1BiQBNyYPceW+t0CK+l7RG+VvW9lfG+7P/vu/bf6fv3xWZgVGW8KlbKr5eE0+", + "rTjMQc5sRPyaiEAbyhHRs5VdbStTJbuwDCHYmIizaPE9CozWbHxAXCDCQktpeK/MzvGihxPao2aoRm/4", + "lbCpmnUOD3a9xN2wf/Te/9X91EBYkUbEI4Ve8xT0MHhcjGBzY8j0r2Ur4qibRuCNiyk7NZ9t19W0T1th", + "N5FlK2088I1LrU+J7Hp1xUDqoYzadIt9asCcCKG3opFrx2cnaCOiV8QKNG0qomE6GOwG8AL8SewvAY9j", + "zELz22ZZZ7HOGI+68luHBDMO/qQo4utEjoF6CtoJjpa6W5aRxkvt46zdunPmZy5VL8YMa+MxHwAaC35F", + "9EBNPAklEl2RhTYVFmiqG+3NqYRgJsLmaI7NjVV/yN7A/Q684h5JiAuhc4JiHlyZKMcZB03ZiMUuup7R", + "yKiCguAol5YxpmzItIDuyYAnWktj9jWYGrokbH6JYpzANseCwB7Xsp8IiiP6wUSrQoQKCamW0UNGYGOg", + "BOs9HwRchBBMxhHBwaxAhe8kujR+pUto/pIyzdaXZmNW4jL/7Lx6++bHV29fnoxenT9/eXQ6+uX5/+qf", + "zUedw9/+7Jio5MyC/ZFgQQT6y58w34/GCwnOkc5RqmZc0A/mpg9isaQCxR8ntM8TwjDtBzzudDt/Lf7z", + "/cf3zm9mQiDmeht4BvbRqygaZccjkk7cTbJE9nbSxcVokmkR9eL87ZZWnxIspZoJnk5n5Y1hdbe1tgQY", + "GZSPxon0XvJdodOtV0hrliiieoNmmuT2YHD245YcdvQ/9t0/NvvoxOxaGL6WQVxYBVfONPtkAc7H528R", + "jiIe2KuuSVMso+vKJ+AJU2KRaMt7pXDKX63LqF4vf7qGKNoaU7Yl9TL0gvXoDnxza4/vczangrOYMIVA", + "XRpHRJb3ystXJ89Hz1++6xzqgyBMA3ujff7q9ZvOYWd3MBh0fAyqOWiFDHxx/tZEzMG2IThSs1EwI8HV", + "qg9/hneP4VXYcSqJ0inYuPXVO8pIg2ISc2EuSew3aGNWVlLMlkewrsPO7osfDV9uvwCWdOtpQ5OyVkzD", + "lWiyFz/6GG22SIiYU+m7Sf05e+aYph4UX9oW1hx3/J67Eq0TI4h4GvYKXXY7EypIAEHI+l9/kFgbafMP", + "5Wgoz3f+C85WxskKqwNHCWVkidnxhWiX11xcRRyH4Fj4nMqljePzBJCbB+X1zawhxxK1vIwxZuE1DdVs", + "FPJrpofsEcn2CcpezuTyjZ4Jjv79z3+9O8tt6O0X48QK6e2d/U8U0hWxrJv23pJlE0kT/zTeJv5JvDv7", + "9z//5WbysJMwOsyt9EG7/s9NC9XQcptxY/zoDQGJ2cGfR9Zw6yyBz5HjvZXRhz4Zz+dERHhRELx2TJ3t", + "AUi/yqgEhVwiZL8reB2Xi2HdmtMPXlQdODsDv6D95qz4wp0VgoDoHSVZltuyDfLavJ3bkR6m8/Dcj/os", + "sOd+G07LGG1758z+uVNnOT/HySuajMCgGuFpFrWxLDHu4oom1kqDL8w2jSIj6MMU7Lox56o/ZCbwXe9N", + "4BxyQwLgBamwQkfnpxJd0ygC7y4cGnXVQdt8hYwJeF0q/b8iZV00TpU25LgiyJrUNgJPjwVeHhOUMuzC", + "bCtmlZ1gPWoZyHJFBCPRyJhNsiVlzEfIftRIHJjqBEubqCNUmpTpdfLL2QXaOFkwHNMA/WJaPeNhGhF0", + "YYKWN8vU6w5ZIiD6WXei2ZHafvkE8VT1+KSnBCFuiDE0lvnHbQzo/MX5WxtFLDf7Q/aaaMISFtp0R6dR", + "2FS4kLPvtEQmYbnZYv8VojdFikuGEznjbTfXhX09313t/UzdzjxI0vKS7nQbU+HmVKgUR/osLVka3rhh", + "k8rrsShNpnDRsrXnWp46qMqxlW2dcaZlyOv1Jg16fGpGE27tUyt4eWreNeeC+LPdYFe0f8rcQJb6FHMv", + "xCf0dWEaqeUEmJ+7bma3oNJpRpOKJ/LzkOdIFrw2rVJwzclujj+JNi5xQvuWj/sBjy+76PKvpR/03nem", + "o9Yfr5GhBsgTpn8qtl/1V630JK2V9FpcHCxvvx5HsjGBAs23kRKYSZP6MsMJ6aOfQYgjRWKtmkDGmURZ", + "xghi/Pp7xI3S6j4dMj00iTa0rLTkyPyJkk4ZZdNNbcbpgwmHoXE6TlKVCv3enMqcmmXWcY69mmZoRmfj", + "zyFPnLIgSkOCLp3z77KsFNVdg3WT3/oKaxasIQlYrmDMq604Vbp7PeFM0eSpMvkodurlXKGKA3JVSKQd", + "Sxadc4v1v8jERRUOYO4xYfXk7AUteIwLrusmD7FVVPze6yuygCV3nmpc81UXndR+V7IgkkdzYo/dopt7", + "DIAH3ChOuYfb+Kqte1pv/2qqv89xu2opNL1ak79sCnqADqTqucnmHGOtO5dq6qSQnpzpr6uVf0mA+GBa", + "HiJQxy67xhYmYHkgppklQiEVJFC15imbDhlEgV/aX/q2tUu9ybWO8lngIyAbG5T24tKiwso6tQ+a0VPj", + "MVWKhN2ybnBFSCJXT0qr1/ZOw3PxIsi1oE6QuVzFluoZYRMuAhJbI+HTHAPPC40tCSpp20Q9KNvQtzBm", + "l6WObawOrL1ZD/DAl5LXqxguYcVqM0HA5S4vcRRdog370iYS5HdI8LVrxTjLmf3N8bljgSxk5d1ZV3Ok", + "lgKXM6WSkf4fOdK7+LLamP3W7fAcX+PpAOyrvb1du6rWqWoGXGm27D/1BjY3L815KmeNF66Vy8ZlvKLb", + "OS683uyldUAmxlPrTqAklTO0ofBUbx0bwa7Pic1P9eBSJkmQCrJ6rx+BemSjqGFMEaas9/ObN+dIkCnV", + "bRZj0IvZseDWaUi3sYZtHjDjpqt42Y23s7u3f/Dk6bN+eCX6JBD9VPYIlqq33ccx/sAZvpZa7dqKFzhJ", + "DufbK+W8WYJsfM0i31lhzazAYy2SXcJAG4vuOP8kv2u5oixs28Av+t3VgXPO4LxrP3wiSC9NpgJDAufn", + "9MLfOrICqNm8qitArZYHRaZS8biYzb2xPBayTKw5j3ohVnidUEgz3HrUY7wwTRmT3Ov+agoxvajGlZZC", + "sL3QNp8a5uLG4luWpvRw40gg4UhxT9az0x1OTwBNwr7bKg8WstkVH80nlC8HD7AhjCUwGqOVWPeGbqKX", + "BNR6lUDVDWYmVdAQAWyHd2fF2/3+kPVACztEJ1kHWbNZkxhMDByaC9INLgqDMDABaLzYRBi9O+ujN9lo", + "v5NI261z4vLlZ1iiMSEMpXDDBEpRz6hkxQGkEnQnVf3cutBMav0mBDFw+6yfAXCBsy6DE4No1zGtzMdA", + "3sBC2bNEH3i5M7SV83JZoslrcx4tSzYpgYvs9wbbve39N9uDw4H+/3+0z0b5/OgBvraOyrLFxg8Xpc/x", + "29OTHesz37x1fslnxxfwi7KTPPAZbaSSiJ4Tk3Dl4Ql3LkQVN4Qz3zpK+Y6CjvNMymXvGkq80W/eBXCC", + "L7XD5l6uD21QFZgr82cLk6s7dBYJuB/yXVJQ5G2AekA9maRdCFr6URB8BYAw9XNba3VyZM4zf8RTKk0Y", + "HrmxTi7BuZpIo3SXnd/be0/2nu4e7D0dDDyp+XWG5wEdQQpCqwG8Oj5FEV5kWWobcK8donHEx2VG3989", "ePpk8Gx7p+04zE1uOzpk9rf7Cm1YivzNgXa6J6VB7ew8Odjd3R0cHOzstRqVvTZoNSh3xVBSSZ7sPtnb", - "frqz14oKPrvuhYNKqCrwPtye5wbqTv+rJxMS0AkNEKQYIf0B2ojhCCPZpWV5T45x6NCN/GeHwjSSSwOj", - "TGf2TeNvjdNI0SQi5hksSKsrCZj5MbTkhYtkLEOTWq8lCzCxMhDIzSV7BdSZkIzT6dRmb2WkOzVwWQXl", - "iZIoPESZybZUzsFq5gP70MQHdg4tueFXbTr1IjInUZEJzNFlcJsEQRmfmEUrzcrFAFCWpF6WaCTlT6kA", - "XdQ0ivCYp8rcNlv4r7yTE+M5gNvXlIXt3B0/cXG5MsJcn8QZyllLh8HEeuzgFMdZZrHNNS8ofdmtsLk7", - "t88lemO+MI7C/OckLUO8dqEn61BkSBCpOEhS6ze2zbTVLv16C/jMXRiJ6S+XnfcU4tabmKig27WwxZQA", - "up9aqbFoTjFxOufweuuEFf3hSkdKC7ozcnUfRIeUq55m255kOLkbii+LOc18DflLcAoLGpI+gt0FwW8O", - "6KWy084VTxISZv6f/pDZjI/sJ2ku0vSHhg5qRqhAXNApLXdc9rPeZfDqOqzouOnG7Fj8sCkYTWb3fr5N", - "b8ACJiYW1GBYFXOV7SJ0up3zDPfQSqIyad5k2JE1iuTB2LUhvjx7t24IaiL4hPrAdyEkxj61lpkLzvx1", - "b3De2/4fE2it+Q1UNMpMGE3MwwpMoX2/3cnz8uzdWdOYMuBAVBxdbU5Z4NMypGdHEXu3aC+nrQXj2F8f", - "LFknue79zKfLTgSOyTidTIgYxR7n2k/6OTIvZBncpz+W9VmtN7e1ms9KiwNm8wQHlE03W1Pf45CrTKNb", - "oOYH/3K9sUB0TTn8eqkysDqTvt9HrzKoRvTy7J1EebCax1NXSZNvyqg5my0kDXBkWjQ4S5QVHWzAnK01", - "5LP8Q+uK9OjJfoRPtxHQxnyapLANz9/0Tl6/34pDMu+WxgQBZjMeET3uzYK0mLvE7Dz9pyQk5k2eDsMY", - "su0GKtAq28GtiVTYrx7qKK5wNJIR98XsvNUPETxEG+9/MnmZegRdlJSWUv9eoEKJvw+8OwYwURq6PYcO", - "qy7T0gb32o7l0hDGvVKYXqlT31YxyTB1HacOr8svywvNL1dDuppGmvs9cvk6Fae2VSSRSetBkNbjwhCQ", - "+dR4ra1rRJIEC6xItDCaRXb0lSE6SP1GmVyTYI2EoRf69U8GlikVZKRmgsgZj8pxCLvdOrS3hFjYObFo", - "hmZOBce74ijG4hIORqdIo5QZCpRDqXdXQWXMlErWmNTPb9+eGetaETHHUTUYX9Zu2I9JhBdoTNQVIcxN", - "BUuE0UuewUZWs+dkA6KcUKOECMrLNOzsevo9N/G5aCpwQJD5yhUGsEsi4dRsS0rbiweuOQiIlA3ru71s", - "fe2nkzRqt8a+YW2vrKMRrLPAb4/OHDJahjzvyLxTp/IZET2z5RwE/fKl3ZHLMfpcV4wzUu9M8DEB0D6b", - "AVFM5XPxRQBKqD8vpc8VhIMs5q3ZfmAXGFJ1zT7/0ErZq25330V6jFnoQ/A3ge8mR3maxnpB9JBFCndH", + "frqz14oKPrvuuYNKqCrwPtyeIwN1p//VkwkJ6IQGCFKMkP4AbcRwhJHs0rK8J8c4dOhG/rNDYRrJpYFR", + "pjP7pvG3xmmkaBIR8wwWpNWVBMz8BFrywkUylqFJrdeSBZhYGQjk5pK9AupMSMbpdGqztzLSnRm4rILy", + "REkUHqLMZFsq52A184G9b+IDO4eW3PCrNp16EZmTqMgE5ugyuE2CoIxPzKKVZuViAChLUi9LNJLyp1SA", + "LmoaRXjMU2Vumy38V97JqfEcwO1rysJ27o6fuLhaGWGuT+IM5aylw2BiPXZwiuMss9jmmheUvuxW2Nyd", + "2+cSvTZfGEdh/nOSliFeu9CTdSgyJIhUHCSp9RvbZtpql369BXzmLozE9JfLznsKcetNTFTQ57WwxZQA", + "up9aqbFoTjFxOhfweuuEFf3hSkdKC7ozcn0fRIeUq55m255kOLkbii+LOc18DflLcAoLGpI+gt0FwW8O", + "6KWy0y4UTxISZv6f/pDZjI/sJ2ku0vSHhg5qRqhAXNApLXdc9rPeZfDqOqzouOnW7Fj8sCkYTWb3fr5N", + "b8ACJiYW1GBYFXOV7SJ0up2LDPfQSqIyaV5n2JE1iuTB2LUhvjh/u24IaiL4hPrAdyEkxj61lpkLzvx1", + "b3DR2/4fE2it+Q1UNMpMGE3MwwpMoX2/3cnz4vztedOYMuBAVBxdbU5Z4NMypGdHEXu3aC+nrQXj2F8f", + "LFknue79zKfLTgSOyTidTIgYxR7n2k/6OTIvZBncZz+W9VmtN7e1ms9LiwNm8wQHlE03W1Pf45CrTKNb", + "oOZ7/3K9tkB0TTn8eqkysDqTvt9HLzOoRvTi/K1EebCax1NXSZNvyqg5ny0kDXBkWjQ4S5QVHWzAnK01", + "5PP8Q+uK9OjJfoRPtxHQxnyapLANL173Tl+924pDMu+WxgQBZjMeET3uzYK0mLvE7Dz9pyQk5k2eDsMY", + "su0GKtAq28GtiVTYrx7qKK5wNJIR98XsvNEPETxEG+9+MnmZegRdlJSWUv9eoEKJvw+8OwYwURq6vYAO", + "qy7T0gb32o7l0hDGvVKYXqlT31YxyTB1HacOr8uvygvNr1ZDuppGmvs9dvk6Fae2VSSRSetBkNbjwhCQ", + "+dR4ra1rRJIEC6xItDCaRXb0lSE6SP1GmdyQYI2Eoef69Y8GlikVZKRmgsgZj8pxCLvdOrS3hFjYObFo", + "hmZOBce74ijG4goORqdIo5QZCpRDqXdXQWXMlErWmNTPb96cG+taETHHUTUYX9Zu2E9IhBdoTNQ1IcxN", + "BUuE0QuewUZWs+dkA6KcUKOECMrLNOzsevq9MPG5aCpwQJD5yhUGsEsi4dRsS0rbiweuOQiIlA3ru71s", + "fe2nkzRqt8a+YW2vrKMRrLPAb47PHTJahjzvyLxTp/I5ET2z5RwE/fKl3ZHLMfpcV4wzUu9M8DEB0D6b", + "AVFM5XPxRQBKqD8vpc8VhIMs5q3ZfmAXGFJ1zT5/30rZq25330V6jFnoQ/A3ge8mR3maxnpB9JBFCndH", "NDTBRyZt2qjlxfh9QXBIGZGykuAZpCLqdDu9iZ3V4dZWxAMczbhUh3u720+3lodxLo3fteFKo5Aus+9c", - "UJMJe3F5kAZJHyZdZoktnCQtPGCGjivOBxBP9XhBQO3XZ1tBw3MB5INBLVf1GgfK4YyCS6x05YmL2zbR", - "LFmaDzSYVbrZf/asuD8H3jvovNCUY/+tGu9DXArEFGoeMe6GCh0Nk3/0alRc+MAkuVA2E21MXNBhdh66", + "UJMJe3F5kAZJHyZdZoktnCQtPGCGjivOBxBP9XhBQO3XZ1tBw3MB5INBLVf1BgfK4YyCS6x05YmL2zbR", + "LFmaDzSYVbrZf/asuD8H3jvovNCUY/+tGu9DXArEFGoeMe6GCh0Nk3/walRc+MAkuVA2E21MXNBhdh66", "kD57qVLq7OngaXGWrer5gLCpbHO77zxTNW+Xs1NJgdx2/9oGIDyqrHO4Lb0cD1vTZQVPaYlYY6nlBOUJ", - "YWvRc39vd2c9eradyIkLy6rIJR82xtHpsdGJAs4UpowIFBOFbfGygpABX5KWMoBWhkkM6SqT75eLlob4", - "hSLYxU2hFm/r9rsBdfWNCeMNUYwZnWiBbN8s9ixneGf/4NBgQ4dksrd/0O/314UAeJHn/Ldaii0TJFdA", - "A+jL2eetwx1k+reZy5+ds+dvf9aCLJXCHFpbckzZYeHf2T/zB/CH+eeYMi9CQCs4cTqpwYiX48G0wW9+", - "PyxUz3J6T6uSM35nMEQHAyyJF2+rFCV5d8BaGY+bAdwkZ3VJDqdWV15DocMG2twYGTyvmaEKiODFbK8W", - "6OD04/L7befugndsn1AFKwdOr99s3wj6Xi5FB66BRyaEZXjAUWT+Cjib6+3qAwcuHZHuWQtsSThGLIhk", - "1mXxx6z3wo9HxYEUfnfYlIWfLErwhzVDUpYqpH+v66GrpZBTR1dsZr/vMTsV2iKyW0Aubyz2A5+FNwnb", - "Kvf+evrff/w/8uzJ79t//Pr+/f/OX/738Sv6v++js9efhb2xHIvtQQHVbg1DzVQZKQKptWWlU6wCj49O", - "m38NFLZPjMdBBTPAkEZjcjhkPfQrVUTg6BANO5Xkw2EHbRCwlOArre7qpmwO9ab++MzcKOqP/3Rq8Kdq", - "G6FNlhZ2QTIMDJmOQx5jyjaHbMhsW8hNRIJdoP8KUYATBe4cypC2fxdoLKBsqr3xyTvvoj9xknzaHDJb", - "90UJPYMEC5Whe7oegCnsqEwYqn2dZHAI5tJlyLLTOgMjM9d+/Uz5h3CPai6XnyjL7TdrOT0d+GDbIJ9E", - "L2QEmOEou0CjEhg9S3RBTwebdXtuhY2R8dAS9oOdUK+n7JiyxV4yDAxdG8E9ci7KFeEZWjaZPYLAglQc", - "/nuOXEM5LbIlNvcGJrtImmtnFclCXtHmigSQFhMy967wWdQCj+CFSTx7++s5UkTELhV4I9DknNBAzw8i", - "SqmUqWZFitHzo9MXm/0WBaGBtksTRMqD9qSN23vYpuvl3NzFMemik2NI/LM7NFdrIVL7Jy5QZARMvq8P", - "0TsLAV1qCplAUbOS0SK/zDUnwLCz6VpMqpLiEL3JtGmcDaVUhLp8P5zvS2jWxvKYMPJa691aeVfhNGkr", - "2iBoHKss/VSfuM2ioL37xqHL6D1fcTasvbeL9+uNroTC2t82VOftqzu766k7rmRfMsPSx92z4gURvLSk", - "rC+teL1FczREqd+VpWqz7qBYMMlqXJjPfUVG93vb22+399b3hKyLslhGuykgYWVAi+0REu8CabDuFbim", - "atQYJ4v0YxsV60zM96dohiX7TsHDiqG5vfukVWkz3WvbCNNibCmfmCFlUspB52SRkQZE6JJGkQk4lnTK", - "cISeoY3zk5e/nPz66ybqodevT6tLsewL3/q0AFx0ouLl2Tu4ZMRy5IK0mvOScJ7bR66pVLKOLNQq1vFz", - "AB7Np+1KVLhJmjbyShXLUSJ/LiE5eqGiNm8R3tEFqNbIeB/AjQ+Z+fPlgUYuhXn8XKxGa7zcEVRjo3D3", - "wRxW0pz3m+T8zUEX72Q4K6vpF896l5Z5Y5TDbod6UtKeSy2CSYhOzvJKGLmH0zVfmdOznf72wdP+9mDQ", - "3x60K30ULOn79PlR+84HO0a1OMTjwyA8JJPPcIRbxjbKOI6u8EKioTOXhh1jnxUMs8K2tSZVq0v7Opjk", - "zbAjqwpNAzokKHYuGkLaGqpNyk2LnKMqElKcRiattVh3tawtzrBEMjGwfwbRPNNlhwwG2LXwQ3BU0Jgg", - "HAQizf0ZrgCW0XzTxPL9kAkiE860NqytGvQLWUgUU7hZybqHeCqJstD2cMg2hEuDyPIdEpxKEuofIMa4", - "62JZ9dCoAuR2/cGQyVkKtdM3++iIM5nGRFhXDxpTcENvIpka4w7GC9RYaIEpaUjEkOnXPMCCf2aK+uHB", - "YDAYZDXdO4e7+t8DHzfd6Y1K3wJnmkQ+gPBiFkITsMFEylDKQiLQGy6JUhgRww7VuJs1b2M+EzHTfd5O", - "j7Kf5wqUf2OuAthsh5z5uXCFywrJnpdLyLY2U/f/8VnVZklbRdVmFNivRutcixIU8DQKte0z1qedcU2R", - "0HrQJFF5dV44IN+xS6b3aGnqNp5RcfRHSsQCvT89Ld2lCjKxxUdbTBykRMM68GStZdhZ4S1YOZobglje", - "B3BlVVMpaIi3DlNZvLZwmY2GQ1tcX+QWozcanTKzNJpPlsyp4ngOyXyUpj5DRD9yeAbv3p0cl5gD44Pt", - "p4Onz3pPx9sHvb1wsN3D27sHvZ19PJjsBk92G8p/t89GuXmCidfs9NVzdmGmIxfu6otCbAo2rpz7NoDy", - "irKQX5WOFm9EW7F3Gy23qvt6LGzrIXgj6KG0KLTUICVO4QAlgW7bRJBWqv02uNMO3g62V7jTVsoLGFyD", - "/H0rUhYYvDmQxJmjOi4MuLhY5XHeTJzCgFyk+ipqFTtvT7TB4f6zw/3PJZqLtl41xio73ePiNoVwONDS", - "Sji3SykqOGUcyF/H6hvGd2yjvzvdThagDn/DQVsJfswet8q6aNqwXb8YWSa/G5IPT0q2AFwtG9Cq8FBr", - "AVne2jhVKMtp1erFUcTTEBUcWgbDB257Tgp2gW4GLl+sv8tg8pnoaW0/AAgsQJBTpgUx3HLpRmym4iF6", - "Ce/CIxwbk8kOwgDhFy94cLgwF9x6f7mujQGzfMjn1naBb7Qhg/S/YNqaDNbvubwJo/kcolccvsksKcar", - "DlTzOpgw9derztYNm0LoUs2hM6vGHaKfMtUtU/6ssrchif1zZAVWjvCwWcqztSve0dySr1whhbTbMRTt", - "dDuOUJBqWk86fZdzfW3/FVnRF/lBcGQqHmdJfamikcW1hZlQqWggbbS3Xtwm/cLW6CDhyBgmTUFkJlPM", - "Gi/ZR059eX+KNgC67G/Ieof1vzazgLPSWbfzbO/ZwZOdZwetAEryAa5WO48gj7E+uJU6aJCkI+t3aJr6", - "0dk741cIjMUO1w127oV88ERwLXr0zF2Dxc6f9Z8VcVlCno6jwvWVBXEyMJCwYF7ooUwWNQQu/UGjOZ1M", - "2B8fg8ud3wWNt68P5M54uwFv0nTkd2mdFK+wa/5fMu6Zeht+6AxgKCEb0WXeEAkzQOdEIeCfHsIBmA5Z", - "+qFlOYdBYynuZay93d3dp0/2d1rxlR1dYeOMwMHlOZTtCApbDN5EG2/Oz9FWgeFMmy4nGyCBmTUr/fsM", - "2eqMg7JC2t8e7Pq4pOHgzrnGtj2PG0n+3ppmdlKW6JBFmZlttV3upfbu7uDJ3v7T/Xbb2LpeR+J6uYRx", - "OQaGPBa5urjyG6BNvn1+hiCDb4KDst/EoZKuNSq11qgAdd2gJa8xsKdPDvb3dne228Ek+UIbLABYacOW", - "ZZdn03mYwrMaHlLURW+36bTwqVOGwd6QIMI0fh64mOjK6WNQkUfCvJYvQpuDwXr7awdXi29bOY4yd5CJ", - "qDeqARcoZRkWf3/1vebNrimbxbQ5D1aLcV/gPNPksngepvjODWiXCDKnPJW30BBXJqttEnEu1vq2yUJ5", - "Q2QaKXOXSCV6f/odCBHNXEgqkpSNJst+S1BPbji5tTZwiSf8XN1ErFar0Wbpl02427BNu8tS3kvbvxFc", - "KNSiKmWrYwqPcBSkUG4CZ+upZwUwIZC0myTRwkTfRhHnDAUzzODGQdjiNmyKMJrxKOx7IyL1k9HEG4vA", - "r1DEDSzqJSGJrcRgBqE/0zoLnRO0UUgaRoaVKhXz9mMjVSzWfpkb92N/6S8sfekkWbKqpidWvIDYaT4p", - "+RgjPpVgBSqIK+5XgaITLEy4MGamssg8NsZjGWVpR5/2niFWpLfvCDVHJ59Yi9bqGJAKaiiJA8GlRCSi", - "U6hi8f60kmG4JCslyzNcHSdYHmwL1jW3g56zC8402boAke9A9ETcf86RCDwMWT1LIvCcNzLGLIXaDAVG", - "JtcJFYY92kXZzbhUoww5Zs3BSjUCwPVUkBxeKsuLzRxA7h3vuehE203IZcNZb/R1jav8TTUNsFmmeinq", - "p1Y340EfG9exc5bC9eT4P1Wwl3XQnXKEbiqhVVoAFkIbjKuSWCqgTG+2iTjx26i6n5p5auvK/bo3OG8L", - "vLQcZ+kMq9kJm3BPdv4a15DO9WyDIBMiYgolB1BIGCWhMx6z+0jr24KUzkgSFKbEUs4opAJbgmOzvSHD", - "njmnGGXTiqyvdtjGH2zGsByPHfq1L7aJHZL+lLe3IgVamWg/iXCe/NYqdJLKkf/+qt6wINM0wgJVwcWW", - "DFku4oiyyzaty0U85hENkP6gesk84VHEr0b6kfwB5rLZanb6g1FT8Y5zMzibuWMWpNJvPoUf9Cw3K3mD", - "4HrZMt9vAaZDm1AsbwDyTzQiFn/rHaPXBUYvAxbv7Qya8lkbGi1lstax29aV3JZlvTs+lZ5EtKVajis7", - "QkKLIm3UniSVsyzUoGJXmmebnVYOCwdp6C4Eb3a/U05W+DxkgSMjzCu4AmhMINcE5uaNVWwjM730MgJ0", - "s808vbDtqZyh3/m47DttG33rKQa0wfL0d0Em3oBz4IWlvmvzRktGqfPFOsnnIK01FWz+ua+vPA99nYTv", - "VQWX8gCpJhn2plZ7aEYssTMKmDpELXD1XQBHltRse22f3VytEtVU12JRqHYm0ZgLAVCwWfQ/kFvTuYss", - "2oVUi4gcmgjEAEcREeCpsK1FfEqZjajLbs+DiBKmvpPo/9vqm1a2TAR1/3fJ2eaQCRoSiTDYqg49NsvZ", - "0noWwSHwGp1DsUTDCsaB30evyJyIIUuIkFQqU/gh4tMpCb9H2MwAHJsiTazjEyMbJgd7Uha6GDJBlAB7", - "3BnoakbiPvp7wfrtFrr/TiJ+xTICDFmRnto2TaWrGFeBscFSXnERLlka9wpUsoCLYaT4JWFFiZs14zVv", - "TUMj81U9rM+UtYSnCDz+WPZoFv1RKjqJqkUll3edSiL8Om82u+yVVqE3hV1YsLXMbgcIBigmZv+Cn3LY", - "hdroHOzo86zOvyeKx6TaVGKJyhd1KyNhIJdrGbpF/aoCbbjoZ1cwoax3FAoXtDp426WTVeP83Wi2JAnK", - "ve893X9y0LJyxGfdBRo0n9u++ZvHS278GlbqtM210tP9p8+e7e7tP9tZ6wLHpYQ0rE9TWkhxfdAGuVZ6", - "L0X//ue/3p9WLpX2IXx6sNagTFKIf0gNiSHlAb0//fc//+VGdeMB+eRAHTG4ISygMQgoKq6ki0Mo3xC2", - "u4Nb4k14XnJJ4EzMoA0ymRBwuo4M3Xr5YCqwAO3UapzggKqFR87iKxOonr1SQb5tc9tUHqxPhzZtW5RE", - "LblkOs4zJzdc5+iv5uq5wgtPWxegkem46Zr7dbVXc8md35EUQyhaRDDkpdLr7vRsPldYlmKx9d8B6AYu", - "OayeKGPeWI7CWc1igCAZW2epEGnoQ2+uaJL2o+LyV5azcC1aciJVKf5hyT5s3oJr+Zg9J7LHxRysTn+t", - "yAd7AN7sq9G4WBpqae2tUh2p/NRdv98WGb513PTsBFu/v0Ku4zofVjFCgR/tGCzJ87a7JZZo4KZCGovH", - "Xccj0sviAJ3yLlNz/6j3vIWd9iRfBpd8MiljX+43YyUDSDHkablesFIkTlQXkWvnIakC7aKIXhI07OzL", - "YUcr68POdjzsVC7JvJmPMb4e2Q7KACWDZeDFWVHE6iClm8E44sGlqXoExXT7aIBigplEKYPNX7nD2x4s", - "v4vqdpLC2mRQwcREUNXEFoxpTGZ4TgGh3t7gTEtxnuSaKgnxqNDOIQo5+H/LJR/tDPVrJi/xMJ80HDqY", - "LWzDukH9HmcuYDZ/F3wFEyg0yT4Swbs2415L7NevT7smPgIiG83ASuGTbqJmBFpAZl1U4Nbz3/3hyeOI", - "jGDcVfjuuE7HYv442MaCSKKkxfPN2aHCBCjgKVNVXO+4nZ1VzgirH0kpg1hC63ABnCbbu60XHpIAdqSs", - "78Uyo9+AuStpCZbSvryEXR8Lw6aAmyv/zfQbe/1cHYAx5AvVWk07xbBxcyc5korb8j7Zrh6R64CQsAoA", - "6H+lbSi+/dIbiv8rtkA3WSFV+zaEU9dn17+73CwYaxO1iykDjLMeQGy4JbVwGAYqzAKulBmtBEVcgJ0Y", - "+eAWfS+0SZYm18tp/YpcK8BLDtNIk7eJda2osofRKorfOCmxaUNzsbru+B2UkTLh7DcqJGUj4e+llpT9", - "+U7qR9WW45wo9+655ZvmsuGlSg+lGzOXQOBeKYfwGN7pInuio+14s8JzezO/G8SicrbMp2Q4JqNEkAm9", - "XsIt5gVjCZchSPKdUyotL9FGjK/R3hMUzLCQlbEzOp2paFGO79nzIAB9VlU1QRRhao0i/Plqug/rwXR2", - "OYut+7Th8wJeT62mgdVBR8uAc4/yyzwbrpTgBbhtGu8gn+zuDQa7O4MbIee6Ya1BrqP8E1uSrNxOU4pe", - "4TsbR1CKei22kCVE1wtbXgkKedUZmaQSBMeHkMiT4ICgiEwA2i0rKLz6brLa9fLBWw3Kgsdk/O8Wyq6b", - "u+Iv18zIurKgw24aHXc9WcYLKj5fcaXaIGaCGhCcJ4dvtzc4eLu9e7h/cLi9fRdotxmRmrJHnnzcvnoS", - "7eDJXvR08eSP7dmT6U686zW8LqkpDdKGV3/R7zYG8eSnYhl3qCTS0IadQ0JEtWJptdKvJBFlpCezjKvV", - "aY9LZIG53l+5/9dz7JsZLFUWzsuTLOoMWOXEKXHWPYFj2dEvvZ2oDv/kePmwb5TCVB2In8GqQwF+ajcY", - "wKTf/tzq8ylree68K7zY+uRZmla36uzx3Z7D1vaucgPFffxcEoylHbbsxK6fah7v6JQLqmbx8uMhey0D", - "DoY47I9ShWUwpj46mTKoT1z8OQu7K5pJ+uNOtxN93CvvGft7e1guC5SbMaBd6qIa0CIsDcpfL6cCvJKb", - "FsJExmtrXI/5h+3e9jMIDo8+7v0w6D2rXtMDtYrk23Zvl34dtKFhseiXKxaz/WytCG5Hz2Uc9Av1lazK", - "D2ILoWt5PC8O684Kl7FbWuD8cW2NKzA7jRrn56p29jQbFbWkkETY4+0tuWJlxT4sMhkakyllso1ndneQ", - "uWb342Gnj55b4GmwVvPa36XmoepzgU9oHJOQaqXSGPfNGRE7Lb1tVeNhvWoE7iuPetb362fPVmMsrErg", - "WnVM9j8jofezzN12Ju4y+A/wnDmbFAC+4MUuohOEWaUkoS3AbzPxIbMSAtUOHYpazrJWBshc8XOekC6a", - "coXyHPyWHrWUNXv+svGTa/CoLgHdMAyxcyuIKhm6F10mvk6OUSJ4mAZ5AmoEg84hQ0RawU9botWvDvG9", - "S4cGZHZPuECrHRpNHox2Hsim9a54HzXDNi/19mD1Ut+JF6TbSZNwtQwzL7WTYGsBjK9IafT4ZMpkr2iC", - "hcl8aCHR3xQpWDdyjbc40CpRmrgrFM1TdU7yXKjAJYIv2veYREQfU/VGEI/CPOuCylyKrhap2wdPZ02X", - "mHDnVB/IL4Qk2lYBACXoL8Zs4R1Ytd482hi4IqLSXGn1TIESS63y4J6s1MQal6p96f6KV9sArlwWHNwZ", - "0vbt1u23X7rAtkaH8V344R5SSXttLxcqmKoO8TeLUnb9m4xSwGRmUeW83vNdwPvY4q21jJtwYqtZpUU3", - "8/PeP4xbGY36h1s//O3/7n34q9e9XLGbJRG9kEwglOiSLHpQ5AZpG71fRkkFgH6tTE8tqxAcg9MouCTG", - "SRXj6+J49weZ0Fi8wnFtChCDFVOW/XvlhP72l+YIpgIZ34GcXMmyn12/4i5KJirujqONmIiptgupSxKD", - "3Oghg1y3S7KQqFCByKo0jlG/k9knhTB7dGHUwD5h8ws0plDSTQ6ZtmpxEJBEWxO2Egs1dYk5SB9BcFRs", - "x1ZCcnHq9srRRAwQ9P60BrH7+t3bH1+/e3U8en324tXzk9EvL/4XgjiueqaHsKd5b2//wFYjLlJy27PE", - "n4H2/1kotz52M1iZHv6CpE0o8exRmKkEyAUXUFB4GW2QOFELV97Q5X5urofd+Txr0BvOdsulVwbPbqPS", - "3LulpeXmPOppjboBOd/rwDS08IZjQ1MmzL3T5Niejj1qo/UmTukUe3zZ3mLst1ERzg1oZQJObf0b6zn5", - "g+OPq2UFjDQwpKrA4FfsUql6zbHzsVakRnnZ63JERsps+iUtBGyVcy1jprZs5UYf5EPIAdx6WcJtvssc", - "ol8PPlqdR7pUlS/MrDCS5rU5dRprRadeQqAzTZqrGRGksBDwQQ7XvibJbLJHCyARU18tISIPhHSZIloR", - "gitNiTYyZ4MjQZYwW/fALofjP8XXWQ/gvceydscF88gL42y//BGg09+4qu104pqAYVTsCT9QeJmLltHE", - "cVV9MYpcVZ+3ed+78aysWiL9mvZWhTnzPkqs6ePHv2OqfuICLJBm2I47xxsH6yYkAnDLqmjiraC4aUzC", - "EU/V8v1vyzZbzI7QGRF57UZnbWFg4qCUiNskCxywRD6GOqU1OUiQCqoW51Dj3kQIQxbc89RseFcq3/6c", - "dwylED99Aj/lxJOF8JIwImiAnp+dwH6MMQMlHb0/LZQvM5XsahijoF6+PjqxFq6DqQWLhSpgPRfM9/zs", - "pNPtzIkwVl5n0N/tD2AzJ4ThhHYOO7v97f6gA4r8DKa4BWWbbSKvTV/NbKWT0GpCP7qX9JcCx0TBF795", - "ktkhmM2+DlovnhbslgRTYQ2XJIIke8MwVH8N8PPuQD00p3LXkL21mw5yUCGlgiSv7eJ+AKUS9g5Mc2cw", - "sGDcyh6/kBBiotC3frchiXm/rbQ6SyIPGnvNsnC6ZUb6T93O3mB7rTEtGwrsXV/H7xi2iZsEDML9NQlx", - "o05PmMn1slmxNuSmuOOAkYp77bcPes1kGsdYLBzBitRKuGxSjIlE2L1rMIKVRIEWFVAzpo9eM2JrlWOF", - "sAmHFSmTEH9hP9QcWt4Fpm23yBnOzo88XNwaCUt9OLP4U1mc6e3yqcbPt8c7GRvXF9I+cqjQhmvvgYF+", - "xKHL+H6wnbI3eHb3nR5xNolooFAvY2Ab5EolRJlEgHHt4HO4QH+kXGGUxYg/oi1tddZxxm7d/Cja+pOG", - "n8z2jojP83pGRIyZibg376zY9LXtbLzg+XZeeqo5xodyFHBSORwZc1CBIlfeosVjq6oM1o+jPU8Cvu3T", - "TC98QMbfu4cdbiebFc18yC1nUA5SSR7TdrK3OuNcCfHqci+J+lJ4fnCfR5YFvv8Kd9FjYeCXJNPw8tWq", - "HQpbiUiZMYC9GuCbPAvOfvddWfl7mz8pBGaAK103DSUYlLnKw+GijxxNjdGvFoDYIwjMM6wfK2d6eF/K", - "Dtu5jx0GM84uJ74dU9+OqWW73HCLmwJszMIub+GDWMsD8fX5H9b2PnzzPbT3PbTyPDByZb0Lv/NxH9kg", - "SKhYL2c8jUI0JsiA6LhwB4VFf/oRYRHM6JwMmb0tiNNI0QSQyLiIUYgVNte2jY6JpW6JrLkt3VzPhb7l", - "BK6CI0gyArC5URNKYh70RhkjIdKfWFy7HJ2uVl7a7H2vgz1rMD8a0dWMS7jaAAA3pgqnOeTMSmMdQ7P9", - "IXtrsUo1ASF+18kaSSJAXF3i/+EM4SGzH3zvRIiLPZI4ziUXFgDyRg1EolmWevqUHulIBtwH4PKWMMxU", - "TyYkoBMa2GldkoUNIfQ22KpWkB6wG+f70yxHAO34URYN6p4fX/Y4e4YsJ5XvbxjE3QZRGuaXXA6XBosx", - "jiJvMYlpxMc4Ghn6XBLPneBLeMMSpViG3t0mMR4SU1I8WagZZ+bvdJwylZq/x4JfSSKGnc3+kEHsv6W1", - "A+kzPHAFxcfihOt9Jnhs+twyQ9z685IsPvWH7HkYU+Y4Aj7BkeSIXMN3UJMJgBiM9GrgB7Ob/PfgR6lU", - "PC5CcDq+M8PkqUpSZZMYJFFdH4jkkCmO/nTQfp+2/sx7/FTESiy8YqYEunXTqOUI69mP4FXPdTsBAgw7", - "+iAddvTfU4GZMuiKGTYhmhaXdCMD+NebdLNK4QAzlPDEFEcAppphzXKlNgAAAEcRUrCV3LdacYeVbJiP", - "xXOLx41gbgZ9q7KNKEOnPxY202DvqX8/SRII4oso+e/z168QnMp6DcxreYSQySJgWmFAYQpXp06mvcDB", - "DJmLKiiAN+zQcNjJrnPDTRhrKm1KfK8Hd4o/6KH9YLrp0vCHfl83Za4rD9Fvf5pWDvVeSmIDAznsfOqi", - "woMpVbN0nD374CdoEybWeUkQoA1zzG2CJMEU4EsKJ745IjELEbenQLRAGOUSqBi4MqYMi8Wy3DUP6S0F", - "+cQEzxWI8ecQguWGncOhC5cbdrrDDmFz+M3G1A07n/wUsLeWzdXW4DzLLjczJjoYDDZXgzlb+nruLFtc", - "DNyyDdhoFWWlIvUKWujNr+t+4D/a/syufjDTned4N8bwd873R3gBUdDYi5ao5wqionZjFpDIqd2rHT33", - "f3mgFysgUXTfDPpQ7Jldj2V48o+KHWGx8m201H3/wBw3uK9DpeS2fxj+fXT+c4/33PrOydyFOvtLbQDO", - "iTWlkXkZYYnOYUy9c218v4Bf+/a/zvYDoL6LiE8vDo3pjiI+RRFlNgS9EKis1QNLS/jIQJ1k31nkE1fn", - "bMNoEv/+579gUJRN//3Pf1k873//81+w3bds7QRobkawUGOC1cUh+oWQpIcjOiduMoBjTuZELNDuwPr8", - "4REqlGe3WpocsiF7Q1QqWCFU35Qck7ZBe1Wg50NZSqSFitEv0omth2JiGz1+G7eXDSnvdUd3PRh7MIPC", - "BPSp6HgAAMqoKQ5tLdGO32Vq5lxymlbDNGvBeqvliyLXynBvzwxwTQEDJPbtO3hgJ402zs9fbPYRWFuG", - "K6DmDdgOeTPWjOh/k0mrZZKRKGWBAlQ2ssngJy13+h/bd9p5/W2LX5Pb35ZBW8Pvb5w/AJvoVuDbHUCL", - "OwA/3dx9gM8pf+wAwu4uWNB08UCxgo736jQ3TwokewhnANpwQAzgUOUCnR2dIByGgki5+Z/tKtAzNVya", - "Hx2IM4D9f4hbazsWLixElTXVygzyWMTBGztqhN28qtUli+fbVqkYRONJl9WFyI+8uz89Kp2uc4zkFTBz", - "Xvt2kqyM06My4PrbArf0ApwAIZ36ku3TIhetckiZCMDsyFmqLlnxfHLsNuT9uaZs1ymrng33IBSPKwLx", - "AQVhOUuzWDP2MXHzu2wVHRjqEs/Vl8Wag/vTgu7bi+Vj88fkxgorZNNS0OAJNB6gL4kyKAKdO1xo24Nn", - "4udEuF3tSnzDrLNpmU+RgUOACcHV/HLb98S80s70Ne19TZYvkGcdjcWS/JuK0sLYzWm1zMA9scVI786+", - "hR7WMm9v78bbMpiHyBB2M3Yea6FIiDawXLBg89ul961ztAmJyo1Y4eZNQpREWEF0JACxZHaWHtvOPeh1", - "WcFSgRWxcUOPMRnvLI0idzUzJ0Kh10cnRgQUD6utPyGSbLUR4sTC0nPr3Ztfe4QFHEIHs7A3v7Znn9yy", - "KWI4q5Rgd//8/AiTzKg7eJtUsc9Yf1sf2gSl9in/r52fIjoWWCz+a+cnHCWUkf/afR5hRaTavDNmGdzX", - "GXLfpsEjZj5tGdAy0UA0sSlAB65QpbO3WmrT7v2vSqE2k15Lpc7o+k2rbqNVF8m1VLG2S3GnqrXp44Hu", - "jjJm81EbHn3DmbgHd6TlyALOROl+JkeamHGp4NHjSzq0kZ4047jisdHSr55vyKXHh2Pdk+MuEBJqhwKy", - "uc3puScvuxvHvSu3tt/7d7E/j8d0mvJUFtOFYqyCGZE2lS4iZQH82NTu/HhuVLy/YC4d3OfRce969Te+", - "vyONv7qgRnibq7JVOr97q63Ob9/XOr+BGbTphhZ+vetKc2w2RD86oMG2bFzCY6xHZfrG5bNF0DttqOTm", - "AgIL4nDI/o+2P35TBMcffnB5TelgsHMAvxM2//CDS21ip45VCIOq4JDi+vzVMdxPTgGhEYot5VmU1XGY", - "6qzAeg5e+j/OQMqvaNtbSI4Lv1lIrSykArmWW0h2Le7WRCpD1N+7jeT4zUdwC/T7dVpJX/DFw71bcDKd", - "TGhACQOgf8gWlbVIO2PJfbsZuWGWILM3fYUwnZIm0tqMzKTWCg09ry167yFaJ3kxlfu2Hl0Z08eZ7cAT", - "WxfQ2mu5ttBssH1p/DC439Pr/g21x8xixiKqky7RSrenXIcpVBOnCsJLc4AfiN9Fwpg1WYt9dJTldcs0", - "SbhQ0hS7AQvBlMOcaQvBVxinXOvGV9wGCrpQIrtDBuVO9WODT7F1SRamlA3lLKtak83UVoTxZdGVSwk9", - "6Da6fSXUXyeplRJ6z9vYVr57OCX0wUTHvah7J6WCohvZxgCLe0yyncyzNE36kbLp5qOKJTbCKptbAY7M", - "o2pt4VRxVwd/a8YNMpEfnO0swgFgs+nXDGyQzfs1OGHFpiCZV/AoIsLAQSWpcnWzhiwbHGWFusC2SMWF", - "bn6UMkWji66JpoGcfokwW1hMlCErdYaVInGiBZtF+YERCpKYEVcKhulBU55KeKuLJC91iXB0hRdyyASZ", - "RCSwc4PiioIEBjktivroZw6J1AhPMWU2t1e/acptfSeH7IKGERnZPOgLRCWSMy4UYSREMZ8TWe6XYBFR", - "ImASR1hTTqIYLwCQyGCzGfrwhBjQn1K2Ndf/xiykUIxK95xN+XDIMNoZDFBMMJOIQkKuxBOiv7JtIBhE", - "aUDfI4z2Bs/sV5V1A9BMR/4NvV+EIHMe4HG0QERzMZyIahMWMNtfUNdRL9+ECmnWK3Mv2qo/pYWl0tWn", - "DLsoZQGUT0yF/hcXKGX2eNUtCsgxh3naSzhCRVZ2zCbEj0mANT0ZL/cDUGQ8CFLhOxz1UhcK4/0nKpmF", - "6Z0DqXzSSdMBwZ4KYc0ZVzPY0xy20ub3DVyVM9XXcch4NwkXCKMCX+cOBaghzaZoA6C7LvKSW8xVbbzY", - "/N7tHb19rSBw29+AZz2W8wmYiE8mpQ24+mgyG3hZ4kKdhb/WfXrkai0WRVxI8ZRxqWjghGG1GvA347G1", - "8bicsl5unnBxWdStyvz7ExeXba2vc1fi/lEZYcUZfoH3AHp4AL768NcB4Iw2hopmmns30Kr8le1SULqo", - "ki7OmKOIs6neRblT/N699hWLLohSUMudKeecINoIGdkfTblGPRlbDA88/IFt9aFlke79Hu6CXnGFaJxE", - "JCZQzrFnmE0vdqZVm2rLVKJZVkdvPVmpd1UxKdfYgtJc/3edOgR85RZsA7T3+nJ5hWrEp6uBuLLOHeqU", - "B4lryEw1aOJKR1+gTAZrhdbAXqOrGQ1mgMoFdqtu34B24SS5yABJNw/RS9jIRVxW6HzDgF1rXpM8IgZs", - "ax7HF4f1goXvT0/hIwPIZUoTXhwiV6QwOz+kfquIsqVnEWGp0CuLHbaRGeOwohcKa3szm9+mxd/KAWOH", - "zIfFxciVbZBO0EUBluuiAZfLydtf+fTBlLFuM8y3mYviyJqOwJuEhZ2mGAsa+RG5tgcDH/psS3QwM4w7", - "BgerDeZXPs0gxkusjJOkLfvaYQIXz+N4CQ+jjVyCIKlCnqq/SRUSIeBjy91NzI02cGDLy+BLzajMSCW3", - "sTeB/byRRAbz10sqLVQ73Q5hadw5/M3+ax7HnW7HjqeAFbyGcr8CZa3aYD3iRa9MAUrtm1q+DkhaWdgX", - "UNIqJ4c1p5s18jfmha/+ZtH57B6QDUE/qDhxvyQVtDDessOHcSQZTuSMq8eFy2RdTRWtrdlV42bZ08ML", - "U1cDo00Ix7n99Nx9+QVYv6siO9yYkZvuvYd41EfwmDNhZW02Ey6qYD6rYj++eEa6vSWpTbUNh3zjzfX9", - "fK0YM/FV43dLE5qaSDhVPMaKBlCPI5hxLgtsPyYzPKfcXpW6O6uMM8G5YexMG0J/oVn1wjqCL6wif2id", - "VggXH9k++vC5Dbz3f+Ee5V/8VLDLM4nfdco3YFZDwWBByQQlOJVE61VpTJApxW8LsBAczFCAE5UKArWl", - "CIopo3EaF1wN2nAScxwhKtHFdnzRReNUoQiLKdhF5qEJpxck4HFMWEjAQzZkM4LnVBt1AkVYERYsepJA", - "Tco5ySv9ayPfRuGYmlaCaA6knHVRTBQOscKgalzoHT8yWTwXWZlKY1gzcp1zQzhkImXfG5xt3eyFG+gF", - "IlLhcUTlLCtnFuCQsMALYn3+ZYux2/cGnxNVnegDxeXcSJY+ZKBO0evphvNlxPA8smBkLuwythHzS5Re", - "2WxEltMfHBv9Z25pM1c3xwe64slIvGwXfxl3OxnTfTH3Ow9/gcMFClPTXWFXApt/rbcymUAphjtBaqVZ", - "xptezWR1mzIyryXztv50f57cwJv2hUjCbqNh31QhJJ/0lyByLVVvJHMfyI1ofUkFr9gDimAXU/Vg6hMX", - "BSn3WNydVmCbrZnJ7aJ0UgKD9cXZN7FdFds25OCmYtv5ZmuX6gVBTlkPojT9Ety6cRtFtXUd/IfmglRm", - "VxCZDy4i87uDexOLJ5kgNKIxwYuI4/BrCNNdcoMTcCEM/gMgSjwm/NGC17AYoA++uW4mIbout/L96elm", - "k5QQaqmMEOoRS4hCUoz+LPYV0Z8TIWjoioMfnR7bgFkqkUhZH72OKVTsviQkyXNKAMijr+fnkDDqZY5L", - "kBfdDmFKLBJOmVo5ivzVuxnMpxsVR75nOWmhor9dSLe+kAbP/uMTZyBlIGvCTGC5ZaqwagwFdKFxlJna", - "51ovw2Oe6ta1DNJk0us5hVNwQiMiF1KR2MQFTtIIthuUHbBVKe13ZpW7EBWrd45JWEuIiKmUlDM5ZDZb", - "IyFC960/1+0XQpy8FwIKZ/L1zAjJLyN8Tg/GRIxh1UQ1wCyCmvCdw84WTpKtECvcEKJlh/cZQ/oJ4uGQ", - "XMRjHtEARZRdSrQR0UtjnqC5RJH+Y3NpQN0Ivrvtmps331ma0idswr1lyQzPZsz8VeVVWbHmLiYfnVh7", - "SYqbxckfWGi/WJMr5ZogOOopGpMMuQalikb0oxF1uhEqFQ1M0k8OWfD+NEctGLJTooR+B0NyWRSRQDmH", - "zVYieLA1TAeD3SChAH+2S2BwIPCaH8fQ49HZO5MISmIuFt0h0/+Aht8+PzO3uxNsvQmFgTKirri4RCdb", - "r1eEGJ8Dmf6DY/TMBJdiB3gX/NuV4PqIII17SDZsUZ4sM5V48tUHkVoN7ptf4XH6FQCSKZvNxlTgAJRi", - "OUtVyK+Y34cw51Ea63+YP05WAXspHMzew6tfjLZrhrOyGzfBR7Ep7ZxCYsomPsilhyHYY41Z1YRzUwAl", - "phQN6D0Fnquvkbtv331fpOMXeN1pKepKkn4xe+u+Tz47BodxUaTHY9nmhtPcTBRf7n26wrTZ+/RjxINL", - "acFQim5DbbcBwLj+MQeEtleEoCZAbiayIEKIXCdUAPJbxQFpMHckwkgREVOGoy2Ys2kEoK2dFwvPOYUU", - "6SCikKRGQ0AtigCd7mpGGNKzAUeVa6BwoyttaaniO8XLSMXRmAQ8Jg7ue9Nnuv0dU/UTF2Xs7i9FLr4t", - "0F/PR09Vz3MFXHlzj58FX36KryFUOkzthbIb0cZLnv9oXEFdBGsz7OwO5LDTRcPOTjzs6BU4wuBCxQrt", - "o5iyVBHZR8fGvwVJsAcDJEnAWSgd6rjz4O0OZFNKrGHLhvzKA/juPtUey1VAyje2E5940O8h/T0k7aCN", - "4oazezLswqYLEU+VcffbfWXfCokC98jmvd/VFvbIN9u+jST/u92+JRkFq6zFZWHpjWRPUjkjzS63X00l", - "n1SNAc3aXPvob9DvfCy7iJEr4w0XUvVrck9/fWY6uA+kfd3VOij7du7fIPZbQOzntPLDJZoAS30kO+4w", - "mInk2iDCYpf2bnkILAnAbuABjtDro5MhC7QoMuB+gsQcpJMFBDen8PO/n6MXR2+66BgqPaKf0/FmHx1l", - "YLHgzB2yMReCXzlnru7ERXzAeZ8IPqehPh5YiBjRVEmIkFQqEn6PuJoRcUUlGTLQU4Bs30nEr1g+nKDc", - "JUolCfvoNYsWrqS2uSwaMqMSGikaYIbGZvuQ0KcnGCICG99l1Lru4IGqI5st6rnicUzzrTDB/RXxvJ+L", - "LBPKYUrCjVOA42SQB4DDxeMKN5IzlMkvn5QqnqgZtH9TIq/d5UttAOiyKTj7C/KKL93VJbD2/9TdBTN9", - "tHdBSWmdNBNnJUFWXsa6/N2ZwQy2d0kBTnBA1aKLcKRP+OxSKZVZ0Egv01DHguDLkF+x/pC9yYqR2Jxb", - "dHT2ruvuUlFI5WXXntxwXdpHr+dEyHScDQ7BRjMHM9CchEOmOApwFKSRPojJZEICSJeFGiOy4bo1G0rn", - "DvdO3om3IEoh8Dx9dHXY/DwBq5ezRZXjtsxSbwkSRJjGzQjdVoWBmECIBhjrRjlDlE0iG/UUCC4lsk31", - "SESndBzZGB7ZR2+1hodjMmRJhBkjAjQ5NF7A0HuJIFKmJgdbNwBItoajuihH30sEVzZ6IOJcSHPhrzn8", - "/SmSiiRL2OyNafkU5nxHWp9p3Pb0QH7kyhiavRX2FaQXxHCKIbjmozRyMYb3Gi1uBvTQWuJj2fhvBZ1O", - "tU0lODZC1kTMmW3tyGk2fSmpuLEm43n2VruajFmrhcTBQlLdUvS0UQ5IHXbWC8zzdH5JGwH27KP1En1/", - "0R+17LucUOofhH30mbP8Wkrdnxfy+Nr6mHIOf2wen8LIS1u1lAu7GvmqdfLrXSajtoa4ejBkq8cMaIVL", - "Ga5NBu+XxwiD+wViuO+qZY+bt0qAVCXbtCErfzXk/BfBgXeDNf/AQCQ3wJr/olLjAQz84SBKvBv1oVLd", - "S9fDriDsVw8Xf1cZ7gYzHhDTmjLcjdSz8aVLDaX39p12ZpJt8WvS4G1I4hr6uyP7N6u/hclQINaqW2LN", - "8CRO1MLFnPFJJS5M0o+k33BFmoWW3t0l6Q2iLm+PPRyfNsZcfp3XpA8S1mnr61GJTo49hdEfGQxgcc+V", - "DpYtfer0sAhmdE6ane7lHWxJlAjSS3gClyuhIZilhzvLFBb96Udkm7ewqPZfUKAR8OxJiEIqSKCihSmW", - "qSWC6eM7iQTXlgA852LRHD9htshPgsfP7WxWnId2T1lnWB4KGC96IVa4N3fSZokL7TMCMF3IoxZ4iDL0", - "8ke0Qa6VMGUg0ERbPohOMpKaivgSeHKzOODtQYNnk34ko+m4zSiXFPR4bQumoCCVisdu7U+O0QYUCJsS", - "ptdCq/oT0GRdnE1pjJ05jwxVtxsIuq7fVSsVWXU3Z1yYwT2IDtPmQJp+pElZLJiI1s5hZ0wZhsGtLJ1R", - "3lMmz173hymztWfdGrlRfDvCrOW34YwdzYlQrNISUXFuUJg3vx1zj/mYK+YruTOtdNq58Jzlzut2KUwt", - "M4vuojZDlt52v27r919O1g2VjzLhxrrO55lB2uQ2/7JYcHB/58N9u8vfP+IszZfEGd8FVzk0oFv0Mcyv", - "EHYdkjmJeBJD0XB4t9PtpCLqHHZmSiWHW1sQnj3jUh3uPXuy2/n04dP/HwAA///SB8fhkcsBAA==", + "YWvRc39vd2c9eradyKkLy6rIJR82xvHZidGJAs4UpowIFBOFbfGygpABX5KWMoBWhkkM6SqT75eLlob4", + "hSLYxW2hFj/X7XcD6uprE8YbohgzOtEC2b5Z7FnO8M7+waHBhg7JZG//oN/vrwsB8DzP+W+1FFsmSK6A", + "BtCXs09bhzvI9G8zlz8750dvftaCLJXCHFpbckzZYeHf2T/zB/CH+eeYMi9CQCs4cTqpwYiX48G0wW9+", + "PyxUz3J6T6uSM35nMEQHAyyJF2+rFCV5d8BaGY+bAdwmZ3VJDqdWV15BocMG2twaGTyvmaEKiODFbK8W", + "6OD0w/L7befugndsn1AFKwdOr99s3wr6Xi5FB66BRyaEZXjAUWT+Cjib6+3qAwcuHZHuWQtsSThGLIhk", + "1mXxx6z3wo/HxYEUfnfYlIWfLErw+zVDUpYqpH+v66GrpZBTR1dsZr/vMTsV2iKyW0Aubyz2A5+Ftwnb", + "Kvf+avrff/w/8vzJ79t//Pru3f/OX/z3yUv6v++i81efhL2xHIvtQQHVPhuGmqkyUgRSa8tKZ1gFHh+d", + "Nv8aKGyfGI+DCmaAIY3G5HDIeuhXqojA0SEadirJh8MO2iBgKcFXWt3VTdkc6k398bm5UdQf/+nU4I/V", + "NkKbLC3sgmQYGDIdhzzGlG0O2ZDZtpCbiAS7QP8VogAnCtw5lCFt/y7QWEDZVHvjk3feRX/iJPm4OWS2", + "7osSegYJFipD93Q9AFPYUZkwVPs6yeAQzKXLkGWndQZGZq79+pnyD+Ee1VwuP1GW22/Wcno68MG2QT6J", + "XsgIMMNRdoFGJTB6luiCng426/bcChsj46El7Ac7oV5P2TFli71kGBi6NoJ75FyUK8IztGwyewSBBak4", + "/PcCuYZyWmRLbO4NTHaRNNfOKpKFvKLNFQkgLSZk7l3hs6gFHsFzk3j25tcLpIiIXSrwRqDJOaGBnh9E", + "lFIpU82KFKOj47Pnm/0WBaGBtksTRMqD9qSN23vYpuvl3NzFMemi0xNI/LM7NFdrIVL7Jy5QZARMvq8P", + "0VsLAV1qCplAUbOS0SK/zDUnwLCz6VpMqpLiEL3OtGmcDaVUhLp8P5zvS2jWxvKYMPJa691aeVfhNGkr", + "2iBoHKss/VSfuM2ioL37xqHL6D1fcTasvbeL9+uNroTC2n9uqM7Pr+7srqfuuJJ9yQxLH3fPihdE8NKS", + "sr604vUWzdEQpX5XlqrNuoNiwSSrcWE+9xUZ3e9tb7/Z3lvfE7IuymIZ7aaAhJUBLbZHSLwLpMG6V+CG", + "qlFjnCzSj21UrDMx352hGZbsOwUPK4bm9u6TVqXNdK9tI0yLsaV8YoaUSSkHnZNFRhoQoSsaRSbgWNIp", + "wxF6hjYuTl/8cvrrr5uoh169OqsuxbIvfOvTAnDRiYoX52/hkhHLkQvSas5LwnluH7mhUsk6slCrWMdP", + "AXg0n7YrUeEmadrIK1UsR4n8uYTk6IWK2vyM8I4uQLVGxvsAbnzIzJ8vDzRyKczjp2I1WuPljqAaG4W7", + "D+awkua83yTnbw+6eCfDWVlNv3jWu7TMW6McdjvUk5J2JLUIJiE6Pc8rYeQeTtd8ZU7PdvrbB0/724NB", + "f3vQrvRRsKTvs6Pj9p0PdoxqcYjHh0F4SCaf4Ai3jG2UcRxd44VEQ2cuDTvGPisYZoVta02qVpf2dTDJ", + "22FHVhWaBnRIUOxcNIS0NVSblJsWOUdVJKQ4jUxaa7HuallbnGGJZGJg/wyieabLDhkMsGvhh+CooDFB", + "OAhEmvszXAEso/mmieX7IRNEJpxpbVhbNegXspAopnCzknUP8VQSZaHt4ZBtCJcGkeU7JDiVJNQ/QIxx", + "18Wy6qFRBcjt+oMhk7MUaqdv9tExZzKNibCuHjSm4IbeRDI1xh2MF6ix0AJT0pCIIdOveYAF/8wU9cOD", + "wWAwyGq6dw539b8HPm660xuVvgXONIl8AOHFLIQmYIOJlKGUhUSg11wSpTAihh2qcTdr3sZ8ImKm+7yd", + "HmU/zxUo/8ZcBbDZDjnzU+EKlxWSvSiXkG1tpu7/45OqzZK2iqrNKLBfjda5FiUo4GkUattnrE8745oi", + "ofWgSaLy6rxwQL5lV0zv0dLUbTyj4uiPlIgFend2VrpLFWRii4+2mDhIiYZ14Mlay7CzwluwcjS3BLG8", + "D+DKqqZS0BA/O0xl8drCZTYaDm1xfZFbjN5odMrM0mg+WTKniuM5JPNRmvoMEf3I4Rm8fXt6UmIOjA+2", + "nw6ePus9HW8f9PbCwXYPb+8e9Hb28WCyGzzZbSj/3T4b5fYJJl6z01fP2YWZjly4qy8KsSnYuHLu2wDK", + "a8pCfl06WrwRbcXebbTcqu7rsbCth+CNoIfSotBSg5Q4gwOUBLptE0Faqfbb4E47eDPYXuFOWykvYHAN", + "8veNSFlg8OZAEmeO6rgw4OJilcd5O3EKA3KR6quoVey8PdEGh/vPDvc/lWgu2nrVGKvsdI+L2xTC4UBL", + "K+HcLqWo4JRxIH8dq28Y37GN/u50O1mAOvwNB20l+DF73CrromnDdv1iZJn8bkg+PC3ZAnC1bECrwkOt", + "BWR5a+NUoSynVasXxxFPQ1RwaBkMH7jtOS3YBboZuHyx/i6DyWeip7X9ACCwAEFOmRbEcMulG7GZiofo", + "BbwLj3BsTCY7CAOEX7zgweHCXHDr/eW6NgbM8iFfWNsFvtGGDNL/gmlrMli/5/ImjOZziF5y+CazpBiv", + "OlDN62DC1F+vOls3bAqhSzWHzqwad4h+ylS3TPmzyt6GJPbPkRVYOcLDZinP1q54R3NLvnKFFNJux1C0", + "0+04QkGqaT3p9G3O9bX9V2RFX+QHwZGpeJwl9aWKRhbXFmZCpaKBtNHeenGb9Atbo4OEI2OYNAWRmUwx", + "a7xkHzn15d0Z2gDosr8h6x3W/9rMAs5KZ93Os71nB092nh20AijJB7ha7TyGPMb64FbqoEGSjqzfoWnq", + "x+dvjV8hMBY7XDfYuRfywRPBtejRM3cNFjt/1n9WxGUJeTqOCtdXFsTJwEDCgnmhhzJZ1BC49AeN5nQy", + "YX98CK52fhc03r45kDvj7Qa8SdOR36V1WrzCrvl/ybhn6m34oTOAoYRsRJd5TSTMAF0QhYB/eggHYDpk", + "6YeW5RwGjaW4l7H2dnd3nz7Z32nFV3Z0hY0zAgeX51C2IyhsMXgTbby+uEBbBYYzbbqcbIAEZtas9O8z", + "ZKszDsoKaX97sOvjkoaDO+ca2/Y8biT5O2ua2UlZokMWZWa21Xa5l9q7u4Mne/tP99ttY+t6HYmb5RLG", + "5RgY8ljk6uLKb4A2+eboHEEG3wQHZb+JQyVda1RqrVEB6rpBS15jYE+fHOzv7e5st4NJ8oU2WACw0oYt", + "yy7PpvMwhWc1PKSoi95u02nhU6cMg70mQYRpfBS4mOjK6WNQkUfCvJYvQpuDwXr7awdXi29bOY4yd5CJ", + "qDeqARcoZRkWf3/1vebtrimbxbQ5D1aLcV/gPNPksngepvjOLWiXCDKnPJWfoSGuTFbbJOJcrPVtk4Xy", + "msg0UuYukUr07uw7ECKauZBUJCkbTZb9lqCe3HJya23gEk/4ubqJWK1Wo83SL5twt2GbdpelvJe2fyO4", + "UKhFVcpWxxQe4yhIodwEztZTzwpgQiBpN0mihYm+jSLOGQpmmMGNg7DFbdgUYTTjUdj3RkTqJ6OJNxaB", + "X6OIG1jUK0ISW4nBDEJ/pnUWOidoo5A0jAwrVSrm7cdGqlis/TI37sf+0l9Y+tJJsmRVTU+seAGx03xS", + "8jFGfCrBClQQV9yvAkUnWJhwYcxMZZF5bIzHMsrSjj7tPUOsSG/fEWqOTj6xFq3VMSAV1FASB4JLiUhE", + "p1DF4t1ZJcNwSVZKlme4Ok6wPNgWrGtuBz1nF5xpsnUBIt+B6Im4/5QjEXgYsnqWROA5b2SMWQq1GQqM", + "TG4SKgx7tIuym3GpRhlyzJqDlWoEgOupIDm8VJYXmzmA3Dvec9GJttuQy4az3urrGlf5m2oaYLNM9VLU", + "T61uxoM+Nq5j5yyF68nxf6pgL+ugO+UI3VRCq7QALIQ2GFclsVRAmd5sE3Hit1F1PzXz1NaV+3VvcNEW", + "eGk5ztI5VrNTNuGe7Pw1riGd69kGQSZExBRKDqCQMEpCZzxm95HWtwUpnZEkKEyJpZxRSAW2BMdme0OG", + "PXNOMcqmFVlf7bCNP9iMYTkeO/RrX2wTOyT9KW9vRAq0MtF+EuE8+a1V6CSVI//9Vb1hQaZphAWqgost", + "GbJcxBFlV21al4t4zCMaIP1B9ZJ5wqOIX4/0I/kDzGWz1ez0B6Om4h0XZnA2c8csSKXffAo/6FluVvIG", + "wfWyZb7fAkyHNqFY3gDkn2hELP7WW0ZvCoxeBize2xk05bM2NFrKZK1jt60ruS3Lend8Kj2JaEu1HFd2", + "hIQWRdqoPUkqZ1moQcWuNM82O60cFg7S0F0I3u5+p5ys8GnIAsdGmFdwBdCYQK4JzM0bq9hGZnrpZQTo", + "Zpt5emHbUzlDv/Nx2XfaNvrWUwxog+Xp74JMvAHnwAtLfdfmjZaMUueLdZLPQVprKtj8c19feR76Ognf", + "qwou5QFSTTLsda320IxYYmcUMHWIWuDquwCOLKnZ9to+u7laJaqprsWiUO1MojEXAqBgs+h/ILemcxdZ", + "tAupFhE5NBGIAY4iIsBTYVuL+JQyG1GX3Z4HESVMfSfR/7fVN61smQjq/u+Ss80hEzQkEmGwVR16bJaz", + "pfUsgkPgNTqHYomGFYwDv49ekjkRQ5YQIalUpvBDxKdTEn6PsJkBODZFmljHJ0Y2TA72pCx0MWSCKAH2", + "uDPQ1YzEffT3gvXbLXT/nUT8mmUEGLIiPbVtmkpXMa4CY4OlvOYiXLI07hWoZAEXw0jxK8KKEjdrxmve", + "moZG5qt6WJ8pawlPEXj8sezRLPqjVHQSVYtKLu86lUT4dd5sdtkrrUJvCruwYGuZ3Q4QDFBMzP4FP+Ww", + "Cy1AFvLm/8c1mf90njVe/q3yWgFIweGaHhm3s9eFHJhcnkqwUvkmcGWoDSSLLYPPqN+FoA0XXu0qMpQV", + "m0JlhFYne7t8tWoigRvNliRBufe9p/tPDlqWpviky0YDF/S5rxbn8ZIrxYaVOmtzb/V0/+mzZ7t7+892", + "1rohcjknDevTlHdSXB+0QW6U3qzRv//5r3dnlVurfYjPHqw1KJN14h9SQ+ZJeUDvzv79z3+5Ud16QD5B", + "U4ckbog7aIwyioor6QIdyleQ7S75lrgrjko+D5yJGbRBJhMCXt2RoVsvH0wFd6Cd3o4THFC18AhyfG0i", + "4bNXKtC6ba6zyoP1KemmbQvDqCWXTMd5auaG6xz91dxtV3jhaesKNzIdN92jv6r2am7R80uYYoxGixCJ", + "vBZ73V+fzecay1Kwt/47AOXDZZ/VM3HMG8thPqtpEhCFYws5FUIZffDQFVXVflRc/spyFu5dS16qKsXf", + "L9mHzVtwLSe250T2+LCD1fm1FflgD8DbfTUaF2tPLS3uVSpUlZ+66/fbIoW4DsyenWDr91dIplznwyoI", + "KfCjHYMled52t8QSDdxUyJPx+AN5RHpZoKGzDmRqLjj1nre41p7szuCKTyZlcM39ZjBmQEGGRDDXC1aK", + "xInqInLjXDBVJF8U0SuChp19Oexoa2DY2Y6HncotnDe1MsY3I9tBGQFlsAwdOau6WB2kdDMYRzy4MmWV", + "oFpvHw1QTDCTKGWw+SuXhNuD5Zdd3U5SWJsMi5iYEK2a2IIxjckMzylA4NsromkpkJTcUCUh4BXaOUQh", + "BwdzuaaknaF+zSQ+HuaThkMHs4VtWDeo3+PMReTm74IzYgKVLNkHInjXpvRrif3q1VnXBGBA6KQZWCk+", + "003UjEALyKyLCp57/rs//nkckRGMu4oPHtfpWExQB+NbEEmUtIDBOTtUmAAFPGWqChwetzPkyiln9SMp", + "ZRCsaD06AARle7cFyUMSwI6U9b1YZvRbMHcl78FS2pf4sOtjYdgUcDXmv/p+be+3qwMwnoJCOVjTTjEu", + "3Vx6jqTitn5QtqtH5CYgJKwiDPpfaRvrb7/0xvr/ii2STlap1b4N8dr12fXvLvkLxtpE7WJOAuOsBxge", + "bkkt3obBIrOILmVGK2EdF3AtRj48R98LbbKxyc1yWr8kNwoAmcM00uRtYl0rquxhtIrit856bNrQXKwu", + "bH4HdapMvPytKlXZUPt7KVZlf76TAlW15bggyr17YfmmuS55qZRE6UrOZSi4V8oxQoZ3usie6Gg73qzw", + "3N7M7waxsJ8tEzYZjskoEWRCb5Zwi3nBWMJljJN855Rq10u0EeMbtPcEBTMsZGXsjE5nKlqUA4j2PBBD", + "n1S2TRBFmFqjyn++mu7DerSeXc5i6z5t+KIACFQrmmB10NEyZN7j/LbQxkMleAFum8ZLzie7e4PB7s7g", + "VtC8blhrkOs4/8TWPCu305QDWPjOBiqUwmqLLWQZ1/XKmdeCQuJ2RiapBMHxIWQKJTggKCITwI7LKhav", + "vvysdr188FaDsug0Gf+7hbLr5mIIykU5sq4sqrGbRsfdf5YBiYrPV9zZNoiZoIY050kS3O0NDt5s7x7u", + "Hxxub98FnG5GpKb0lCcftq+fRDt4shc9XTz5Y3v2ZLoT73oNrytqao+04dVf9LuNUUL5qVgGNiqJNLRh", + "55AQUS2JWi0lLElEGenJLKVrdV7lEllg4gdW7v/1HPtmBkuVhYvyJIs6A1Y5cUqcdU/oW3b0S28nqsM/", + "PVk+7FvlSFUH4mew6lCAn9oNBkDvtz+1vH3KWp47bwsvtj55lubtrTp7fNfzsLW9q9xAcR8/lwRjaYct", + "O7Hrp5rHOzrlgqpZvPx4yF7LkIkh0PuDVGEZ7amPTqcMCiAXf87i+opmkv640+1EH/bKe8b+3h73yyLx", + "Zgxol7qoBrSIe4P62supAK/kpoUwoffaGtdj/mG7t/0Mos+jD3s/DHrPqnEAQK0i+bbd26VfB21oWKwq", + "5qrRbD9bK0Tc0XMZB/1CfTWx8oPYYvRaHs+rz7qzwqUElxY4f1xb4wqOT6PG+amqnT3NRkUtKSQR9nh7", + "S65YWbEPi0yGxmRKmWzjmd0dZK7Z/XjY6aMji2wN1mpeXLzUPJSVLvAJjWMSUq1UGuO+OeVip6W3rWo8", + "rFfuwH3lUc/6fv3s2WoQh1UZYquOyf4nZAx/krnbzsRdhi8CnjNnkwKCGLzYRXSCMKvUPLQV/m2qP6Ru", + "QiTcoYNpy1nWygCZK37OE9JFU65QnuTf0qOWsmbPXzZ+cgMe1SWoHoYhdj4LZEsGH0aXia/TE5QIHqZB", + "nuEawaBzTBKRVgDalmj1q2OI79KhAanjEy7QaodGkwejnQeyab0r3kfNsM1LvT1YvdR34gXpdtIkXC3D", + "zEvtJNhaCOYrciY9Ppky2SuaYGEy71tI9NdFCtaNXOMtDrRKlCbuCkXzVJ2TPBcqcIngCyc+IRHRx1S9", + "EcSjME/roDKXoqtF6vbB01nTJSbcOdUH8gshibZVAKEJ+osxW3gHVi1ojzYGrkqpNFdaPVMBxVKrPLgn", + "KzWxxqUq+mybSgkYKV/xahtEl6uCgzuD8i6qZnWMFifwS0ra62YofPulC2xrdBjfhR/uIZW0V/ZyoQLa", + "6iCFszBo179JWQXQZxZVzus93wW8jy3eWMu4CYi2mrZadDMf9f5h3Mpo1D/c+uFv/3fv/V+97uWK3SyJ", + "6IVkAqFEV2TRgyo6SNvo/TIMK1QA0Mr01LIKwTE4jYIrYpxUMb4pjnd/kAmNxUsc16YAMVgxZdm/V07o", + "b39pjmAqkPEtyMmVLPvJBTLuoiaj4u442oiJmGq7kLosNEi+HjJIprsiC4kKJY6sSuMY9TuZfVKI40eX", + "Rg3sEza/RGMKNePkkGmrFgcBSbQ1YUu9UFP4mIP0EQRHxXZsqSUXCG+vHE3EAEHvzmoYvq/evvnx1duX", + "J6NX589fHp2Ofnn+vxDEcd0zPYQ9zXt7+we23HGRktueJf6EcgKfBKPrYzcDxunhL8gKhRrSHoWZSsB0", + "cAEFhZfRBokTtXD1E11y6eZ64KBHWYPecLbPXNtl8OxzlLJ7u7R23ZxHPa1RN0Dzex2YhhbecGxoyoS5", + "d5oc29OxR2203sQpnWKPL9tb7f1zlJxzA1qZ4VNb/8aCUf7g+JNq3QIjDQypKjj7FbtUql5z7HysFalR", + "Xle7HJGRMpvfSQsBW+VkzpipLVsa0ocpEXJAz16W0ZvvMgcZ2IOPVieqLlXlCzMrjKR5bc6cxlrRqZcQ", + "6FyT5npGBCksBHyQ48GvSTKb7NECqcQUcEuIyAMhXaaIVoTgSlOijczZ4EiQZeTWPbDL8f7P8E3WA3jv", + "sazdccE88so72y9+BGz2164sPJ24JmAYFXvCj0Re5qJlNHFcVV+MIlfV523e9248K6uWSL+mvVVhzryP", + "Emv6+PHvmKqfuAALpBkX5M4BzcG6CYkAYLQqXHkrrG8ak3DEU7V8/9u60BYUJHRGRF4c0llbGJg4KGX6", + "NskCh1yRj6FOaU0OEqSCqsUFFNE3EcKQZneUmg3vavHbn/OOodbix4/gp5x4shBeEEYEDdDR+Snsxxgz", + "UNLRu7NCfTRTKq8GYgrq5avjU2vhOhxcsFioAtZzwXxH56edbmdOhLHyOoP+bn8AmzkhDCe0c9jZ7W/3", + "Bx1Q5GcwxS2oC20zhW1+bGYrnYZWE/rRvaS/FDgmCr74zZMtD8Fs9nXQevG0YLckmApruCQRZPEbhqH6", + "a8C3dwfqoTmVu4bsrd10kOQKKRUkeWUX9z0olbB3YJo7g4FF+1b2+IWEEBOFvvW7DUnM+22l1VkSeeDe", + "a5aF0y0z0n/sdvYG22uNadlQYO/6On7LsM0MJWAQ7q9JiFt1espMrpdNu7UhN8UdB4xU3Gu/vddrJtM4", + "xmLhCFakVsJlk2JMJMLuXQNCrCQKtKiAojR99IoRWwwdK4RNOKxImYT4C/uh5tDyLjBtu0XOgHx+5OHi", + "s5Gw1Icziz+WxZneLh9r/Pz5eCdj4/pC2kcOdtpw7T0w0I84dCnlD7ZT9gbP7r7TY84mEQ0U6mUMbINc", + "qYQokwhAtB0+Dxfoj5QrjLIY8Ue0pa3OOs7YrZsfRVt/0vCj2d4R8Xlez4mIMTMR9+adFZu+tp2NFzzf", + "zktPNcf4UO8CTioHVGMOKlDkylu0eGxVlcH6cbTnyfC3fZrphQ/I+Hv3sMPtZLOqnA+55QyMQirJY9pO", + "9lZnnCshXl3uBVFfCs8P7vPIssj6X+EueiwM/IJkGl6+WrVDYSsRKTMGsFcDfJ1nwdnvvisrf2/yJ4XA", + "DHCl66ahxoMyV3k4XPSRo6kx+tUCIIEEgXmG9WPlXA/vS9lhO/exw2DG2eXEt2Pq2zG1bJcbbnFTgI1Z", + "2OUtfBBreSC+Pv/D2t6Hb76H9r6HVp4HRq6td+F3Pu4jGwQJJfHljKdRiMYEGRAdF+6gsOhPPyAsghmd", + "kyGztwVxGimaANQZFzEKscLm2rbRMbHULZE1t6Wb67nQt5zAVXAESUaAZjdqgmHMg94oYyRE+hMLnJfD", + "39XqV5u973WwZw3mRyO6nnEJVxuAEMdU4TSHnFlprGNotj9kbywYqiYgxO86WSNJBJCuS/w/nCE8ZPaD", + "750IcbFHEse55MICUOSowWA0y1JPn9IjHcmA+wBc3hCGmerJhAR0QgM7rSuysCGE3gZbFSPSA3bjfHeW", + "5QigHT+Mo4H18wPYnmTPkOWk8v0Ng7jbIErD/JLL4dJgMcZR5K1WMY34GEcjQ58r4rkTfAFvWKIU69y7", + "2yTGQ2JqlicLNePM/J2OU6ZS8/dY8GtJxLCz2R8yiP23tHYogIYHrqG6WZxwvc8Ej02fW2aIW39ekcXH", + "/pAdhTFljiPgExxJjsgNfAdFnwCIwUivBn4wu8l/D36cSsXjIsan4zszTJ6qJFU2iUES1fWhVA6Z4uhP", + "hx34cevPvMePRTDGwitmSqBbN41ajrCe/Qhe9Vy3EyDAsKMP0mFH/z0VmCkD35iBH6JpcUk3sgoCepNu", + "VikcYIYSnpjqC8BUM6xZrtQGAADgKEIKtpL7VivusJIN87F4bvG4EczNoG9VthFl6OzHwmYa7D317ydJ", + "AkF8ESX/ffHqJYJTWa+BeS2PEDJZBEwrDChM4erUybTnOJghc1EFFfaGHRoOO9l1brgJY02lTYnv9eBO", + "8Qc9tB9MN10a/tDv66bMdeUh+u1P08qh3ktJbHAmh52PXVR4MKVqlo6zZ+/9BG3CxLooCQK0YY65TZAk", + "mAJ8SeHEN0ckZiHi9hSIFgijXAIVA1fGlGGxWJa75iG9pSCfmOC5AjH+HEKw3LBzOHThcsNOd9ghbA6/", + "2Zi6YeejnwL21rK5nBucZ9nlZsZEB4PB5mq0aEtfz51li4uBz2wDNlpFWS1KvYIW2/Pruh/4j7Y/s6sf", + "zHTnOd6NMfyd8/0RXkAUNPaiJeq5gqio3ZgFJHJq92pHz/1fHujFCkgU3TeDPhR7ZtdjGWD9o2JHWKx8", + "Gy113z8wxw3u61Apue0fhn8fnf/c4z23vnMyd6HO/loegHNiTWlkXkZYogsYU+9CG9/P4de+/a+z/QCo", + "7zLi08tDY7qjiE9RRJkNQS8EKmv1wNISPjJQJ9l3FvnEFVLbMJrEv//5LxgUZdN///NfFjD83//8F2z3", + "LVucAZqbESzUmGB1eYh+ISTp4YjOiZsMAKWTORELtDuwPn94hAr1362WJodsyF4TlQpWCNU3Nc2kbdBe", + "Fej5UJYSaaFi9It0YguumNhGj9/G7WVDynvd0V0Pxh7MoDABfSo6HgCAMmqqT1tLtON3mZo5l5ym1TDN", + "WrDeavmiyI0y3NszA1xTwACJffsOHthJo42Li+ebfQTWluEKKKoDtkPejDUj+t9k0mqZZCRKWaAAlY1s", + "MvhJy53+J/addl5/2+LX5Pa3ddbW8Psb5w/AJroV+HYH0OIOwE83dx/gc8qfOICwuwsWNF08UKyg4706", + "zc2TAskewhmANhwQAzhUuUDnx6cIh6EgUm7+Z7sK9EwNl+ZHB+IMYP8f4tbajoULC1FlTbUygzwWcfDa", + "jhphN69q+cri+bZVKgbReNJldSHyI+/uT49Kp+scI3mJzZzXvp0kK+P0qAy4/rbALb0AJ0BIp75k+7TI", + "RascUiYCMDtylqpLVjyfnrgNeX+uKdt1yqpnwz0IxZOKQHxAQVjO0iwWpX1M3Pw2W0UHhrrEc/Vlsebg", + "/rSg+/Zi+dj8MbmxwgrZtBQ0eAKNB+gLogyKQOcOF9r24Jn4BRFuV7sa4jDrbFrmU2TgEGBCcDW/3PY9", + "Na+0M31Ne1+T5QvkWUdjsST/pqK0MHZzWi0zcE9ttdO7s2+hh7XM2893420ZzENkCLsZO4+1UCREG1gu", + "WLD57dL7s3O0CYnKjVjh5k1ClERYQXQkALFkdpYe28496HVZRVSBFbFxQ48xGe88jSJ3NTMnQqFXx6dG", + "BBQPq60/IZJstRHixMLSc+vt6197hAUcQgezsDe/tmeffGZTxHBWKcHu/vn5ESaZUXfwNqlin7D+tgC1", + "CUrtU/5fOz9FdCywWPzXzk84Sigj/7V7FGFFpNq8M2YZ3NcZct+mwSNmPm0Z0DLRQDSxKUAHrlCls7da", + "atPu/a9KoTaTXkulzuj6Tatuo1UXybVUsbZLcaeqtenjge6OMmbzURsefcOZuAd3pOXIAs5E6X4mR5qY", + "cang0eNLOrSRnjTjuOKx0dKvnm/IpceHY93Tky4QEmqHArK5zem5Jy+7G8e9K7e23/t3sR/FYzpNeSqL", + "6UIxVsGMSJtKF5GyAH5sand+PDcq3l8wlw7u8+i4d736G9/fkcZfXVAjvM1V2Sqd373VVue372ud38AM", + "2nRDC7/edaU5NhuiHx3QYFs2LuEx1qMyfePy2SLorTZUcnMBgQVxOGT/R9sfvymC4/c/uLymdDDYOYDf", + "CZu//8GlNrEzxyqEQVVwSHE9enkC95NTQGiEYkt5FmV1HKY6K7Ceg5f+jzOQ8iva9haS48JvFlIrC6lA", + "ruUWkl2LuzWRyhD1924jOX7zEdwC/X6dVtIXfPFw7xacTCcTGlDCAOgfskVlLdLOWHLfbkZumSXI7E1f", + "IUynpIm0NiMzqbVCQ89ri957iNZpXkzlvq1HV8b0cWY78MTWBbT2Wq4tNBtsXxo/DO739Lp/Q+0xs5ix", + "iOqkS7TS7SnXYQrVxKmC8NIc4Afid5EwZk3WYh8dZ3ndMk0SLpQ0xW7AQjDlMGfaQvAVxinXuvEVt4GC", + "LpTI7pBBuVP92OBTbF2RhSllQznLqtZkM7UVYXxZdOVSQg+6jT6/Euqvk9RKCb3nbWwr3z2cEvpgouNe", + "1L3TUkHRjWxjgMU9JtlO5lmaJv1A2XTzUcUSG2GVza0AR+ZRtbZwqrirg7814waZyA/Odh7hALDZ9GsG", + "Nsjm/RqcsGJTkMwreBQRYeCgklS5ullDlg2OskJdYFuk4lI3P0qZotFl10TTQE6/RJgtLCbKkJU6w0qR", + "ONGCzaL8wAgFScyIKwXD9KApTyW81UWSl7pEOLrGCzlkgkwiEti5QXFFQQKDnBZFffQzh0RqhKeYMpvb", + "q9805ba+k0N2ScOIjGwe9CWiEskZF4owEqKYz4ks90uwiCgRMIljrCknUYwXAEhksNkMfXhCDOhPKdua", + "639jFlIoRqV7zqZ8OGQY7QwGKCaYSUQhIVfiCdFf2TYQDKI0oO8RRnuDZ/aryroBaKYj/4beL0KQOQ/w", + "OFogorkYTkS1CQuY7S+o66iXb0KFNOuVuRdt1Z/SwlLp6lOGXZSyAMonpkL/iwuUMnu86hYF5JjDPO0l", + "HKEiKztmE+LHJMCanoyX+wEoMh4EqfAdjnqpC4Xx/hOVzML0LoBUPumk6YBgT4Ww5oyrGexpDltp8/sG", + "rsqZ6us4ZLybhAuEUYGvc4cC1JBmU7QB0F2Xeckt5qo2Xm5+7/aO3r5WELjtb8CzHsv5BEzEJ5PSBlx9", + "NJkNvCxxoc7CX+s+PXa1FosiLqR4yrhUNHDCsFoN+Jvx2Np4XE5ZLzdPuLgq6lZl/v2Ji6u21teFK3H/", + "qIyw4gy/wHsAPTwAX3346wBwRhtDRTPNvRtoVf7KdikoXVRJF2fMUcTZVO+i3Cl+7177ikUXRCmo5c6U", + "c04QbYSM7I+mXKOejC2GBx7+wLb60LJI934Pd0EvuUI0TiISEyjn2DPMphc706pNtWUq0Syro7eerNS7", + "qpiUa2xBaa7/u04dAr5yC7YB2nt9ubxCNeLT1UBcWecOdcqDxDVkpho0caWjL1Emg7VCa2Cv0fWMBjNA", + "5QK7VbdvQLtwklxmgKSbh+gFbOQiLit0vmHArjWvSR4RA7Y1j+PLw3rBwndnZ/CRAeQypQkvD5ErUpid", + "H1K/VUTZ0rOIsFTopcUO28iMcVjRS4W1vZnNb9Pib+WAsUPmw+Ji5No2SCfosgDLddmAy+Xk7a98+mDK", + "WLcZ5tvMRXFkTUfgTcLCTlOMBY38iFzbg4EPfbYlOpgZxh2Dg9UG8yufZhDjJVbGSdKWfe0wgYvncbyE", + "h9FGLkGQVCFP1d+kCokQ8LHl7ibmRhs4sOVl8JVmVGakktvYm8B+3kgig/nrJZUWqp1uh7A07hz+Zv81", + "j+NOt2PHU8AKXkO5X4GyVm2wHvGiV6YApfZNLV8HJK0s7AsoaZWTw5rTzRr5a/PCV3+z6Hx2D8iGoB9U", + "nLhfkgpaGG/Z4cM4kgwncsbV48Jlsq6mitbW7Kpxs+zp4YWpq4HRJoTjwn564b78AqzfVZEdbszITffe", + "QzzqI3jMmbCyNpsJF1Uwn1WxH188I32+JalNtQ2HfOPN9f18rRgz8VXjd0sTmppIOFU8xooGUI8jmHEu", + "C2w/JjM8p9xelbo7q4wzwblh7EwbQn+pWfXSOoIvrSJ/aJ1WCBcf2T768LkNvPd/4R7lX/xUsMszid91", + "yjdgVkPBYEHJBCU4lUTrVWlMkCnFbwuwEBzMUIATlQoCtaUIiimjcRoXXA3acBJzHCEq0eV2fNlF41Sh", + "CIsp2EXmoQmnFyTgcUxYSMBDNmQzgudUG3UCRVgRFix6kkBNyjnJK/1rI99G4ZiaVoJoDqScdVFMFA6x", + "wqBqXOodPzJZPJdZmUpjWDNyk3NDOGQiZd8bnG3d7KUb6CUiUuFxROUsK2cW4JCwwAtiffFli7HP7w2+", + "IKo60QeKy7mVLH3IQJ2i19MN58uI4Xlkwchc2GVsI+aXKL2y2Ygspz84NvrP3NJmrm6OD3TFk5F42S7+", + "Mu52Mqb7Yu53Hv4ChwsUpqa7wq4ENv9ab2UygVIMd4LUSrOMt72ayeo2ZWReS+Zt/en+PL2FN+0LkYTd", + "RsO+qUJIPukvQeRaqt5K5j6QG9H6kgpesQcUwS6m6sHUJy4KUu6xuDutwDZbM5PbRemkBAbri7NvYrsq", + "tm3IwW3FtvPN1i7VC4Kcsh5EafoluHXjNopq6zr4D80FqcyuIDIfXETmdwf3JhZPM0FoRGOCFxHH4dcQ", + "prvkBifgQhj8B0CUeEz4owWvYTFAH3xz3UxCdF1u5buzs80mKSHUUhkh1COWEOUa/UHsK6I/J0LQ0BUH", + "Pz47sQGzVCKRsj56FVOo2H1FSJLnlACQR1/PzyFh1MsclyAvuh3ClFgknDK1chT5q3czmI+3Ko58z3LS", + "QkV/u5BufSENnv3HJ85AykDWhJnAcstUYdUYCuhC4ygztc+1XobHPNWtaxmkyaTXcwqn4IRGRC6kIrGJ", + "C5ykEWw3KDtgq1La78wqdyEqVu8ck7CWEBFTKSlncshstkZChO5bf67bL4Q4eS8EFM7k67kRkl9G+Jwe", + "jIkYw6qJaoBZBDXhO4edLZwkWyFWuCFEyw7vE4b0E8TDIbmIxzyiAYoou5JoI6JXxjxBc4ki/cfm0oC6", + "EXz3uWtu3n5naUqfsgn3liUzPJsx81eVV2XFmruYfHRi7QUpbhYnf2Ch/WJNrpRrguCop2hMMuQalCoa", + "0Q9G1OlGqFQ0MEk/OWTBu7MctWDIzogS+h0MyWVRRALlHDZbieDB1jAdDHaDhAL82S6BwYHAa34cQ4/H", + "529NIiiJuVh0h0z/Axp+c3Rubncn2HoTCgNlRF1zcYVOt16tCDG+ADL9B8fomQkuxQ7wLvi3K8H1EUEa", + "95Bs2KI8WWYq8eSrDyK1Gtw3v8Lj9CsAJFM2m42pwAEoxXKWqpBfM78PYc6jNNb/MH+crgL2UjiYvYNX", + "vxht1wxnZTdugo9iU9o5hcSUTXyQSw9DsMcas6oJ56YASkwpGtB7Chypr5G7P7/7vkjHL/C601LUlST9", + "YvbWfZ98dgwO46JIj8eyzQ2nuZkovtz7dI1ps/fpx4gHV9KCoRTdhtpuA4Bx/WMOCG2vCEFNgNxMZEGE", + "ELlJqADkt4oD0mDuSISRIiKmDEdbMGfTCEBbOy8WnnMKKdJBRCFJjYaAWhQBOt31jDCkZwOOKtdA4UZX", + "2tJSxXeKl5GKozEJeEwc3Pemz3T7O6bqJy7K2N1filx8U6C/no+eqp7nCrjy5h4/Cb78DN9AqHSY2gtl", + "N6KNFzz/0biCugjWZtjZHchhp4uGnZ142NErcIzBhYoV2kcxZakiso9OjH8LkmAPBkiSgLNQOtRx58Hb", + "HcimlFjDlg35lQfw3X2qPZargJSvbSc+8aDfQ/p7SNpBG8UNZ/dk2IVNFyKeKuPut/vKvhUSBe6RzXu/", + "qy3skW+2fRtJ/ne7fUsyClZZi8vC0hvJnqRyRppdbr+aSj6pGgOatbn20d+g3/lYdhEj18YbLqTq1+Se", + "/vrcdHAfSPu6q3VQ9u3cv0Hst4DYz2nlh0s0AZb6SHbcYTATyY1BhMUu7d3yEFgSgN3AAxyhV8enQxZo", + "UWTA/QSJOUgnCwhuTuGjv1+g58evu+gEKj2in9PxZh8dZ2Cx4MwdsjEXgl87Z67uxEV8wHmfCD6noT4e", + "WIgY0VRJiJBUKhJ+j7iaEXFNJRky0FOAbN9JxK9ZPpyg3CVKJQn76BWLFq6ktrksGjKjEhopGmCGxmb7", + "kNCnJxgiAhvfZdS67uCBqiObLeq54nFM860wwf0V8byfiywTymFKwo1TgONkkAeAw8XjCjeSM5TJL5+U", + "Kp6oGbR/UyKv3eVLbQDosik4+wvyii/d1SWw9v/U3QUzfbR3QUlpnTQTZyVBVl7GuvzdmcEMtndJAU5w", + "QNWii3CkT/jsUimVWdBIL9NQx4Lgq5Bfs/6Qvc6KkdicW3R8/rbr7lJRSOVV157ccF3aR6/mRMh0nA0O", + "wUYzBzPQnIRDpjgKcBSkkT6IyWRCAkiXhRojsuG6NRtK5w73Tt6JtyBKIfA8fXR12Pw8AauXs0WV47bM", + "Um8JEkSYxs0I3VaFgZhAiAYY60Y5Q5RNIhv1FAguJbJN9UhEp3Qc2Rge2UdvtIaHYzJkSYQZIwI0OTRe", + "wNB7iSBSpiYHWzcASLaGo7ooR99LBFc2eiDiXEhz4a85/N0ZkookS9jstWn5DOZ8R1qfadz29EB+5MoY", + "mr0V9hWkF8RwiiG45qM0cjGG9xotbgb00FriY9n4bwSdTrVNJTg2QtZEzJlt7chpNn0pqbixJuNF9la7", + "moxZq4XEwUJS3VL0tFEOSB121gvM83R+RRsB9uyj9RJ9f9Eftey7nFDqH4R99Imz/FpK3V8U8vja+phy", + "Dn9sHp/CyEtbtZQLuxr5qnXy610mo7aGuHowZKvHDGiFSxmuTQbvl8cIg/sFYrjvqmWPm7dKgFQl27Qh", + "K3815PwXwYF3gzX/wEAkt8Ca/6JS4wEM/OEgSrwb9aFS3UvXw64g7FcPF39XGe4GMx4Q05oy3I3Us/Gl", + "Sw2ld/addmaSbfFr0uBtSOIa+rsj+zerv4XJUCDWqltizfAkTtTCxZzxSSUuTNIPpN9wRZqFlt7dJekt", + "oi4/H3s4Pm2Mufw6r0kfJKzT1tejEp2eeAqjPzIYwOKeKx0sW/rU6WERzOicNDvdyzvYkigRpJfwBC5X", + "QkMwSw93liks+tMPyDZvYVHtv6BAI+DZkxCFVJBARQtTLFNLBNPHdxIJri0BeM7Fojl+wmyRnwSPj+xs", + "VpyHdk9ZZ1geChgveiFWuDd30maJC+0TAjBdyKMWeIgy9OJHtEFulDBlINBEWz6ITjKSmor4Enhyszjg", + "7UGDZ5N+IKPpuM0olxT0eGULpqAglYrHbu1PT9AGFAibEqbXQqv6E9BkXZxNaYydOY8MVbcbCLqu31Ur", + "FVl1N2dcmME9iA7T5kCafqBJWSyYiNbOYWdMGYbBrSydUd5TJs9e94cps7Vn3Rq5UXw7wqzlt+GMHc2J", + "UKzSElFxblCYN78dc4/5mCvmK7kzrXTaufCc5c7rdilMLTOL7qI2Q5bedr9u63dfTtYNlY8y4ca6zueZ", + "QdrkNv+yWHBwf+fDfbvL3z3iLM0XxBnfBVc5NKBb9DHMrxB2HZI5iXgSQ9FweLfT7aQi6hx2Zkolh1tb", + "EJ4941Id7j17stv5+P7j/x8AAP//Z8Z+WvLLAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 6cf2d95c..6a8583e9 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1784,6 +1784,7 @@ components: PushStatus: type: string enum: [queued, pushing, pushed, failed] + x-enum-varnames: [PushStatusQueued, PushStatusPushing, PushStatusPushed, PushStatusFailed] PushCredentials: type: object From a9a45002623875d8591732e415afaa5ba65fb099 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Wed, 5 Aug 2026 19:56:39 +0000 Subject: [PATCH 18/21] Add pushes resource to the Stainless SDK config Without a pushes resource, generated SDK clients omit the push API even though the server exposes it. Mirrors the builds resource mapping. --- stainless.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/stainless.yaml b/stainless.yaml index aec9abba..3c40590a 100644 --- a/stainless.yaml +++ b/stainless.yaml @@ -223,6 +223,17 @@ resources: cancel: delete /builds/{id} events: get /builds/{id}/events + pushes: + models: + push: "#/components/schemas/Push" + push_status: "#/components/schemas/PushStatus" + push_credentials: "#/components/schemas/PushCredentials" + create_push_request: "#/components/schemas/CreatePushRequest" + methods: + list: get /pushes + create: post /pushes + get: get /pushes/{id} + settings: # All generated integration tests that hit the prism mock http server are marked # as skipped. Removing this setting or setting it to false enables tests, but From 13a1a28fc1e1f6f79fe065bbe8d98e73fe66e94e Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:40:11 +0000 Subject: [PATCH 19/21] Address pushes API review: push timeout, drop dead wait surface, simplify dedup --- cmd/api/api/pushes.go | 4 +- cmd/api/api/pushes_test.go | 3 - lib/imagepush/imagepush.go | 43 +++-- lib/imagepush/manager.go | 277 +++++++++---------------------- lib/imagepush/manager_test.go | 301 +++++++++------------------------- lib/imagepush/storage.go | 9 + 6 files changed, 196 insertions(+), 441 deletions(-) diff --git a/cmd/api/api/pushes.go b/cmd/api/api/pushes.go index 1a080369..1b813a85 100644 --- a/cmd/api/api/pushes.go +++ b/cmd/api/api/pushes.go @@ -134,11 +134,9 @@ func pushToOAPI(push imagepush.Push) oapi.Push { CreatedAt: push.CreatedAt, CompletedAt: push.CompletedAt, } - if push.Layers > 0 { + if push.Status == oapi.PushStatus(imagepush.StatusPushed) { layers := push.Layers out.Layers = &layers - } - if push.Bytes > 0 { bytes := push.Bytes out.Bytes = &bytes } diff --git a/cmd/api/api/pushes_test.go b/cmd/api/api/pushes_test.go index ec4daca8..bdfcb93f 100644 --- a/cmd/api/api/pushes_test.go +++ b/cmd/api/api/pushes_test.go @@ -45,9 +45,6 @@ func (f *fakePushManager) ListPushes(_ context.Context) ([]imagepush.Push, error return f.pushes, nil } -func (f *fakePushManager) WaitForPush(_ context.Context, _ string) error { return nil } - -func (f *fakePushManager) InProgressDigests() []string { return nil } func (f *fakePushManager) LiveCacheManifestDigests() []string { return nil } func TestCreatePush_MapsRequestAndCredentials(t *testing.T) { diff --git a/lib/imagepush/imagepush.go b/lib/imagepush/imagepush.go index 236602d5..dbcc2cfb 100644 --- a/lib/imagepush/imagepush.go +++ b/lib/imagepush/imagepush.go @@ -8,11 +8,13 @@ package imagepush import ( + "bytes" "context" "crypto/sha256" + "encoding/base64" "encoding/hex" "errors" - "strings" + "fmt" "time" "github.com/google/go-containerregistry/pkg/authn" @@ -24,6 +26,11 @@ const ( StatusPushing = "pushing" StatusPushed = "pushed" StatusFailed = "failed" + + // pushTimeout bounds a single registry export so a wedged registry cannot + // pin a queue slot forever; with a bounded concurrency pool one hung push + // would otherwise block every later push. + pushTimeout = 30 * time.Minute ) var ( @@ -71,7 +78,22 @@ func credFingerprint(c *authn.AuthConfig) string { if !credsPresent(c) { return "" } - sum := sha256.Sum256([]byte(strings.Join([]string{c.Username, c.Password, c.Auth, c.IdentityToken, c.RegistryToken}, "\x00"))) + // Normalize the precomputed base64 "user:pass" Auth shorthand into its + // username/password parts so the same login supplied either way hashes + // identically (AuthConfig.UnmarshalJSON already expands it, but a config + // built in code may not have gone through JSON). + username, password := c.Username, c.Password + if c.Auth != "" { + if decoded, err := base64.StdEncoding.DecodeString(c.Auth); err == nil { + if i := bytes.IndexByte(decoded, ':'); i >= 0 { + username, password = string(decoded[:i]), string(decoded[i+1:]) + } + } + } + // IdentityToken and RegistryToken are distinct auth modes (token/registry- + // scoped) and are kept as-is; they can carry a different identity than the + // basic-auth pair. + sum := sha256.Sum256([]byte(fmt.Sprintf("%s\x00%s\x00%s\x00%s", username, password, c.IdentityToken, c.RegistryToken))) return hex.EncodeToString(sum[:]) } @@ -95,12 +117,6 @@ type ImageResolver interface { GetImage(ctx context.Context, name string) (*images.Image, error) } -// StatusEvent represents a terminal status change for push notifications. -type StatusEvent struct { - Status string - Err error -} - // Manager orchestrates push jobs. type Manager interface { // CreatePush validates the request, persists a queued job, and enqueues it. @@ -113,13 +129,8 @@ type Manager interface { GetPush(ctx context.Context, id string) (*Push, error) // ListPushes returns all pushes, newest first. ListPushes(ctx context.Context) ([]Push, error) - // WaitForPush blocks until the push reaches a terminal state (pushed or - // failed) or the context is cancelled. - WaitForPush(ctx context.Context, id string) error - // InProgressDigests returns the manifest digests of queued and pushing - // jobs so the OCI cache GC can keep their blobs alive mid-push. - InProgressDigests() []string - // LiveCacheManifestDigests implements ocicachegc.RootsProvider by - // delegating to InProgressDigests. + // LiveCacheManifestDigests implements ocicachegc.RootsProvider: the + // manifest digests of queued and pushing jobs, so the OCI cache GC keeps + // their blobs alive mid-push. LiveCacheManifestDigests() []string } diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index 5f8c56cd..95f38662 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -32,9 +32,6 @@ type manager struct { mu sync.Mutex inflight map[string]inflightPush // key = pushKey(digest, target, insecure) - - subscriberMu sync.RWMutex - subscribers map[string][]chan StatusEvent // keyed by push ID } // NewManager creates a push manager. provider may be nil, in which case @@ -49,12 +46,11 @@ func NewManager(p *paths.Paths, resolver ImageResolver, provider registrypush.Pr } m := &manager{ - paths: p, - resolver: resolver, - provider: provider, - queue: queue.New(maxConcurrent), - inflight: make(map[string]inflightPush), - subscribers: make(map[string][]chan StatusEvent), + paths: p, + resolver: resolver, + provider: provider, + queue: queue.New(maxConcurrent), + inflight: make(map[string]inflightPush), } m.recoverInterruptedPushes() @@ -114,63 +110,56 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error // a concurrent request for the same digest+target cannot slip in between // and create a duplicate job. The write is one small fsync'd file; keeping // it under the lock is what lets the dedup path hand back a durable record, - // and it only briefly stalls InProgressDigests — cheap next to the registry - // I/O that dominates a push. + // and it only briefly stalls the GC live-digest read — cheap next to the + // registry I/O that dominates a push. var meta *pushMetadata - for { - m.mu.Lock() - if existing, ok := m.inflight[key]; ok { - // Merge only when the in-flight job runs under the same credentials - // as the request. The manager never stores credential values, so it - // compares fingerprints: a request that borrowed credentials cannot - // merge into an anonymous in-flight push (its auth would be silently - // dropped), an anonymous request cannot merge into a credentialed one - // (it would silently inherit another caller's login), and two - // requests that borrowed different logins cannot merge either — one - // would run under the other caller's auth, and an instance can serve - // more than one principal. Surface the conflict instead so the caller - // can retry once the in-flight job completes or match its credentials. - if existing.credFingerprint != fingerprint { - m.mu.Unlock() - return nil, fmt.Errorf("%w: a push of %s to %s is already in flight with different credentials; retry once it completes or match its credentials", ErrCredentialConflict, img.Digest, dstRef.String()) - } - id := existing.id + m.mu.Lock() + if existing, ok := m.inflight[key]; ok { + // Merge only when the in-flight job runs under the same credentials + // as the request. The manager never stores credential values, so it + // compares fingerprints: a request that borrowed credentials cannot + // merge into an anonymous in-flight push (its auth would be silently + // dropped), an anonymous request cannot merge into a credentialed one + // (it would silently inherit another caller's login), and two + // requests that borrowed different logins cannot merge either — one + // would run under the other caller's auth, and an instance can serve + // more than one principal. Surface the conflict instead so the caller + // can retry once the in-flight job completes or match its credentials. + if existing.credFingerprint != fingerprint { m.mu.Unlock() - push, err := m.GetPush(ctx, id) - if errors.Is(err, ErrNotFound) { - // The job's terminal record could not be persisted and its - // directory was dropped; the queue completion hook releases the - // inflight entry moments later. Wait for that entry (not just the - // key) to go away, then retry the dedup: a concurrent waiter that - // got here first may already have registered a successor, which - // this retry merges into instead of surfacing a bare ErrNotFound - // from a create call. - if err := m.waitForInflightRelease(ctx, key, id); err != nil { - return nil, err - } - continue - } - return push, err - } - - meta = &pushMetadata{ - ID: cuid2.Generate(), - Status: StatusQueued, - Image: img.Name, - Digest: img.Digest, - Target: dstRef.String(), - Insecure: req.Insecure, - HadCredentials: credsPresent(req.Credentials), - CreatedAt: time.Now(), + return nil, fmt.Errorf("%w: a push of %s to %s is already in flight with different credentials; retry once it completes or match its credentials", ErrCredentialConflict, img.Digest, dstRef.String()) } - if err := writeMetadata(m.paths, meta); err != nil { - m.mu.Unlock() - return nil, fmt.Errorf("write initial metadata: %w", err) + id := existing.id + m.mu.Unlock() + push, err := m.GetPush(ctx, id) + if errors.Is(err, ErrNotFound) { + // The job's terminal record could not be persisted and its + // directory was dropped, so the in-flight entry points at nothing + // readable. This is a rare mid-finalization window; surface a + // clear retryable error rather than orchestrating this caller into + // a successor job that does not yet exist. A retry once the entry + // drops creates a fresh job. + return nil, fmt.Errorf("%w: push job %s is being finalized after a record write failure; retry", ErrNotFound, id) } - m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest, credFingerprint: fingerprint} + return push, err + } + + meta = &pushMetadata{ + ID: cuid2.Generate(), + Status: StatusQueued, + Image: img.Name, + Digest: img.Digest, + Target: dstRef.String(), + Insecure: req.Insecure, + HadCredentials: credsPresent(req.Credentials), + CreatedAt: time.Now(), + } + if err := writeMetadata(m.paths, meta); err != nil { m.mu.Unlock() - break + return nil, fmt.Errorf("write initial metadata: %w", err) } + m.inflight[key] = inflightPush{id: meta.ID, digest: meta.Digest, credFingerprint: fingerprint} + m.mu.Unlock() metaCopy := *meta queuePos := m.queue.Enqueue(key, func() { @@ -185,9 +174,13 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error } func (m *manager) executePush(ctx context.Context, meta *pushMetadata, provider registrypush.Provider) { - // Contain panics in the job goroutine: record a failed terminal and - // notify waiters instead of leaving the job stuck as pushing. The queue - // slot is released by its own deferred completion. + // Bound each export so a wedged registry cannot pin a queue slot forever. + ctx, cancel := context.WithTimeout(ctx, pushTimeout) + defer cancel() + + // Contain panics in the job goroutine: record a failed terminal instead + // of leaving the job stuck as pushing. The queue slot is released by its + // own deferred completion. defer func() { if r := recover(); r != nil { fmt.Fprintf(os.Stderr, "Warning: push %s to %s panicked: %v\n", meta.ID, meta.Target, r) @@ -196,10 +189,7 @@ func (m *manager) executePush(ctx context.Context, meta *pushMetadata, provider meta.Status = StatusFailed meta.Error = &errorMsg meta.CompletedAt = &now - if err := m.writeTerminal(meta); err != nil { - os.RemoveAll(m.paths.PushDir(meta.ID)) - } - m.notify(meta.ID, StatusFailed, fmt.Errorf("push panicked: %v", r)) + m.persistTerminal(meta) } }() @@ -225,25 +215,21 @@ func (m *manager) executePush(ctx context.Context, meta *pushMetadata, provider } meta.CompletedAt = &now + m.persistTerminal(meta) +} + +// persistTerminal writes a terminal status and, if that write fails, drops the +// record so GetPush/ListPushes do not surface a half-written job. Both the +// normal completion path and the panic handler use it so they agree on what +// "couldn't persist" means; the actual push outcome only reaches the log. +func (m *manager) persistTerminal(meta *pushMetadata) { if err := m.writeTerminal(meta); err != nil { - // The outcome cannot be recorded: drop the record and report the job - // as failed with the persistence problem, so WaitForPush and GetPush - // agree instead of diverging into success-then-not-found. The actual - // push outcome goes to the log. fmt.Fprintf(os.Stderr, "Warning: push %s to %s finished as %s but the job record could not be persisted: %v\n", meta.ID, meta.Target, strings.ToLower(meta.Status), err) os.RemoveAll(m.paths.PushDir(meta.ID)) persistErr := fmt.Errorf("job record could not be persisted: %w", err) errorMsg := persistErr.Error() meta.Status = StatusFailed meta.Error = &errorMsg - m.notify(meta.ID, StatusFailed, persistErr) - return - } - - if pushErr != nil { - m.notify(meta.ID, StatusFailed, pushErr) - } else { - m.notify(meta.ID, StatusPushed, nil) } } @@ -272,30 +258,6 @@ func (m *manager) releaseInflight(key string) func() { } } -// waitForInflightRelease blocks until the key's torn-down inflight entry is -// dropped — or replaced by a successor job a concurrent create registered -// first, which the caller then merges into by retrying the dedup. Waiting on -// the entry's id rather than the key's absence is what keeps a second waiter -// from parking until the successor finishes and then starting a duplicate -// push. The queue releases the key's active slot before it runs the -// completion hook, so once the torn-down entry is gone a fresh Enqueue for -// the key starts immediately. The caller's context bounds the wait. -func (m *manager) waitForInflightRelease(ctx context.Context, key, tornDownID string) error { - for { - m.mu.Lock() - existing, ok := m.inflight[key] - m.mu.Unlock() - if !ok || existing.id != tornDownID { - return nil - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(5 * time.Millisecond): - } - } -} - func (m *manager) GetPush(ctx context.Context, id string) (*Push, error) { if err := ctx.Err(); err != nil { return nil, err @@ -305,10 +267,7 @@ func (m *manager) GetPush(ctx context.Context, id string) (*Push, error) { return nil, err } - push := meta.toPush() - if meta.Status == StatusQueued { - push.QueuePosition = m.queue.GetPosition(pushKey(meta.Digest, meta.Target, meta.Insecure)) - } + push := m.toPushWithPosition(meta) return push, nil } @@ -323,68 +282,23 @@ func (m *manager) ListPushes(ctx context.Context) ([]Push, error) { pushes := make([]Push, 0, len(metas)) for _, meta := range metas { - push := meta.toPush() - if meta.Status == StatusQueued { - push.QueuePosition = m.queue.GetPosition(pushKey(meta.Digest, meta.Target, meta.Insecure)) - } - pushes = append(pushes, *push) + pushes = append(pushes, *m.toPushWithPosition(meta)) } return pushes, nil } -// WaitForPush blocks until the push reaches a terminal state (pushed or -// failed) or the context is cancelled. -func (m *manager) WaitForPush(ctx context.Context, id string) error { - push, err := m.GetPush(ctx, id) - if err != nil { - return err - } - - switch push.Status { - case StatusPushed: - return nil - case StatusFailed: - return pushError(push) - } - - ch := make(chan StatusEvent, 1) - m.subscribe(id, ch) - defer m.unsubscribe(id, ch) - - // Re-check after subscribing to close the race window. - push, err = m.GetPush(ctx, id) - if err != nil { - return err - } - switch push.Status { - case StatusPushed: - return nil - case StatusFailed: - return pushError(push) - } - - select { - case event := <-ch: - if event.Status == StatusPushed { - return nil - } - if event.Err != nil { - return fmt.Errorf("push failed: %w", event.Err) - } - return fmt.Errorf("push failed") - case <-ctx.Done(): - return ctx.Err() - } -} - -func pushError(push *Push) error { - if push.Error != nil { - return fmt.Errorf("push failed: %s", *push.Error) +// toPushWithPosition projects a stored record to its domain form and, for a +// queued job, enriches it with the live pending-queue position. GetPush and +// ListPushes share this so the projection cannot drift between them. +func (m *manager) toPushWithPosition(meta *pushMetadata) *Push { + push := meta.toPush() + if meta.Status == StatusQueued { + push.QueuePosition = m.queue.GetPosition(pushKey(meta.Digest, meta.Target, meta.Insecure)) } - return fmt.Errorf("push failed") + return push } -func (m *manager) InProgressDigests() []string { +func (m *manager) inProgressDigests() []string { m.mu.Lock() defer m.mu.Unlock() @@ -404,7 +318,7 @@ func (m *manager) InProgressDigests() []string { // LiveCacheManifestDigests implements ocicachegc.RootsProvider so in-flight // push digests are treated as live alongside the OCI layout index. func (m *manager) LiveCacheManifestDigests() []string { - return m.InProgressDigests() + return m.inProgressDigests() } func (m *manager) recoverInterruptedPushes() { @@ -460,7 +374,6 @@ func (m *manager) recoverInterruptedPushes() { // failRecovered marks a recovered job failed. If the status cannot be // persisted, the record is removed instead of being left permanently queued. -// Subscribers are notified so a WaitForPush racing the close does not hang. func (m *manager) failRecovered(meta *pushMetadata, reason string) { meta.Status = StatusFailed meta.Error = &reason @@ -470,40 +383,4 @@ func (m *manager) failRecovered(meta *pushMetadata, reason string) { fmt.Fprintf(os.Stderr, "Warning: dropping unrecoverable push record %s: %v\n", meta.ID, err) os.RemoveAll(m.paths.PushDir(meta.ID)) } - m.notify(meta.ID, StatusFailed, errors.New(reason)) -} - -func (m *manager) subscribe(id string, ch chan StatusEvent) { - m.subscriberMu.Lock() - defer m.subscriberMu.Unlock() - m.subscribers[id] = append(m.subscribers[id], ch) -} - -func (m *manager) unsubscribe(id string, ch chan StatusEvent) { - m.subscriberMu.Lock() - defer m.subscriberMu.Unlock() - - subs := m.subscribers[id] - for i, sub := range subs { - if sub == ch { - m.subscribers[id] = append(subs[:i], subs[i+1:]...) - break - } - } - if len(m.subscribers[id]) == 0 { - delete(m.subscribers, id) - } -} - -func (m *manager) notify(id, status string, err error) { - m.subscriberMu.RLock() - defer m.subscriberMu.RUnlock() - - event := StatusEvent{Status: status, Err: err} - for _, ch := range m.subscribers[id] { - select { - case ch <- event: - default: - } - } } diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index d77e9145..d88dac86 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -2,6 +2,7 @@ package imagepush import ( "context" + "encoding/base64" "errors" "fmt" "io" @@ -162,17 +163,32 @@ func testManager(t *testing.T, maxConcurrent int, provider registrypush.Provider return mgr, digest } +// waitTerminal polls GetPush until the push reaches a terminal (pushed or +// failed) state and returns it. It replaces the removed WaitForPush surface: +// pushes complete asynchronously and tests observe the result by polling. +func waitTerminal(t *testing.T, mgr Manager, id string) *Push { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for { + got, err := mgr.GetPush(context.Background(), id) + if err != nil { + t.Fatalf("GetPush %s: %v", id, err) + } + if got.Status == StatusPushed || got.Status == StatusFailed { + return got + } + if time.Now().After(deadline) { + t.Fatalf("push %s never reached a terminal state (status=%s)", id, got.Status) + } + time.Sleep(5 * time.Millisecond) + } +} + // mustPushed waits for the push to reach a terminal pushed state and returns // it, failing the test otherwise. func mustPushed(t *testing.T, mgr Manager, id string) *Push { t.Helper() - if err := mgr.WaitForPush(context.Background(), id); err != nil { - t.Fatalf("WaitForPush %s: %v", id, err) - } - got, err := mgr.GetPush(context.Background(), id) - if err != nil { - t.Fatalf("GetPush %s: %v", id, err) - } + got := waitTerminal(t, mgr, id) if got.Status != StatusPushed { t.Fatalf("push %s status = %s, want pushed (error: %v)", id, got.Status, got.Error) } @@ -226,8 +242,8 @@ func TestCreatePushEndToEnd(t *testing.T) { } // No in-flight digests once done. - if digests := mgr.InProgressDigests(); len(digests) != 0 { - t.Errorf("InProgressDigests = %v, want empty", digests) + if digests := mgr.(*manager).inProgressDigests(); len(digests) != 0 { + t.Errorf("inProgressDigests = %v, want empty", digests) } } @@ -250,14 +266,12 @@ func TestCreatePushDedupesInFlight(t *testing.T) { t.Errorf("duplicate push got new ID %s, want %s", second.ID, first.ID) } - if digests := mgr.InProgressDigests(); len(digests) != 1 || digests[0] != digest { - t.Errorf("InProgressDigests = %v, want [%s]", digests, digest) + if digests := mgr.(*manager).inProgressDigests(); len(digests) != 1 || digests[0] != digest { + t.Errorf("inProgressDigests = %v, want [%s]", digests, digest) } close(gate) - if err := mgr.WaitForPush(context.Background(), first.ID); err != nil { - t.Fatalf("WaitForPush: %v", err) - } + mustPushed(t, mgr, first.ID) } func TestCreatePushQueuesBehindConcurrencyLimit(t *testing.T) { @@ -306,12 +320,8 @@ func TestCreatePushQueuesBehindConcurrencyLimit(t *testing.T) { } close(gate) - if err := mgr.WaitForPush(context.Background(), first.ID); err != nil { - t.Fatalf("WaitForPush first: %v", err) - } - if err := mgr.WaitForPush(context.Background(), second.ID); err != nil { - t.Fatalf("WaitForPush second: %v", err) - } + mustPushed(t, mgr, first.ID) + mustPushed(t, mgr, second.ID) } func TestCreatePushRejectsInvalidRequests(t *testing.T) { @@ -377,15 +387,7 @@ func TestCreatePushFailureRecorded(t *testing.T) { t.Fatalf("CreatePush: %v", err) } - err = mgr.WaitForPush(context.Background(), push.ID) - if err == nil { - t.Fatal("WaitForPush should fail for a failed push") - } - - got, err := mgr.GetPush(context.Background(), push.ID) - if err != nil { - t.Fatalf("GetPush: %v", err) - } + got := waitTerminal(t, mgr, push.ID) if got.Status != StatusFailed { t.Errorf("status = %s, want failed", got.Status) } @@ -404,9 +406,7 @@ func TestListPushesNewestFirst(t *testing.T) { if err != nil { t.Fatalf("CreatePush a: %v", err) } - if err := mgr.WaitForPush(context.Background(), first.ID); err != nil { - t.Fatalf("WaitForPush a: %v", err) - } + mustPushed(t, mgr, first.ID) second, err := mgr.CreatePush(context.Background(), PushRequest{ Image: "myapp:v1", Target: host + "/export/b:v1", Insecure: true, @@ -414,9 +414,7 @@ func TestListPushesNewestFirst(t *testing.T) { if err != nil { t.Fatalf("CreatePush b: %v", err) } - if err := mgr.WaitForPush(context.Background(), second.ID); err != nil { - t.Fatalf("WaitForPush b: %v", err) - } + mustPushed(t, mgr, second.ID) pushes, err := mgr.ListPushes(context.Background()) if err != nil { @@ -495,14 +493,12 @@ func TestCreatePushDedupesConcurrently(t *testing.T) { } } - if digests := mgr.InProgressDigests(); len(digests) != 1 || digests[0] != digest { - t.Errorf("InProgressDigests = %v, want [%s]", digests, digest) + if digests := mgr.(*manager).inProgressDigests(); len(digests) != 1 || digests[0] != digest { + t.Errorf("inProgressDigests = %v, want [%s]", digests, digest) } close(gate) - if err := mgr.WaitForPush(context.Background(), ids[0]); err != nil { - t.Fatalf("WaitForPush: %v", err) - } + mustPushed(t, mgr, ids[0]) } func TestCreatePushCredentialConflict(t *testing.T) { @@ -539,12 +535,8 @@ func TestCreatePushCredentialConflict(t *testing.T) { close(gateA) close(gateB) - if err := mgr.WaitForPush(context.Background(), seeded.ID); err != nil { - t.Fatalf("WaitForPush seeded: %v", err) - } - if err := mgr.WaitForPush(context.Background(), seeded2.ID); err != nil { - t.Fatalf("WaitForPush seeded 2: %v", err) - } + mustPushed(t, mgr, seeded.ID) + mustPushed(t, mgr, seeded2.ID) // The conflicted requests must not have created duplicate jobs: only the // two seeds exist. @@ -592,131 +584,7 @@ func TestCreatePushCredentialMismatch(t *testing.T) { } close(gate) - if err := mgr.WaitForPush(context.Background(), seeded.ID); err != nil { - t.Fatalf("WaitForPush: %v", err) - } -} - -func TestCreatePushDedupSurvivesTornDownKey(t *testing.T) { - mgr, digest := testManager(t, 1, nil, nil) - host := openRegistry(t) - target := host + "/export/app:v1" - - dstRef, err := name.ParseReference(target, name.Insecure) - if err != nil { - t.Fatalf("ParseReference: %v", err) - } - key := pushKey(digest, dstRef.String(), true) - - // Simulate the persist-failure teardown window: the job's record is gone - // from disk but its inflight entry is still registered, released by the - // queue completion hook moments later. - m := mgr.(*manager) - m.mu.Lock() - m.inflight[key] = inflightPush{id: "ghost", digest: digest} - m.mu.Unlock() - go func() { - time.Sleep(50 * time.Millisecond) - m.mu.Lock() - delete(m.inflight, key) - m.mu.Unlock() - }() - - // The dedup path must wait out the torn-down key and create a fresh job, - // not surface ErrNotFound from a create. - push, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: target, Insecure: true}) - if err != nil { - t.Fatalf("CreatePush: %v", err) - } - if push.ID == "ghost" { - t.Fatal("CreatePush returned the torn-down job") - } - mustPushed(t, mgr, push.ID) -} - -func TestCreatePushDedupWaitersMergeIntoSuccessor(t *testing.T) { - mgr, digest := testManager(t, 1, nil, nil) - host := openRegistry(t) - target := host + "/export/app:v1" - - dstRef, err := name.ParseReference(target, name.Insecure) - if err != nil { - t.Fatalf("ParseReference: %v", err) - } - key := pushKey(digest, dstRef.String(), true) - - // Two concurrent creates racing the same torn-down key: one must create - // the successor job and the other must merge into it — not wait out the - // successor and then start a duplicate push. - m := mgr.(*manager) - m.mu.Lock() - m.inflight[key] = inflightPush{id: "ghost", digest: digest} - m.mu.Unlock() - go func() { - time.Sleep(50 * time.Millisecond) - m.mu.Lock() - delete(m.inflight, key) - m.mu.Unlock() - }() - - ids := make([]string, 2) - errs := make([]error, 2) - var wg sync.WaitGroup - for i := range ids { - wg.Add(1) - go func(i int) { - defer wg.Done() - push, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: target, Insecure: true}) - if push != nil { - ids[i] = push.ID - } - errs[i] = err - }(i) - } - wg.Wait() - - for i := range ids { - if errs[i] != nil { - t.Fatalf("CreatePush #%d: %v", i, errs[i]) - } - } - if ids[0] != ids[1] { - t.Errorf("concurrent creates got IDs %s and %s, want one shared successor job", ids[0], ids[1]) - } - mustPushed(t, mgr, ids[0]) - - pushes, err := mgr.ListPushes(context.Background()) - if err != nil { - t.Fatalf("ListPushes: %v", err) - } - if len(pushes) != 1 { - t.Errorf("len(pushes) = %d, want 1 (no duplicate after the successor)", len(pushes)) - } -} - -func TestWaitForPushCancellation(t *testing.T) { - mgr, _ := testManager(t, 1, nil, nil) - host, gate := gatedRegistry(t) - - push, err := mgr.CreatePush(context.Background(), PushRequest{Image: "myapp:v1", Target: host + "/export/app:v1", Insecure: true}) - if err != nil { - t.Fatalf("CreatePush: %v", err) - } - - ctx, cancel := context.WithCancel(context.Background()) - errCh := make(chan error, 1) - go func() { errCh <- mgr.WaitForPush(ctx, push.ID) }() - cancel() - if err := <-errCh; !errors.Is(err, context.Canceled) { - t.Errorf("WaitForPush err = %v, want context.Canceled", err) - } - - // Let the in-flight job finish so its writes land before the fixture's - // TempDir cleanup. - close(gate) - if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { - t.Fatalf("WaitForPush after cancel: %v", err) - } + mustPushed(t, mgr, seeded.ID) } func TestInProgressDigestsDedupesAcrossTargets(t *testing.T) { @@ -735,8 +603,8 @@ func TestInProgressDigestsDedupesAcrossTargets(t *testing.T) { pushes = append(pushes, push.ID) } - if digests := mgr.InProgressDigests(); len(digests) != 1 || digests[0] != digest { - t.Errorf("InProgressDigests = %v, want [%s]", digests, digest) + if digests := mgr.(*manager).inProgressDigests(); len(digests) != 1 || digests[0] != digest { + t.Errorf("inProgressDigests = %v, want [%s]", digests, digest) } // Drain the gated jobs so their writes land before the fixture's TempDir @@ -766,9 +634,7 @@ func TestRecoveryDedupesSameKey(t *testing.T) { t.Fatalf("NewManager: %v", err) } - if err := mgr.WaitForPush(context.Background(), "older"); err != nil { - t.Fatalf("WaitForPush older: %v", err) - } + mustPushed(t, mgr, "older") got, err := mgr.GetPush(context.Background(), "newer") if err != nil { @@ -780,9 +646,9 @@ func TestRecoveryDedupesSameKey(t *testing.T) { if got.Error == nil || !strings.Contains(*got.Error, "duplicate of push job older") { t.Errorf("newer error = %v, want duplicate-of-older explanation", got.Error) } - // WaitForPush on the superseded job surfaces the failure rather than hanging. - if err := mgr.WaitForPush(context.Background(), "newer"); err == nil { - t.Error("WaitForPush on superseded job should fail") + // The superseded job surfaces the failure rather than hanging. + if got := waitTerminal(t, mgr, "newer"); got.Status != StatusFailed { + t.Errorf("superseded job status = %s, want failed", got.Status) } } @@ -802,15 +668,6 @@ func TestSequentialSameKeyPushesAllComplete(t *testing.T) { } } -func TestWaitForPushNotFound(t *testing.T) { - mgr, _ := testManager(t, 1, nil, nil) - - err := mgr.WaitForPush(context.Background(), "missing") - if !errors.Is(err, ErrNotFound) { - t.Errorf("err = %v, want ErrNotFound", err) - } -} - // erroringProvider always fails, proving a push that succeeds used the // request's borrowed credentials instead of the manager default. type erroringProvider struct{} @@ -867,9 +724,7 @@ func TestCredentialsNeverPersisted(t *testing.T) { if err != nil { t.Fatalf("CreatePush: %v", err) } - if err := mgr.WaitForPush(context.Background(), push.ID); err != nil { - t.Fatalf("WaitForPush: %v", err) - } + mustPushed(t, mgr, push.ID) data, err := os.ReadFile(p.PushMetadata(push.ID)) if err != nil { @@ -949,14 +804,12 @@ func TestCreatePushMissingBlobs(t *testing.T) { if err != nil { t.Fatalf("CreatePush: %v", err) } - err = mgr.WaitForPush(context.Background(), push.ID) - if err == nil { - t.Fatal("WaitForPush should fail when cache blobs are missing") + got := waitTerminal(t, mgr, push.ID) + if got.Status != StatusFailed { + t.Fatalf("status = %s, want failed (error: %v)", got.Status, got.Error) } - // Depending on timing the failure is observed via the live event (typed) - // or via persisted metadata (string), so accept both forms. - if !errors.Is(err, ocicache.ErrNotFound) && !strings.Contains(err.Error(), ocicache.ErrNotFound.Error()) { - t.Errorf("err = %v, want ocicache.ErrNotFound", err) + if got.Error == nil || !strings.Contains(*got.Error, ocicache.ErrNotFound.Error()) { + t.Errorf("error = %v, want ocicache.ErrNotFound", got.Error) } } @@ -987,14 +840,12 @@ func TestRecoveryFailsWhenBlobsReclaimed(t *testing.T) { t.Fatalf("NewManager: %v", err) } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - err = mgr.WaitForPush(ctx, "recovered-missing-blobs") - if err == nil { - t.Fatal("WaitForPush should fail when cache blobs were reclaimed") + got := waitTerminal(t, mgr, "recovered-missing-blobs") + if got.Status != StatusFailed { + t.Fatalf("status = %s, want failed (error: %v)", got.Status, got.Error) } - if !errors.Is(err, ocicache.ErrNotFound) && !strings.Contains(err.Error(), ocicache.ErrNotFound.Error()) { - t.Errorf("err = %v, want ocicache.ErrNotFound", err) + if got.Error == nil || !strings.Contains(*got.Error, ocicache.ErrNotFound.Error()) { + t.Errorf("error = %v, want ocicache.ErrNotFound", got.Error) } } @@ -1017,23 +868,13 @@ func TestExecutePushContainsPanic(t *testing.T) { t.Fatalf("CreatePush: %v", err) } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - err = mgr.WaitForPush(ctx, push.ID) - if err == nil { - t.Fatal("WaitForPush should fail for a panicked push") - } - if !strings.Contains(err.Error(), "panicked") { - t.Errorf("err = %v, want panic explanation", err) - } - - got, err := mgr.GetPush(context.Background(), push.ID) - if err != nil { - t.Fatalf("GetPush: %v", err) - } + got := waitTerminal(t, mgr, push.ID) if got.Status != StatusFailed { t.Errorf("status = %s, want failed", got.Status) } + if got.Error == nil || !strings.Contains(*got.Error, "panicked") { + t.Errorf("error = %v, want panic explanation", got.Error) + } } func TestRecoveryTreatsInsecureAsDistinctKey(t *testing.T) { @@ -1090,3 +931,25 @@ func TestRecoverySweepsOrphanDirs(t *testing.T) { } mustPushed(t, mgr, "real") } + +func TestCredFingerprintNormalizesAuth(t *testing.T) { + // The same login supplied as Username/Password and as the precomputed + // base64 "user:pass" Auth shorthand must hash identically, so in-flight + // dedup does not report a false credential conflict between the two forms. + basic := &authn.AuthConfig{Username: "pusher", Password: "hunter2"} + shorthand := &authn.AuthConfig{ + Auth: base64.StdEncoding.EncodeToString([]byte("pusher:hunter2")), + } + + basicFp := credFingerprint(basic) + if basicFp == "" { + t.Fatal("basic-auth fingerprint should be non-empty") + } + shorthandFp := credFingerprint(shorthand) + if shorthandFp != basicFp { + t.Errorf("Auth-shorthand fingerprint %q != basic %q", shorthandFp, basicFp) + } + if credFingerprint(nil) != "" || credFingerprint(&authn.AuthConfig{}) != "" { + t.Error("anonymous configs should share the empty fingerprint") + } +} diff --git a/lib/imagepush/storage.go b/lib/imagepush/storage.go index a269f374..e231bf07 100644 --- a/lib/imagepush/storage.go +++ b/lib/imagepush/storage.go @@ -85,6 +85,15 @@ func writeMetadata(p *paths.Paths, meta *pushMetadata) error { return fmt.Errorf("rename metadata: %w", err) } + // Sync the directory so the rename itself is durable: the file is fsync'd + // and renamed above, but without a directory sync a crash right after the + // rename can still lose the directory entry. Best-effort — a directory + // sync failure is not worth failing the write over. + if dir, err := os.Open(dir); err == nil { + _ = dir.Sync() + _ = dir.Close() + } + return nil } From bddeebc3b30f6483c4d4a2034119eb72c1d29052 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:18:01 +0000 Subject: [PATCH 20/21] Fix push API quality issues --- cmd/api/api/pushes.go | 7 ++++++- cmd/api/api/pushes_test.go | 16 ++++++++++++++++ cmd/api/main.go | 3 --- lib/imagepush/manager.go | 13 ++++--------- lib/imagepush/manager_test.go | 16 ++++++++-------- 5 files changed, 34 insertions(+), 21 deletions(-) diff --git a/cmd/api/api/pushes.go b/cmd/api/api/pushes.go index 1b813a85..391b8555 100644 --- a/cmd/api/api/pushes.go +++ b/cmd/api/api/pushes.go @@ -41,6 +41,11 @@ func (s *ApiService) CreatePush(ctx context.Context, request oapi.CreatePushRequ Code: "not_found", Message: "image not found", }, nil + case errors.Is(err, imagepush.ErrNotFound): + return oapi.CreatePush409JSONResponse{ + Code: "conflict", + Message: err.Error(), + }, nil case errors.Is(err, imagepush.ErrImageNotReady): return oapi.CreatePush409JSONResponse{ Code: "image_not_ready", @@ -134,7 +139,7 @@ func pushToOAPI(push imagepush.Push) oapi.Push { CreatedAt: push.CreatedAt, CompletedAt: push.CompletedAt, } - if push.Status == oapi.PushStatus(imagepush.StatusPushed) { + if push.Status == imagepush.StatusPushed { layers := push.Layers out.Layers = &layers bytes := push.Bytes diff --git a/cmd/api/api/pushes_test.go b/cmd/api/api/pushes_test.go index bdfcb93f..5bff473e 100644 --- a/cmd/api/api/pushes_test.go +++ b/cmd/api/api/pushes_test.go @@ -178,6 +178,22 @@ func TestCreatePush_ErrorStatusMapping(t *testing.T) { } } +func TestCreatePush_FinalizationConflict(t *testing.T) { + t.Parallel() + + svc := &ApiService{PushManager: &fakePushManager{ + createErr: fmt.Errorf("%w: push job is being finalized; retry", imagepush.ErrNotFound), + }} + resp, err := svc.CreatePush(context.Background(), oapi.CreatePushRequestObject{ + Body: &oapi.CreatePushRequest{Image: "alpine:latest", Target: "registry.example.com/app:v1"}, + }) + require.NoError(t, err) + got, ok := resp.(oapi.CreatePush409JSONResponse) + require.True(t, ok) + require.Equal(t, "conflict", got.Code) + require.Contains(t, got.Message, "retry") +} + func TestGetPush_NotFound(t *testing.T) { t.Parallel() diff --git a/cmd/api/main.go b/cmd/api/main.go index 50a6a4e6..8ec068c0 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -127,9 +127,6 @@ type compositeOCICacheRoots []ocicachegc.RootsProvider func (c compositeOCICacheRoots) LiveCacheManifestDigests() []string { out := make([]string, 0) for _, roots := range c { - if roots == nil { - continue - } out = append(out, roots.LiveCacheManifestDigests()...) } return out diff --git a/lib/imagepush/manager.go b/lib/imagepush/manager.go index 95f38662..b6dedeb6 100644 --- a/lib/imagepush/manager.go +++ b/lib/imagepush/manager.go @@ -112,7 +112,6 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error // it under the lock is what lets the dedup path hand back a durable record, // and it only briefly stalls the GC live-digest read — cheap next to the // registry I/O that dominates a push. - var meta *pushMetadata m.mu.Lock() if existing, ok := m.inflight[key]; ok { // Merge only when the in-flight job runs under the same credentials @@ -144,7 +143,7 @@ func (m *manager) CreatePush(ctx context.Context, req PushRequest) (*Push, error return push, err } - meta = &pushMetadata{ + meta := &pushMetadata{ ID: cuid2.Generate(), Status: StatusQueued, Image: img.Name, @@ -298,7 +297,9 @@ func (m *manager) toPushWithPosition(meta *pushMetadata) *Push { return push } -func (m *manager) inProgressDigests() []string { +// LiveCacheManifestDigests implements ocicachegc.RootsProvider so in-flight +// push digests are treated as live alongside the OCI layout index. +func (m *manager) LiveCacheManifestDigests() []string { m.mu.Lock() defer m.mu.Unlock() @@ -315,12 +316,6 @@ func (m *manager) inProgressDigests() []string { return digests } -// LiveCacheManifestDigests implements ocicachegc.RootsProvider so in-flight -// push digests are treated as live alongside the OCI layout index. -func (m *manager) LiveCacheManifestDigests() []string { - return m.inProgressDigests() -} - func (m *manager) recoverInterruptedPushes() { // A crash between the push-dir MkdirAll and the metadata rename leaves an // empty dir no listing can read and no recovery can act on; sweep it now, diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index d88dac86..0f6df3ee 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -242,8 +242,8 @@ func TestCreatePushEndToEnd(t *testing.T) { } // No in-flight digests once done. - if digests := mgr.(*manager).inProgressDigests(); len(digests) != 0 { - t.Errorf("inProgressDigests = %v, want empty", digests) + if digests := mgr.LiveCacheManifestDigests(); len(digests) != 0 { + t.Errorf("LiveCacheManifestDigests = %v, want empty", digests) } } @@ -266,8 +266,8 @@ func TestCreatePushDedupesInFlight(t *testing.T) { t.Errorf("duplicate push got new ID %s, want %s", second.ID, first.ID) } - if digests := mgr.(*manager).inProgressDigests(); len(digests) != 1 || digests[0] != digest { - t.Errorf("inProgressDigests = %v, want [%s]", digests, digest) + if digests := mgr.LiveCacheManifestDigests(); len(digests) != 1 || digests[0] != digest { + t.Errorf("LiveCacheManifestDigests = %v, want [%s]", digests, digest) } close(gate) @@ -493,8 +493,8 @@ func TestCreatePushDedupesConcurrently(t *testing.T) { } } - if digests := mgr.(*manager).inProgressDigests(); len(digests) != 1 || digests[0] != digest { - t.Errorf("inProgressDigests = %v, want [%s]", digests, digest) + if digests := mgr.LiveCacheManifestDigests(); len(digests) != 1 || digests[0] != digest { + t.Errorf("LiveCacheManifestDigests = %v, want [%s]", digests, digest) } close(gate) @@ -603,8 +603,8 @@ func TestInProgressDigestsDedupesAcrossTargets(t *testing.T) { pushes = append(pushes, push.ID) } - if digests := mgr.(*manager).inProgressDigests(); len(digests) != 1 || digests[0] != digest { - t.Errorf("inProgressDigests = %v, want [%s]", digests, digest) + if digests := mgr.LiveCacheManifestDigests(); len(digests) != 1 || digests[0] != digest { + t.Errorf("LiveCacheManifestDigests = %v, want [%s]", digests, digest) } // Drain the gated jobs so their writes land before the fixture's TempDir From a4350d0703b8af19edb19d3f74afdfa954ba0a16 Mon Sep 17 00:00:00 2001 From: chruffins <23645059+chruffins@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:28:00 +0000 Subject: [PATCH 21/21] Wait for inflight push cleanup in tests --- lib/imagepush/manager_test.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/imagepush/manager_test.go b/lib/imagepush/manager_test.go index 0f6df3ee..8f9b1ffb 100644 --- a/lib/imagepush/manager_test.go +++ b/lib/imagepush/manager_test.go @@ -184,6 +184,20 @@ func waitTerminal(t *testing.T, mgr Manager, id string) *Push { } } +func waitNoInflight(t *testing.T, mgr Manager) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for { + if digests := mgr.LiveCacheManifestDigests(); len(digests) == 0 { + return + } + if time.Now().After(deadline) { + t.Fatalf("in-flight digests never cleared: %v", mgr.LiveCacheManifestDigests()) + } + time.Sleep(5 * time.Millisecond) + } +} + // mustPushed waits for the push to reach a terminal pushed state and returns // it, failing the test otherwise. func mustPushed(t *testing.T, mgr Manager, id string) *Push { @@ -242,9 +256,7 @@ func TestCreatePushEndToEnd(t *testing.T) { } // No in-flight digests once done. - if digests := mgr.LiveCacheManifestDigests(); len(digests) != 0 { - t.Errorf("LiveCacheManifestDigests = %v, want empty", digests) - } + waitNoInflight(t, mgr) } func TestCreatePushDedupesInFlight(t *testing.T) {