diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index adcaf05f..8ff70223 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -342,7 +342,26 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + // Checked first: it wraps the original create error, so a later + // errors.Is case would match the cause and hide the pending vGPU cleanup. + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image) + message := fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.Err, vgpuPending.InstanceID) + innerCode := "vgpu_retained_instance" + if !vgpuPending.Retained { + message = fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) + innerCode = "vgpu_unretained_instance" + } + return oapi.CreateInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(vgpuPending.InstanceID), + }, + }, nil case errors.Is(err, instances.ErrImageNotReady): return oapi.CreateInstance400JSONResponse{ Code: "image_not_ready", @@ -798,7 +817,26 @@ func (s *ApiService) StartInstance(ctx context.Context, request oapi.StartInstan result, err := s.InstanceManager.StartInstance(ctx, inst.Id, startReq) if err != nil { + var vgpuPending *instances.VGPUCleanupPendingError switch { + // Checked first: it wraps the original start error, so a later + // errors.Is case would match the cause and hide the pending vGPU cleanup. + case errors.As(err, &vgpuPending): + log.ErrorContext(ctx, "failed to start instance", "error", err) + message := fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it or retry start to release it", vgpuPending.Err, vgpuPending.InstanceID) + innerCode := "vgpu_retained_instance" + if !vgpuPending.Retained { + message = fmt.Sprintf("failed to start instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID) + innerCode = "vgpu_unretained_instance" + } + return oapi.StartInstance500JSONResponse{ + Code: "vgpu_cleanup_pending", + Message: message, + InnerError: &oapi.ErrorDetail{ + Code: lo.ToPtr(innerCode), + Message: lo.ToPtr(vgpuPending.InstanceID), + }, + }, nil case errors.Is(err, instances.ErrInvalidState): return oapi.StartInstance409JSONResponse{ Code: "invalid_state", @@ -1205,6 +1243,9 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if inst.GPUMdevUUID != "" { gpu.MdevUuid = lo.ToPtr(inst.GPUMdevUUID) } + if inst.GPUDevicePath != "" { + gpu.DevicePath = lo.ToPtr(inst.GPUDevicePath) + } oapiInst.Gpu = gpu } diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index d23e7c21..4f04692e 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -16,6 +16,7 @@ import ( "github.com/kernel/hypeman/lib/instances" "github.com/kernel/hypeman/lib/instances/phasetracking" mw "github.com/kernel/hypeman/lib/middleware" + "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/paths" restartpolicy "github.com/kernel/hypeman/lib/restart-policy" @@ -46,6 +47,73 @@ func TestGetInstance_NotFound(t *testing.T) { require.Error(t, err) } +type createErrorInstanceManager struct { + instances.Manager + err error +} + +func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) { + return nil, m.err +} + +// A retained-assignment error must win over the mapping of the create error +// it wraps, or the response omits the instance the caller has to delete. +func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: network.ErrNameExists, + }} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{Image: "test-image"}, + }) + require.NoError(t, err) + + pending, ok := resp.(oapi.CreateInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, network.ErrNameExists.Error(), + "the underlying create failure must survive the cleanup guidance") + assert.Contains(t, pending.Message, "delete it to retry") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) +} + +func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: network.ErrNameExists, + }} + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{Image: "test-image"}, + }) + require.NoError(t, err) + + pending, ok := resp.(oapi.CreateInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, network.ErrNameExists.Error(), + "the underlying create failure must survive the cleanup guidance") + assert.Contains(t, pending.Message, "startup reconcile") + assert.NotContains(t, pending.Message, "delete") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) +} + func TestCreateInstance_AutoPullImage(t *testing.T) { t.Parallel() if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) { @@ -779,6 +847,68 @@ func (m *errActionInstanceManager) RestoreSnapshot(context.Context, string, stri return nil, m.err } +// A retained-assignment error must win over the mapping of the start error +// it wraps, or the response omits the pending vGPU cleanup the caller has to +// resolve. +func TestStartInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) { + t.Parallel() + + resolved := &instances.Instance{ + StoredMetadata: instances.StoredMetadata{Id: "inst-1", Name: "inst-1"}, + State: instances.StateStopped, + } + + t.Run("retained", func(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Retained: true, + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, rerr) + + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "inst-1") + assert.Contains(t, pending.Message, instances.ErrInsufficientResources.Error(), + "the underlying start failure must survive the cleanup guidance") + assert.Contains(t, pending.Message, "delete it or retry start") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_retained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) + }) + + t.Run("unretained", func(t *testing.T) { + t.Parallel() + svc := newTestService(t) + svc.InstanceManager = &errActionInstanceManager{Manager: svc.InstanceManager, err: &instances.VGPUCleanupPendingError{ + InstanceID: "inst-1", + Err: fmt.Errorf("create vGPU for profile p: %w", instances.ErrInsufficientResources), + }} + + resp, rerr := svc.StartInstance(mw.WithResolvedInstance(ctx(), resolved.Id, resolved), oapi.StartInstanceRequestObject{Id: resolved.Id}) + require.NoError(t, rerr) + + pending, ok := resp.(oapi.StartInstance500JSONResponse) + require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp) + assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code) + assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved") + assert.Contains(t, pending.Message, "startup reconcile") + assert.NotContains(t, pending.Message, "delete") + require.NotNil(t, pending.InnerError) + require.NotNil(t, pending.InnerError.Code) + assert.Equal(t, "vgpu_unretained_instance", *pending.InnerError.Code) + require.NotNil(t, pending.InnerError.Message) + assert.Equal(t, "inst-1", *pending.InnerError.Message) + }) +} + func TestInstanceActions_ImageNotFoundMapsTo404(t *testing.T) { t.Parallel() diff --git a/cmd/api/main.go b/cmd/api/main.go index 98d7be72..c1c7f1dc 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -172,6 +172,63 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta }, logger), nil } +func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, time.Duration, error) { + allInstances, err := instanceManager.ListInstancesForReconcile(ctx) + if err != nil { + return nil, 0, err + } + protected := make(map[string]struct{}) + var retryAfter time.Duration + for _, inst := range allInstances { + if inst.GPUDevicePath == "" { + continue + } + if inst.HypervisorPID != nil && instances.HypervisorProcessIdentityExists(*inst.HypervisorPID, inst.HypervisorStartTime, inst.HypervisorBootID, inst.SocketPath) { + protected[inst.GPUDevicePath] = struct{}{} + continue + } + if inst.GPUAssignedAt == nil { + continue + } + remaining := instances.VGPUAssignmentStartupGracePeriod - time.Since(*inst.GPUAssignedAt) + if remaining <= 0 { + continue + } + protected[inst.GPUDevicePath] = struct{}{} + if retryAfter == 0 || remaining < retryAfter { + retryAfter = remaining + } + } + return protected, retryAfter, nil +} + +func reconcileVGPUs(ctx context.Context, instanceManager instances.Manager, logger *slog.Logger) { + protected, retryAfter, err := liveInstanceVGPUDevicePaths(ctx, instanceManager) + if err != nil { + // Operator-actionable: vendor VFIO reconciliation stays disabled + // host-wide (and releases fail closed on the same inventory) until + // the unreadable instance metadata is repaired. + logger.Error("failed to list instances for vGPU reconcile protection; reconciling mdev only", "error", err) + protected = nil + retryAfter = 0 + } + if err := devices.ReconcileVGPUs(ctx, protected); err != nil { + logger.Warn("failed to reconcile vGPU devices", "error", err) + } + if retryAfter <= 0 { + return + } + go func() { + timer := time.NewTimer(retryAfter) + defer timer.Stop() + select { + case <-ctx.Done(): + case <-timer.C: + reconcileVGPUs(ctx, instanceManager, logger) + } + }() +} + func run() error { // Load config early for OTel initialization // Config path can be specified via CONFIG_PATH env var or defaults to platform-specific locations @@ -362,12 +419,9 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile mdev devices (clears orphaned vGPUs from previous runs) - logger.Info("Reconciling mdev devices...") - if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil { - // Log but don't fail - mdev cleanup is best-effort - logger.Warn("failed to reconcile mdev devices", "error", err) - } + // Reconcile vGPU devices (clears orphaned vGPUs from previous runs) + logger.Info("Reconciling vGPU devices...") + reconcileVGPUs(ctx, app.InstanceManager, logger) // Wire up resource validator for aggregate limit checking // This enables the instance manager to validate CPU, memory, network, and GPU diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go index 34dbba42..217e9db8 100644 --- a/cmd/api/main_test.go +++ b/cmd/api/main_test.go @@ -2,15 +2,18 @@ package main import ( "bytes" + "context" "net/http" "net/http/httptest" "net/url" + "os/exec" "testing" "time" "github.com/getkin/kin-openapi/openapi3filter" "github.com/go-chi/chi/v5" "github.com/golang-jwt/jwt/v5" + "github.com/kernel/hypeman/lib/instances" mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/oapi" nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware" @@ -338,3 +341,38 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) { }) } } + +type vgpuReconcileManagerStub struct { + instances.Manager + list []instances.Instance +} + +func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) { + return s.list, nil +} + +func TestLiveInstanceVGPUDevicePathsBoundsStartupProtection(t *testing.T) { + dead := exec.Command("true") + require.NoError(t, dead.Run()) + deadPID := dead.Process.Pid + recent := time.Now().Add(-time.Minute) + stale := time.Now().Add(-instances.VGPUAssignmentStartupGracePeriod - time.Minute) + + manager := vgpuReconcileManagerStub{list: []instances.Instance{ + {StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUAssignedAt: &recent}}, + {StoredMetadata: instances.StoredMetadata{Id: "orphaned", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", GPUAssignedAt: &stale}}, + {StoredMetadata: instances.StoredMetadata{Id: "legacy", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.6"}}, + {StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.7", HypervisorPID: &deadPID}}, + {StoredMetadata: instances.StoredMetadata{Id: "stale-pid-booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.8", HypervisorPID: &deadPID, GPUAssignedAt: &recent}}, + }} + + protected, retryAfter, err := liveInstanceVGPUDevicePaths(context.Background(), manager) + require.NoError(t, err) + require.Positive(t, retryAfter) + require.LessOrEqual(t, retryAfter, instances.VGPUAssignmentStartupGracePeriod) + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.6") + assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.7") + assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.8") +} diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 7873aa35..1f1a8277 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) { if framework == devices.VGPUFrameworkNone { return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" } - if framework == devices.VGPUFrameworkVendorVFIO { - // CreateVGPU rejects vendor VFIO until the instance lifecycle - // integration lands. - return "vGPU test requires the vendor VFIO instance lifecycle integration", "" - } // Check for available profiles profiles, err := devices.ListGPUProfiles() diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index dfdc9fca..14ccf8c2 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -51,6 +51,10 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc return result, nil } +func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) { + return m.ListInstances(ctx, nil) +} + func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) { return nil, nil } diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index 72fe3b94..a4ccd2db 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -73,10 +73,7 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic MdevUUID: mdev.UUID, }, nil case VGPUFrameworkVendorVFIO: - // The instance lifecycle does not yet persist vendor VFIO assignments - // durably or guard their release against live claims, so keep the - // backend out of the create path until that integration lands. - return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle") + return hostVendorVFIO.create(ctx, profileName, instanceID) default: return nil, fmt.Errorf("vGPU framework not available") } diff --git a/lib/instances/create.go b/lib/instances/create.go index 2dedd081..c471fbb0 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -56,7 +56,7 @@ func wrapCreateVGPUErr(profile string, err error) error { if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) { return fmt.Errorf("%w: %w", ErrInvalidRequest, err) } - return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err) + return fmt.Errorf("create vGPU for profile %s: %w", profile, err) } // generateVsockCID converts first 8 chars of instance ID to a unique CID @@ -262,11 +262,23 @@ func (m *manager) createInstance( var gpuFramework devices.VGPUFramework var gpuDevicePath string var gpuMdevUUID string - - // Setup cleanup stack early so device attachment errors trigger cleanup + var gpuAssignedAt *time.Time + var stored *StoredMetadata + var retainedVGPU *StoredMetadata + + // Setup cleanup stack early so device attachment errors trigger cleanup. + // When rollback cannot release a vGPU assignment, report whether its + // retention record was persisted. The wrapping defer is registered first + // so it runs after cu.Clean has attempted to retain the metadata. + vgpuPersisted := false + defer func() { + if retErr != nil && retainedVGPU != nil { + retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuPersisted, Err: retErr} + } + }() cu := cleanup.Make(func() { log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id) - m.deleteInstanceData(id) + vgpuPersisted = m.cleanupFailedCreate(ctx, id, retainedVGPU) }) defer cu.Clean() @@ -283,8 +295,23 @@ func (m *manager) createInstance( // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) - gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id) + gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id) if err != nil { + stub := StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: time.Now(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + DataDir: m.paths.InstanceDir(id), + } + if starterErr == nil { + stub.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) + } + retainedVGPU = retainedVGPUFromCreateError(stub, m.nowUTC(), err) log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } @@ -292,6 +319,8 @@ func (m *manager) createInstance( gpuFramework = gpuDevice.Framework gpuDevicePath = gpuDevice.SysfsPath gpuMdevUUID = gpuDevice.MdevUUID + assignedAt := m.nowUTC() + gpuAssignedAt = &assignedAt log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID) // Add vGPU cleanup to stack @@ -303,8 +332,28 @@ func (m *manager) createInstance( MdevUUID: gpuDevice.MdevUUID, InstanceID: id, } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) + retainedVGPU = stored + if retainedVGPU == nil { + retainedVGPU = &StoredMetadata{ + Id: id, + Name: req.Name, + Image: req.Image, + ResolvedImage: resolvedImageRef, + Platform: imageInfo.Platform, + CreatedAt: time.Now(), + HypervisorType: hvType, + HypervisorVersion: hvVersion, + SocketPath: m.paths.InstanceSocket(id, starter.SocketName()), + DataDir: m.paths.InstanceDir(id), + GPUProfile: gpuDevice.ProfileName, + GPUFramework: gpuDevice.Framework, + GPUDevicePath: gpuDevice.SysfsPath, + GPUMdevUUID: gpuDevice.MdevUUID, + GPUAssignedAt: gpuAssignedAt, + } + } } }) } @@ -340,7 +389,7 @@ func (m *manager) createInstance( } // 11. Create instance metadata - stored := &StoredMetadata{ + stored = &StoredMetadata{ Id: id, Name: req.Name, Image: req.Image, @@ -375,6 +424,7 @@ func (m *manager) createInstance( GPUFramework: gpuFramework, GPUDevicePath: gpuDevicePath, GPUMdevUUID: gpuMdevUUID, + GPUAssignedAt: gpuAssignedAt, Entrypoint: req.Entrypoint, Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, @@ -574,6 +624,56 @@ func (m *manager) createInstance( return &finalInst, nil } +// cleanupFailedCreate reports whether the retention record for a vGPU +// assignment whose release failed during rollback was persisted. +func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool { + if retainedVGPU == nil { + m.deleteInstanceData(id) + return false + } + + log := logger.FromContext(ctx) + retentionSurvives := func() bool { + meta, err := m.loadMetadata(id) + if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" { + return true + } + if err := m.deleteInstanceData(id); err != nil { + log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err) + } + return false + } + if err := m.ensureDirectories(id); err != nil { + log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err) + return retentionSurvives() + } + // Retain identity fields so the instance lists as a recognizable, + // deletable record rather than a nameless phantom, but drop resource + // claims (network, volumes, devices) that rollback already released. + retained := StoredMetadata{ + Id: id, + Name: retainedVGPU.Name, + Image: retainedVGPU.Image, + ResolvedImage: retainedVGPU.ResolvedImage, + Platform: retainedVGPU.Platform, + CreatedAt: retainedVGPU.CreatedAt, + HypervisorType: retainedVGPU.HypervisorType, + HypervisorVersion: retainedVGPU.HypervisorVersion, + SocketPath: retainedVGPU.SocketPath, + DataDir: retainedVGPU.DataDir, + GPUProfile: retainedVGPU.GPUProfile, + GPUFramework: retainedVGPU.GPUFramework, + GPUDevicePath: retainedVGPU.GPUDevicePath, + GPUMdevUUID: retainedVGPU.GPUMdevUUID, + GPUAssignedAt: retainedVGPU.GPUAssignedAt, + } + if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err) + return retentionSurvives() + } + return true +} + // validateCreateRequest validates the create instance request. // The request is mutated in-place to persist normalized egress/credential policy fields. func validateCreateRequest(req *CreateInstanceRequest) error { diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go index e6e4f55c..05b3e9d8 100644 --- a/lib/instances/create_mdev_test.go +++ b/lib/instances/create_mdev_test.go @@ -63,7 +63,7 @@ func TestWrapCreateVGPUErr(t *testing.T) { { name: "other vGPU error", err: errors.New("boom"), - wantMessage: "create vGPU mdev for profile profile: boom", + wantMessage: "create vGPU for profile profile: boom", }, } { t.Run(tc.name, func(t *testing.T) { diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 9dc83711..cd7a0f19 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -152,7 +152,7 @@ func (m *manager) deleteInstanceWithOptions( if hadVGPUAssignment { log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) } - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { // Log error but continue with cleanup. log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } else if hadVGPUAssignment { diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index a9b918df..c74d352f 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -3,6 +3,7 @@ package instances import ( "context" "errors" + "net" "os" "path/filepath" "sync" @@ -195,6 +196,90 @@ func TestDeletePersistsVGPUReleaseBeforeTeardown(t *testing.T) { assert.Equal(t, restartpolicy.BlockedReasonManualStop, persisted.RestartStatus.BlockedReason) } +// A failed create whose vGPU release also failed retains a minimal +// GPU-fields-only stub, and the API tells the caller to delete it to retry +// the release. Exercise that recovery path against the exact stub shape +// cleanupFailedCreate writes. +func TestDeleteReleasesRetainedCreateStub(t *testing.T) { + p := paths.New(t.TempDir()) + var destroyed []devices.VGPUAssignment + m := &manager{ + paths: p, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + now: time.Now, + lifecycleEvents: newLifecycleSubscribers(), + destroyVGPU: func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }, + } + const id = "retained-stub" + require.NoError(t, m.ensureDirectories(id)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + require.Len(t, destroyed, 1) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", destroyed[0].DevicePath) + assert.Equal(t, id, destroyed[0].InstanceID) + _, err := m.loadMetadata(id) + require.Error(t, err, "retained stub must be fully deleted") +} + +func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) { + now := time.Now().UTC() + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + claimantID := "inst-live-claimant" + require.NoError(t, m.ensureDirectories(claimantID)) + pid := os.Getpid() + // Bind under /tmp: a t.TempDir()-derived path exceeds the macOS AF_UNIX + // path limit. + socketDir, err := os.MkdirTemp("/tmp", "hypeman-claimant-socket-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(socketDir) + }) + socketPath := filepath.Join(socketDir, "noop.sock") + listener, err := net.Listen("unix", socketPath) + require.NoError(t, err) + defer listener.Close() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + Name: claimantID, + Image: "test-image", + CreatedAt: now, + HypervisorType: lifecycleNoopHypervisorType, + HypervisorPID: &pid, + SocketPath: socketPath, + DataDir: m.paths.InstanceDir(claimantID), + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + }})) + + require.NoError(t, m.DeleteInstance(context.Background(), id)) + + _, err = m.loadMetadata(id) + require.Error(t, err, "deleted instance metadata should be gone") + claimant, err := m.loadMetadata(claimantID) + require.NoError(t, err) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", claimant.GPUDevicePath, "live claimant keeps its assignment") +} + func TestDeleteContinuesTeardownAfterFailedVGPURelease(t *testing.T) { m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) deviceManager := &recordingDeviceManager{} @@ -300,6 +385,20 @@ func (m *recordingDeviceManager) UnbindFromVFIO(ctx context.Context, id string) return nil } +func TestLifecycleNoopStandbyRejectsVendorVFIOVGPU(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFrameworkVendorVFIO + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + _, err = m.StandbyInstance(context.Background(), id, StandbyInstanceRequest{}) + require.ErrorIs(t, err, ErrInvalidState) + assert.ErrorContains(t, err, "standby is not supported for instances with vGPU attached") +} + func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) { t.Helper() diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 85f75975..16a6b419 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "sync" "time" @@ -27,6 +28,7 @@ import ( type Manager interface { ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) + ListInstancesForReconcile(ctx context.Context) ([]Instance, error) ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error) GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error) CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error) @@ -179,6 +181,8 @@ type manager struct { tracer trace.Tracer now func() time.Time writeFile func(string, []byte, os.FileMode) error + createVGPU func(context.Context, string, string) (*devices.VGPUDevice, error) + destroyVGPU func(context.Context, devices.VGPUAssignment) error deleteSnapshotFn func(context.Context, string) error egressProxy *egressproxy.Service egressProxyServiceOptions egressproxy.ServiceOptions @@ -277,6 +281,8 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa defaultHypervisor: defaultHypervisor, now: time.Now, writeFile: os.WriteFile, + createVGPU: devices.CreateVGPU, + destroyVGPU: devices.DestroyVGPU, meter: meter, tracer: tracer, guestMemoryPolicy: policy, @@ -696,6 +702,27 @@ func (m *manager) UpdateInstance(ctx context.Context, id string, req UpdateInsta return inst, err } +// ListInstancesForReconcile returns every instance's stored metadata or an +// invalid metadata error. It does not derive state: reconcile protection only +// needs raw metadata fields, and hydration would query the hypervisor of +// every instance on the host before the API serves. +func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) { + files, err := m.listMetadataFilesWithStatErrors(true) + if err != nil { + return nil, err + } + result := make([]Instance, 0, len(files)) + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + meta, err := m.loadMetadata(id) + if err != nil { + return nil, fmt.Errorf("load metadata for instance %s: %w", id, err) + } + result = append(result, Instance{StoredMetadata: meta.StoredMetadata}) + } + return result, nil +} + // ListInstances returns instances, optionally filtered by the given criteria. // Pass nil to return all instances. func (m *manager) ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) { diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go index 0608e78e..0ffd5f83 100644 --- a/lib/instances/process_identity_linux_test.go +++ b/lib/instances/process_identity_linux_test.go @@ -16,7 +16,9 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -351,6 +353,47 @@ func TestRefreshHypervisorPIDBackfillsStartTime(t *testing.T) { assert.Equal(t, hostBootID(), stored.HypervisorBootID) } +func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) { + socketPath := filepath.Join(t.TempDir(), "test.sock") + owner := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$") + owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath) + stdin, err := owner.StdinPipe() + require.NoError(t, err) + stdout, err := owner.StdoutPipe() + require.NoError(t, err) + require.NoError(t, owner.Start()) + t.Cleanup(func() { + _ = stdin.Close() + _ = owner.Process.Kill() + _ = owner.Wait() + }) + _, err = bufio.NewReader(stdout).ReadString('\n') + require.NoError(t, err) + + stale := exec.Command("sleep", "30") + require.NoError(t, stale.Start()) + t.Cleanup(func() { + _ = stale.Process.Kill() + _ = stale.Wait() + }) + + m := &manager{paths: paths.New(t.TempDir())} + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + stalePID := stale.Process.Pid + require.NoError(t, m.ensureDirectories("live-claimant")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "live-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + HypervisorPID: &stalePID, + SocketPath: socketPath, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath) + require.NoError(t, err) + assert.True(t, claimed) +} + func TestKillHypervisorSurvivesConcurrentReaper(t *testing.T) { socketPath := filepath.Join(t.TempDir(), "test.sock") process := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$") diff --git a/lib/instances/query.go b/lib/instances/query.go index afca9a48..f7005f95 100644 --- a/lib/instances/query.go +++ b/lib/instances/query.go @@ -988,7 +988,7 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) { return time.Time{}, false } -// listInstances returns all instances +// listInstances returns all instances, skipping metadata files that cannot be loaded. func (m *manager) listInstances(ctx context.Context) ([]Instance, error) { ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata") defer span.End() diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go index 8bb0bb46..ab3db29c 100644 --- a/lib/instances/query_test.go +++ b/lib/instances/query_test.go @@ -14,6 +14,34 @@ import ( "github.com/stretchr/testify/require" ) +func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + + require.NoError(t, m.ensureDirectories("valid")) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "valid", + Name: "valid", + CreatedAt: time.Now(), + DataDir: m.paths.InstanceDir("valid"), + }})) + require.NoError(t, m.ensureDirectories("invalid")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid"), []byte("{"), 0644)) + + listed, err := m.ListInstances(context.Background(), nil) + require.NoError(t, err) + require.Len(t, listed, 1) + + _, err = m.ListInstancesForReconcile(context.Background()) + require.Error(t, err) + assert.ErrorContains(t, err, "load metadata for instance invalid") + + require.NoError(t, os.Remove(m.paths.InstanceMetadata("invalid"))) + listed, err = m.ListInstancesForReconcile(context.Background()) + require.NoError(t, err) + require.Len(t, listed, 1) + assert.Equal(t, "valid", listed[0].Id) +} + func TestParseExitSentinelLine(t *testing.T) { t.Parallel() tests := []struct { diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 8963a264..cb6eaa7e 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -313,6 +313,7 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str restored.GPUFramework = sourceMeta.GPUFramework restored.GPUDevicePath = sourceMeta.GPUDevicePath restored.GPUMdevUUID = sourceMeta.GPUMdevUUID + restored.GPUAssignedAt = sourceMeta.GPUAssignedAt restored.HypervisorType = targetHypervisor restored.HypervisorVersion = targetHypervisorVersion restored.SocketPath = m.paths.InstanceSocket(id, starter.SocketName()) diff --git a/lib/instances/snapshot_test.go b/lib/instances/snapshot_test.go index c2bef2ef..bef6c0d2 100644 --- a/lib/instances/snapshot_test.go +++ b/lib/instances/snapshot_test.go @@ -113,6 +113,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { meta.GPUFramework = devices.VGPUFramework("future-framework") meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" meta.GPUMdevUUID = "retained-uuid" + assignedAt := time.Now().UTC().Truncate(time.Second) + meta.GPUAssignedAt = &assignedAt require.NoError(t, mgr.saveMetadata(meta)) _, err = mgr.RestoreSnapshot(ctx, sourceID, snapshot.Id, RestoreSnapshotRequest{ @@ -126,6 +128,8 @@ func TestRestoreSnapshotKeepsCurrentVGPUAssignment(t *testing.T) { assert.Equal(t, devices.VGPUFramework("future-framework"), restored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", restored.GPUDevicePath) assert.Equal(t, "retained-uuid", restored.GPUMdevUUID) + require.NotNil(t, restored.GPUAssignedAt) + assert.True(t, assignedAt.Equal(*restored.GPUAssignedAt)) } func TestStoppedSnapshotLifecycleAndForkAfterSourceDeletion(t *testing.T) { diff --git a/lib/instances/start.go b/lib/instances/start.go index a6c83245..3e164b39 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -5,7 +5,6 @@ import ( "fmt" "time" - "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/instances/phasetracking" "github.com/kernel/hypeman/lib/logger" @@ -54,7 +53,7 @@ func (m *manager) startInstance( // cannot leave on-disk metadata pointing at a device that is already // gone (matching releaseRetainedVGPULocked). if storedVGPUDevicePath(stored) != "" { - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) return nil, fmt.Errorf("release stale vGPU before start: %w", err) } @@ -64,6 +63,12 @@ func (m *manager) startInstance( } } + // Do not persist the previous VMM's identity with a new vGPU assignment. + stored.HypervisorPID = nil + stored.HypervisorStartTime = 0 + stored.HypervisorBootID = "" + rollbackMeta := *meta + // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" @@ -111,6 +116,16 @@ func (m *manager) startInstance( } // Setup cleanup stack for automatic rollback on errors + // Registered before cu.Clean so it runs after cleanup and can report a + // vGPU assignment that rollback failed to destroy, matching create's + // vgpu_cleanup_pending contract. + vgpuRetained := false + vgpuRetentionPersisted := false + defer func() { + if retErr != nil && vgpuRetained { + retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuRetentionPersisted, Err: retErr} + } + }() cu := cleanup.Make(func() {}) defer cu.Clean() @@ -162,27 +177,34 @@ func (m *manager) startInstance( // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { - log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile) - device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id) + log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) + device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) - return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) + if pendingDevice, ok := vgpuDevicePendingCleanup(err); ok { + assignedAt := m.nowUTC() + retentionMeta := rollbackMeta + setStoredVGPUDevice(&retentionMeta.StoredMetadata, pendingDevice, assignedAt) + wrapped := fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + if saveErr := m.saveMetadata(&retentionMeta); saveErr != nil { + log.ErrorContext(ctx, "failed to retain vGPU assignment after create rollback failure", "instance_id", id, "error", saveErr) + return nil, &VGPUCleanupPendingError{InstanceID: id, Err: fmt.Errorf("%w; retain assignment: %v", wrapped, saveErr)} + } + return nil, &VGPUCleanupPendingError{InstanceID: id, Retained: true, Err: wrapped} + } + return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } - setStoredVGPUDevice(stored, device) + assignedAt := m.nowUTC() + setStoredVGPUDevice(stored, device, assignedAt) log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { - log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID) - assignment := devices.VGPUAssignment{ - Framework: device.Framework, - DevicePath: device.SysfsPath, - MdevUUID: device.MdevUUID, - InstanceID: id, - } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) - } + vgpuRetained, vgpuRetentionPersisted = m.cleanupStartVGPU(ctx, id, device, assignedAt, rollbackMeta) }) + if err := m.saveMetadata(meta); err != nil { + log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err) + return nil, fmt.Errorf("save metadata after vGPU creation: %w", err) + } } // 5. Regenerate config disk with new network configuration diff --git a/lib/instances/stop.go b/lib/instances/stop.go index b23ff91c..9276b00d 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -262,7 +262,7 @@ func (m *manager) stopInstance( // 7. Release the vGPU assignment if present (frees the vGPU slot for other VMs). if storedVGPUDevicePath(stored) != "" { log.InfoContext(ctx, "destroying vGPU on stop", "instance_id", id, "uuid", stored.GPUMdevUUID) - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { // Log error but continue - vGPU cleanup is best-effort log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) } diff --git a/lib/instances/storage.go b/lib/instances/storage.go index f33a5962..d44dcbf2 100644 --- a/lib/instances/storage.go +++ b/lib/instances/storage.go @@ -177,8 +177,12 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti } } -// listMetadataFiles returns paths to all instance metadata files +// listMetadataFiles returns paths to all instance metadata files. func (m *manager) listMetadataFiles() ([]string, error) { + return m.listMetadataFilesWithStatErrors(false) +} + +func (m *manager) listMetadataFilesWithStatErrors(failOnStatError bool) ([]string, error) { guestsDir := m.paths.GuestsDir() // Ensure guests directory exists @@ -200,6 +204,8 @@ func (m *manager) listMetadataFiles() ([]string, error) { metaPath := filepath.Join(guestsDir, entry.Name(), "metadata.json") if _, err := os.Stat(metaPath); err == nil { metaFiles = append(metaFiles, metaPath) + } else if failOnStatError && !os.IsNotExist(err) { + return nil, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err) } } diff --git a/lib/instances/types.go b/lib/instances/types.go index ae810414..f9f33c3a 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -154,7 +154,8 @@ type StoredMetadata struct { GPUProfile string // vGPU profile name (e.g., "L40S-1Q") GPUFramework devices.VGPUFramework GPUDevicePath string - GPUMdevUUID string // populated for mdev-backed vGPUs + GPUMdevUUID string // populated for mdev-backed vGPUs + GPUAssignedAt *time.Time // set before hypervisor startup to bound crash recovery protection // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index a8ca6ace..2f9cfa7d 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,41 +2,222 @@ package instances import ( "context" + "errors" + "fmt" "path/filepath" + "time" "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/logger" ) -func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { +// VGPUAssignmentStartupGracePeriod bounds how long an assignment without a +// persisted hypervisor PID is treated as potentially live. +const VGPUAssignmentStartupGracePeriod = 5 * time.Minute + +// VGPUCleanupPendingError reports a failed create whose vGPU release also +// failed during rollback. When Retained is true, deleting the retained instance +// retries the release; otherwise startup reconciliation recovers the assignment. +type VGPUCleanupPendingError struct { + InstanceID string + Retained bool + Err error +} + +func (e *VGPUCleanupPendingError) Error() string { + if e.Retained { + return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID) + } + return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", e.Err, e.InstanceID) +} + +func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err } + +func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) { + create := m.createVGPU + if create == nil { + create = devices.CreateVGPU + } + return create(ctx, profileName, instanceID) +} + +func vgpuDevicePendingCleanup(err error) (*devices.VGPUDevice, bool) { + var pending *devices.VGPUCreateCleanupPendingError + if !errors.As(err, &pending) { + return nil, false + } + return &pending.Device, true +} + +// retainedVGPUFromCreateError fills stub with the pending device's assignment +// fields when err carries a failed device-layer cleanup. The caller provides +// identity fields on stub so the retained record lists as a recognizable, +// deletable instance. +func retainedVGPUFromCreateError(stub StoredMetadata, assignedAt time.Time, err error) *StoredMetadata { + device, ok := vgpuDevicePendingCleanup(err) + if !ok { + return nil + } + stub.GPUProfile = device.ProfileName + stub.GPUFramework = device.Framework + stub.GPUDevicePath = device.SysfsPath + stub.GPUMdevUUID = device.MdevUUID + stub.GPUAssignedAt = &assignedAt + return &stub +} + +func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error { + destroy := m.destroyVGPU + if destroy == nil { + destroy = devices.DestroyVGPU + } + return destroy(ctx, assignment) +} + +func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice, assignedAt time.Time) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath stored.GPUMdevUUID = device.MdevUUID + stored.GPUAssignedAt = &assignedAt } func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUFramework = devices.VGPUFrameworkNone stored.GPUDevicePath = "" stored.GPUMdevUUID = "" + stored.GPUAssignedAt = nil } -func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { +// cleanupStartVGPU wholesale-restores the pre-start metadata snapshot. The +// cleanup stack is LIFO, so cleanups registered after this one run before it +// and this restore would clobber anything they persisted; it is safe only +// while no such cleanup writes metadata and the instance lock serializes +// start. The snapshot is also a shallow copy (Phases shares its map), so it +// must be persisted before any Phases.Record on the live struct. Violating +// either invariant requires switching to targeted field restores. +// +// It reports whether the assignment was retained after a failed destroy and +// whether that retention record was persisted, so start can surface the +// pending cleanup as a typed error like create does. +func (m *manager) cleanupStartVGPU(ctx context.Context, instanceID string, device *devices.VGPUDevice, assignedAt time.Time, rollbackMeta metadata) (retained, persisted bool) { + logger.FromContext(ctx).DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID) + assignment := devices.VGPUAssignment{ + Framework: device.Framework, + DevicePath: device.SysfsPath, + MdevUUID: device.MdevUUID, + InstanceID: instanceID, + } + cleanupMeta := rollbackMeta + releaseErr := m.destroyVGPUAssignment(ctx, assignment) + if releaseErr != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", instanceID, "uuid", device.MdevUUID, "error", releaseErr) + setStoredVGPUDevice(&cleanupMeta.StoredMetadata, device, assignedAt) + retained = true + } + if err := m.saveMetadata(&cleanupMeta); err != nil { + message := "failed to save metadata after vGPU cleanup" + if releaseErr != nil { + message = "failed to retain vGPU assignment metadata after cleanup failure" + } + logger.FromContext(ctx).ErrorContext(ctx, message, "instance_id", instanceID, "error", err) + if !retained { + return false, false + } + // The mid-start save may already have persisted this assignment, in + // which case the on-disk record still points at the device and + // delete or a retried start can release it (matching create's + // retention-survives check). + if meta, loadErr := m.loadMetadata(instanceID); loadErr == nil && storedVGPUDevicePath(&meta.StoredMetadata) == device.SysfsPath { + return true, true + } + return true, false + } + return retained, retained +} + +func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { path := storedVGPUDevicePath(stored) if path != "" { - assignment := devices.VGPUAssignment{ - Framework: stored.GPUFramework, - DevicePath: path, - MdevUUID: stored.GPUMdevUUID, - InstanceID: stored.Id, + // Vendor VFIO VFs are reused across instances, so stale metadata can + // point at a path claimed by a live instance and the release must fail + // closed on an incomplete inventory. mdev UUIDs are unique and never + // reused, so skip the scan there — it would let one unreadable + // metadata file block every mdev release on the host. + claimed := false + if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO { + var err error + claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path) + if err != nil { + return err + } } - if err := devices.DestroyVGPU(ctx, assignment); err != nil { - return err + if claimed { + logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance", + "instance_id", stored.Id, "device_path", path) + } else { + assignment := devices.VGPUAssignment{ + Framework: stored.GPUFramework, + DevicePath: path, + MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, + } + if err := m.destroyVGPUAssignment(ctx, assignment); err != nil { + return err + } } } clearStoredVGPUDevice(stored) return nil } +// vgpuAssignmentClaimedByLiveInstance reports whether another live instance's +// stored metadata claims devicePath. It reads raw metadata instead of +// hydrating full instances: the scan runs on every vendor VFIO release, and +// deriving state would query the hypervisor of every instance on the host. +// A confirmed live claimant returns true. Unreadable metadata, a recent +// assignment without a PID, or unverifiable process ownership returns an error +// so the requester retains its assignment for a later retry. +func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) { + files, err := m.listMetadataFilesWithStatErrors(true) + if err != nil { + return false, fmt.Errorf("list instances for vGPU release check: %w", err) + } + for _, file := range files { + id := filepath.Base(filepath.Dir(file)) + if id == excludeID { + continue + } + meta, err := m.loadMetadata(id) + if err != nil { + return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err) + } + stored := &meta.StoredMetadata + if storedVGPUDevicePath(stored) != devicePath { + continue + } + if stored.HypervisorPID == nil { + if stored.GPUAssignedAt == nil || time.Since(*stored.GPUAssignedAt) >= VGPUAssignmentStartupGracePeriod { + continue + } + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: no persisted hypervisor PID", id, devicePath) + } + pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.HypervisorStartTime, stored.HypervisorBootID, stored.SocketPath) + if err != nil { + return false, fmt.Errorf("cannot confirm liveness of vGPU claimant %s on %s: %w", id, devicePath, err) + } + if pid > 0 { + return true, nil + } + // A dead PID with a recent assignment gets the same bounded grace as + // startup reconcile protection, so the two guards agree in the + // fail-closed direction while a mid-boot claimant hydrates. + if stored.GPUAssignedAt != nil && time.Since(*stored.GPUAssignedAt) < VGPUAssignmentStartupGracePeriod { + return false, fmt.Errorf("cannot confirm liveness of recent vGPU claimant %s on %s: recorded hypervisor is not running", id, devicePath) + } + } + return false, nil +} + // releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped // instance after a failed release during the original stop. It is a no-op // when no assignment is retained, and a failed retry only logs so the @@ -52,7 +233,7 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) { if storedVGPUDevicePath(stored) == "" { return } - if err := releaseStoredVGPU(ctx, stored); err != nil { + if err := m.releaseStoredVGPU(ctx, stored); err != nil { log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err) return } diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f2c4681..d5e988a4 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -2,17 +2,640 @@ package instances import ( "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" "testing" + "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + assignedAt := time.Now().UTC() + stored := &StoredMetadata{ + Id: "failed-create", + Name: "failed-create", + GPUProfile: "NVIDIA L40S-2Q", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUMdevUUID: "mdev-uuid", + GPUAssignedAt: &assignedAt, + NetworkEnabled: true, + IP: "192.0.2.1", + Volumes: []VolumeAttachment{{VolumeID: "volume"}}, + HypervisorType: "qemu", + DataDir: m.paths.InstanceDir("failed-create"), + } + + assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + + retained, err := m.loadMetadata(stored.Id) + require.NoError(t, err) + assert.Equal(t, stored.Id, retained.Id) + assert.Equal(t, stored.GPUFramework, retained.GPUFramework) + assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) + assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID) + assert.Equal(t, stored.GPUAssignedAt, retained.GPUAssignedAt) + // Identity fields survive so the retained record lists as a + // recognizable, deletable instance instead of a nameless phantom. + assert.Equal(t, stored.Name, retained.Name) + assert.Equal(t, stored.GPUProfile, retained.GPUProfile) + assert.Equal(t, stored.HypervisorType, retained.HypervisorType) + assert.Equal(t, stored.DataDir, retained.DataDir) + // Resource claims released by rollback stay dropped. + assert.False(t, retained.NetworkEnabled) + assert.Empty(t, retained.IP) + assert.Empty(t, retained.Volumes) +} + +func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("failed-create")) + + assert.False(t, m.cleanupFailedCreate(context.Background(), "failed-create", nil)) + _, err := m.loadMetadata("failed-create") + require.Error(t, err) +} + +func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, os.Mkdir(m.paths.InstanceMetadata(id), 0o755)) + + stored := &StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored)) + _, err := m.loadMetadata(id) + require.Error(t, err) +} + +func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m := &manager{paths: paths.New(t.TempDir())} + const id = "failed-create" + require.NoError(t, m.ensureDirectories(id)) + stored := &StoredMetadata{ + Id: id, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored})) + + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + assert.True(t, m.cleanupFailedCreate(context.Background(), id, stored)) + retained, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, stored.GPUFramework, retained.GPUFramework) + assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath) +} + +func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) { + t.Parallel() + + cause := errors.New("boot failed") + retained := &VGPUCleanupPendingError{InstanceID: "inst-1", Retained: true, Err: cause} + assert.ErrorIs(t, retained, cause) + assert.Equal(t, "boot failed; vGPU release failed during rollback, instance inst-1 retains the assignment", retained.Error()) + + unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause} + assert.ErrorIs(t, unpersisted, cause) + assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error()) +} + +func TestVGPUDevicePendingCleanup(t *testing.T) { + t.Parallel() + + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("rollback failed") + pending := &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + + wrapped := fmt.Errorf("create failed: %w", pending) + actual, ok := vgpuDevicePendingCleanup(wrapped) + require.True(t, ok) + assert.Equal(t, device, *actual) + + assignedAt := time.Now().UTC() + retained := retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1", Name: "named", Image: "img"}, assignedAt, wrapped) + require.NotNil(t, retained) + assert.Equal(t, "inst-1", retained.Id) + assert.Equal(t, "named", retained.Name, "identity fields must survive into the retention stub") + assert.Equal(t, "img", retained.Image) + assert.Equal(t, device.Framework, retained.GPUFramework) + assert.Equal(t, device.SysfsPath, retained.GPUDevicePath) + assert.Equal(t, assignedAt, *retained.GPUAssignedAt) + + actual, ok = vgpuDevicePendingCleanup(cause) + assert.False(t, ok) + assert.Nil(t, actual) + assert.Nil(t, retainedVGPUFromCreateError(StoredMetadata{Id: "inst-1"}, assignedAt, cause)) +} + +type startRetentionNetworkManager struct { + network.Manager + config network.NetworkConfig + releaseCalls int +} + +func (m *startRetentionNetworkManager) CreateAllocation(context.Context, network.AllocateRequest) (*network.NetworkConfig, error) { + config := m.config + return &config, nil +} + +func (m *startRetentionNetworkManager) ReleaseAllocation(context.Context, *network.Allocation) error { + m.releaseCalls++ + return nil +} + +func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) { + t.Helper() + m := &manager{ + paths: paths.New(t.TempDir()), + imageManager: readyFixtureImageManager{name: "test-image"}, + instanceLocks: sync.Map{}, + bootMarkerScans: sync.Map{}, + createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) { + return &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: profileName, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + }, nil + }, + destroyVGPU: destroy, + } + const id = "start-rollback" + require.NoError(t, m.ensureDirectories(id)) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: id, + Name: id, + Image: "test-image", + GPUProfile: "NVIDIA L40S-2Q", + HypervisorType: hypervisor.TypeQEMU, + SocketPath: m.paths.InstanceSocket(id, "noop.sock"), + DataDir: m.paths.InstanceDir(id), + }})) + return m, id +} + +func TestStartRetainsVGPUWhenCreateRollbackFails(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + networkManager := &startRetentionNetworkManager{config: network.NetworkConfig{ + IP: "192.0.2.20", + MAC: "02:00:00:00:00:20", + TAPDevice: "tap-new", + }} + m.networkManager = networkManager + + previousProgramStart := time.Now().Add(-time.Hour).UTC() + previousExitCode := 23 + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.NetworkEnabled = true + meta.IP = "192.0.2.10" + meta.MAC = "02:00:00:00:00:10" + meta.Entrypoint = []string{"old-entrypoint"} + meta.Cmd = []string{"old-command"} + meta.ProgramStartedAt = &previousProgramStart + meta.ExitCode = &previousExitCode + meta.ExitMessage = "previous exit" + require.NoError(t, m.saveMetadata(meta)) + + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: "NVIDIA L40S-2Q", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{ + Entrypoint: []string{"new-entrypoint"}, + Cmd: []string{"new-command"}, + }) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, device.Framework, stored.GPUFramework) + assert.Equal(t, device.SysfsPath, stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Equal(t, []string{"old-entrypoint"}, stored.Entrypoint) + assert.Equal(t, []string{"old-command"}, stored.Cmd) + assert.Equal(t, previousProgramStart, *stored.ProgramStartedAt) + assert.Equal(t, previousExitCode, *stored.ExitCode) + assert.Equal(t, "previous exit", stored.ExitMessage) + assert.Equal(t, "192.0.2.10", stored.IP) + assert.Equal(t, "02:00:00:00:00:10", stored.MAC) + assert.Equal(t, 1, networkManager.releaseCalls) +} + +func TestStartReportsUnretainedVGPUWhenRetentionSaveFails(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + ProfileType: "1148", + ProfileName: "NVIDIA L40S-2Q", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + cause := errors.New("create verification and rollback failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + return nil, &devices.VGPUCreateCleanupPendingError{Device: device, Err: cause} + } + + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.ErrorIs(t, err, cause) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, id, pending.InstanceID) + assert.False(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Empty(t, stored.GPUDevicePath, "retention save failed, so no assignment should be recorded") +} + +func TestStartDoesNotRestrictVGPUHypervisor(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return nil + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.HypervisorType = hypervisor.TypeCloudHypervisor + require.NoError(t, m.saveMetadata(meta)) + + cause := errors.New("create failed") + m.createVGPU = func(context.Context, string, string) (*devices.VGPUDevice, error) { + return nil, cause + } + + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{}) + assert.ErrorIs(t, err, cause) +} + +func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) { + var destroyed []devices.VGPUAssignment + m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error { + destroyed = append(destroyed, assignment) + return nil + }) + + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err := m.startInstance(context.Background(), id, StartInstanceRequest{}) + require.Error(t, err) + + require.Len(t, destroyed, 1) + assert.Equal(t, devices.VGPUAssignment{ + Framework: devices.VGPUFrameworkVendorVFIO, + DevicePath: "/sys/bus/pci/devices/0000:82:00.4", + InstanceID: id, + }, destroyed[0]) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile) + assert.Empty(t, stored.GPUFramework) + assert.Empty(t, stored.GPUDevicePath) + assert.Empty(t, stored.GPUMdevUUID) +} + +func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) { + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + stalePID := os.Getpid() + meta.HypervisorPID = &stalePID + meta.HypervisorStartTime = 1 + meta.HypervisorBootID = "previous-boot" + require.NoError(t, m.saveMetadata(meta)) + + t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing")) + _, err = m.startInstance(context.Background(), id, StartInstanceRequest{Entrypoint: []string{"new-entrypoint"}}) + require.Error(t, err) + var pending *VGPUCleanupPendingError + require.ErrorAs(t, err, &pending, "a retained rollback assignment must surface as vgpu_cleanup_pending") + assert.Equal(t, id, pending.InstanceID) + assert.True(t, pending.Retained) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.NotNil(t, stored.GPUAssignedAt) + assert.Nil(t, stored.HypervisorPID) + assert.Zero(t, stored.HypervisorStartTime) + assert.Empty(t, stored.HypervisorBootID) + assert.Empty(t, stored.Entrypoint) +} + +func TestCleanupStartVGPUReportsRetainedWhenMidStartSaveSurvives(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses directory permissions") + } + + m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error { + return errors.New("destroy failed") + }) + device := devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + assignedAt := time.Now().UTC() + + // The mid-start save already persisted the assignment. + meta, err := m.loadMetadata(id) + require.NoError(t, err) + rollbackMeta := *meta + setStoredVGPUDevice(&meta.StoredMetadata, &device, assignedAt) + require.NoError(t, m.saveMetadata(meta)) + + // The cleanup save fails, but the surviving on-disk record still points + // at the device, so retention must be reported as persisted. + instanceDir := filepath.Dir(m.paths.InstanceMetadata(id)) + require.NoError(t, os.Chmod(instanceDir, 0o555)) + t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) }) + + retained, persisted := m.cleanupStartVGPU(context.Background(), id, &device, assignedAt, rollbackMeta) + assert.True(t, retained) + assert.True(t, persisted, "a surviving mid-start save keeps the assignment recoverable via delete") +} + +func TestCleanupStartVGPURestoresMetadataAfterBootFailure(t *testing.T) { + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + return nil + }, + } + const id = "failed-start" + require.NoError(t, m.ensureDirectories(id)) + + previousStart := time.Now().Add(-time.Hour).UTC() + previousProgramStart := previousStart.Add(time.Second) + exitCode := 1 + rollbackMeta := metadata{StoredMetadata: StoredMetadata{ + Id: id, + GPUProfile: "NVIDIA L40S-2Q", + Entrypoint: []string{"old-entrypoint"}, + Cmd: []string{"old-command"}, + StartedAt: &previousStart, + ProgramStartedAt: &previousProgramStart, + ExitCode: &exitCode, + ExitMessage: "previous exit", + }} + + partial := rollbackMeta + partial.Entrypoint = []string{"new-entrypoint"} + partial.Cmd = []string{"new-command"} + partial.StartedAt = ptr(time.Now().UTC()) + partial.ProgramStartedAt = nil + partial.ExitCode = nil + partial.ExitMessage = "" + assignedAt := time.Now().UTC() + device := &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + setStoredVGPUDevice(&partial.StoredMetadata, device, assignedAt) + require.NoError(t, m.saveMetadata(&partial)) + + m.cleanupStartVGPU(context.Background(), id, device, assignedAt, rollbackMeta) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, rollbackMeta.Entrypoint, stored.Entrypoint) + assert.Equal(t, rollbackMeta.Cmd, stored.Cmd) + assert.Equal(t, rollbackMeta.StartedAt, stored.StartedAt) + assert.Equal(t, rollbackMeta.ProgramStartedAt, stored.ProgramStartedAt) + assert.Equal(t, rollbackMeta.ExitCode, stored.ExitCode) + assert.Equal(t, rollbackMeta.ExitMessage, stored.ExitMessage) + assert.Empty(t, stored.GPUDevicePath) + assert.Nil(t, stored.GPUAssignedAt) +} + +func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) +} + +func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("legacy-claimant")) + pid := os.Getpid() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "legacy-claimant", + Name: "legacy-claimant", + GPUMdevUUID: "legacy-uuid", + HypervisorPID: &pid, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid") + require.NoError(t, err) + assert.True(t, claimed) +} + +func TestVGPUAssignmentClaimedByLiveInstanceErrorsOnRecentNilPIDClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("booting-claimant")) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "booting-claimant", + Name: "booting-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) + assert.Contains(t, err.Error(), "booting-claimant") +} + +func TestVGPUAssignmentClaimedByLiveInstanceIgnoresStaleNilPIDClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("stale-claimant")) + assignedAt := time.Now().Add(-VGPUAssignmentStartupGracePeriod - time.Minute) + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "stale-claimant", + Name: "stale-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed) +} + +func TestVGPUAssignmentClaimedByLiveInstanceGracesRecentDeadPIDClaim(t *testing.T) { + m := &manager{paths: paths.New(t.TempDir())} + claimantID := "claimant-dead-pid" + require.NoError(t, m.ensureDirectories(claimantID)) + deadPID := 1<<22 - 1 + require.False(t, ProcessExists(deadPID)) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: claimantID, + HypervisorPID: &deadPID, + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + GPUAssignedAt: &assignedAt, + }})) + + // Same bounded grace as startup reconcile: a recent claim whose PID is + // dead fails closed instead of being treated as unclaimed. + _, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") + require.Error(t, err) + + // Past the grace period the dead claim no longer blocks the release. + stale := assignedAt.Add(-2 * VGPUAssignmentStartupGracePeriod) + meta, err := m.loadMetadata(claimantID) + require.NoError(t, err) + meta.GPUAssignedAt = &stale + require.NoError(t, m.saveMetadata(meta)) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "requester", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed) +} + +func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) { + t.Parallel() + + m := &manager{paths: paths.New(t.TempDir())} + require.NoError(t, m.ensureDirectories("dead-claimant")) + deadPID := 1 << 30 + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "dead-claimant", + Name: "dead-claimant", + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + HypervisorPID: &deadPID, + }})) + + claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4") + require.NoError(t, err) + assert.False(t, claimed, "a claim whose hypervisor is gone must not block the release") +} + +func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) { + t.Parallel() + + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil }, + } + require.NoError(t, m.ensureDirectories("invalid-instance")) + require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644)) + + stored := &StoredMetadata{ + Id: "mdev-instance", + GPUFramework: devices.VGPUFrameworkMdev, + GPUMdevUUID: "uuid-1", + GPUDevicePath: "/sys/bus/mdev/devices/uuid-1", + } + require.NoError(t, m.releaseStoredVGPU(context.Background(), stored), + "an unreadable metadata file must not block mdev releases") + assert.Empty(t, stored.GPUDevicePath) +} + +func TestReleaseStoredVGPURetainsRequesterOnAmbiguousClaim(t *testing.T) { + t.Parallel() + + const devicePath = "/sys/bus/pci/devices/0000:82:00.4" + m := &manager{ + paths: paths.New(t.TempDir()), + destroyVGPU: func(context.Context, devices.VGPUAssignment) error { + t.Fatal("destroyVGPU must not be called for an ambiguous claim") + return nil + }, + } + require.NoError(t, m.ensureDirectories("ambiguous-claimant")) + assignedAt := time.Now().UTC() + require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{ + Id: "ambiguous-claimant", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + GPUAssignedAt: &assignedAt, + }})) + + stored := &StoredMetadata{ + Id: "requester", + GPUFramework: devices.VGPUFrameworkVendorVFIO, + GPUDevicePath: devicePath, + } + err := m.releaseStoredVGPU(context.Background(), stored) + require.Error(t, err) + assert.Contains(t, err.Error(), "ambiguous-claimant") + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, devicePath, stored.GPUDevicePath) +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", storedVGPUDevicePath(&StoredMetadata{ - GPUDevicePath: "/sys/bus/mdev/devices/new-uuid", + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", storedVGPUDevicePath(&StoredMetadata{ + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", GPUMdevUUID: "legacy-uuid", })) assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{ @@ -24,11 +647,12 @@ func TestStoredVGPUDevicePath(t *testing.T) { func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { t.Parallel() + m := &manager{paths: paths.New(t.TempDir())} stored := &StoredMetadata{ GPUFramework: devices.VGPUFramework("future-framework"), GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", } - err := releaseStoredVGPU(context.Background(), stored) + err := m.releaseStoredVGPU(context.Background(), stored) assert.Error(t, err) assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) @@ -37,18 +661,19 @@ func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { func TestSetAndClearStoredVGPUDevice(t *testing.T) { t.Parallel() + assignedAt := time.Now().UTC() stored := &StoredMetadata{} setStoredVGPUDevice(stored, &devices.VGPUDevice{ - Framework: devices.VGPUFrameworkMdev, - SysfsPath: "/sys/bus/mdev/devices/new-uuid", - MdevUUID: "new-uuid", - }) - assert.Equal(t, devices.VGPUFrameworkMdev, stored.GPUFramework) - assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", stored.GPUDevicePath) - assert.Equal(t, "new-uuid", stored.GPUMdevUUID) + Framework: devices.VGPUFrameworkVendorVFIO, + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + }, assignedAt) + assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) + assert.Equal(t, assignedAt, *stored.GPUAssignedAt) clearStoredVGPUDevice(stored) assert.Empty(t, stored.GPUFramework) assert.Empty(t, stored.GPUDevicePath) assert.Empty(t, stored.GPUMdevUUID) + assert.Nil(t, stored.GPUAssignedAt) } diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go index bab6f06d..a4246479 100644 --- a/lib/instances/wait_test.go +++ b/lib/instances/wait_test.go @@ -32,6 +32,9 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) { return nil, nil } +func (s *stubManager) ListInstancesForReconcile(context.Context) ([]Instance, error) { + return nil, nil +} func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) { return nil, nil } diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index adb36a22..cdf1c72a 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -1184,7 +1184,10 @@ type InstanceHypervisor string // InstanceGPU GPU information attached to the instance type InstanceGPU struct { - // MdevUuid mdev device UUID + // DevicePath sysfs path of the assigned vGPU device + DevicePath *string `json:"device_path,omitempty"` + + // MdevUuid mdev device UUID (mdev hosts only) MdevUuid *string `json:"mdev_uuid,omitempty"` // Profile vGPU profile name @@ -17767,185 +17770,186 @@ var swaggerSpec = []string{ "nOnv+3uDwWCQlfvv7G/rfw983HSnt4N9iz5qkmIBB41ZHFIAWBMpQykLiUBvuSRKYUQMO1RjyK55s/iV", "sKPu83bqlf0816v8G3MVSmk7+NGvxXxcVmP4rFxduLX1uvuPrypETNrqrzYdx341us4VP0EBT6NQm0Rj", "fdoZjxUJrZ9OEpUXboYD8j27YHqPlqZuY3MVR3+mRCzQh5OTUlyAIBNbl7bFxEFKNKwDT661DFsrnAgr", - "R3NDJND7QP+saioFDfHWsT6LV3AuS9hwaIuruNyQ9GZWUGaWRvPJkjlV3NshmY/S1GeI6EcOG+T9++Oj", - "EnNgvLf5dPD0We/peHOvtxMONnt4c3uvt7WLB5Pt4Ml2Q2X49qlcN8/O8lqjvlLfLmR65EK3fRG1TYHz", - "lXPfBgNfUhbyy9LR4o3OLPZuIz9XdV+P6249BG82CFSdhZYapMQJHKAk0G2baOhKIegGL9veu8HmCi/b", - "SnkBg2uQv+9Eysydnklwz/zXcWHAxcUqj/Nm4hQG5LIuVlGr2Hl7og32d5/t734t0VzmwKoxVtnpHhe3", - "KRzJIb9WUhNcelzBR+OQEjtW3zAuZZvJ0Ol2smQL+BsO2kogb/a4VQZR04bt+sXIMvndkLl7XLIFIEzC", - "AMCF+1oLcLo+4Ktn+eFavTiMeBqigp/L4GHBJdBxwS7QzcCdjHV/GWBDkwmg7QdA0gUcd8q0IIbLL92I", - "TfPdR6/gXXiEY2My2UGYagLFex8cLkywht5frmtjwCwf8pm1XeAbbcgg/S+YtiaDdYcub8JoPvvoNYdv", - "MkuK8apf1bwOJkz99aoPds2CBDvYBujMqnH76GWmumXKn1X21iSxf46swMrRUtZLOet2xTuaW/KVK+Rf", - "dzuGop1uxxEK8rTrGdvvc66v7b8iK/qimAiOTDHsLEE1VTSy4MAwEyoVDaTNXNCL26Rf2EInJBwZw6Qp", - "INJkPVrjJfvIqS8fTtAawAD+DVmnsf7XehY8WTrrtp7tPNt7svVsrxXYTz7A1WrnIeTk1ge3UgcNknRk", - "/Q5NUz88fW/8CoGx2LPAiw8nRWyFRHAtevTMXYPFzp/1nxUxjkKejqPCrZYFRINUO7NgXhivTBY1BOH9", - "SaM5nUzYn5+Ci61/ChpvXu3JrbHXS5t15HdpHRdvtmv+XzLumaIlfhgaYCghG5Ga3hIJM0BnRCHgnx7C", - "AZgOWSqtZTmH52Qp7mWsne3t7adPdrda8ZUdXWHjjMDB5TmU7QgKWwzeRGtvz87QRoHhTJsO0ABwlZk1", - "K/37DNnCnYOyQtrfHGz7uKTh4M65xrY9jxtJ/sGaZnZSluiQEZyZbbVd7qX29vbgyc7u091229i6Xkfi", - "armEcfkyhjwW/ru48mugTb47OEWQjTrBQdlv4sKTrjUqda1RAXS9gZy+xsCePtnb3dne2mwHOeaLeLBg", - "eqUNW5Zdnk3nYQrPanhIURe93abTwqdOGQZ7S4II0/ggcPH9ldPHQEuPhHktX4Q2B4P19tcOrhbftnIc", - "Ze4gkx1iVAMuUMqyggb91dedt3Jr2Sy1zfGwWqr7ckKYpp7FxjEFjW5AykSQOeWpvIWGuDIJm5OIc3Gt", - "b5sMlrdEppEyV4tUog8nP4FM0byGpCJJ2Yay3LgEQeiGk7vWfi6xiJ/Jm4jVajXaLP2yCXcbdm13GZpD", - "SRo04naFWnKlbHXk4SGOghRKeOBsPfWsAIAK8tGTJFqYwPIo4pyhYIYZXECIAtwOmvEo7HvDMPWT0cQb", - "scAvUcQN4vAFIYmtbmEGoT/TKgydE7RWyIdHhpUqVQh3YyNkbP2CMjfuxv5yalj6MqWyPGxNT6x4AQzX", - "fFJyOUZ8KsEoVBAy369isCdYmEh4zEy1lnlsbElPtK9niBVh7jtRzUnKJ9bAtSoHZDkbSuJAcCkRiegU", - "KoN8OKkkzy5JuMpSaFdHE5YH24J1zWWh5ygzUEytizr5zkdPMsnXnJDAw5CwtiROzzknY8xSqHdRYGRy", - "lVBh2KNdLN6MSzXKQJGuOVipRlDLIBUkh2rLUr4zf5B7x3suOtF2E3LZoNcbfV3jKn9TTQNslqleivqp", - "1c140MfGdViopUhUObRVFcfoOkhpOfg9ldAqLWBmoTVIuCiIpQL+2XqbABS/yar7qVmrtlbf7zuDs7aY", - "YsshxE6xmh2zCfcAT1zjVtJ5om2oZEJETKGaBwoJoyR0tmR2PWldXZCtHEmCwpRYyhn9VGBLcGy2N4BH", - "MOcjo2xakfXVDtu4h80Ylpc6gH7ti21CiaQ/m/OdSIFWJhZQIpzndbYKsKRy5L/OqjcsyDSNsEAWDbDN", - "kOUijii7aNO6XMRjHtEA6Q+qd84THkX8cqQfyV9gLuutZqc/GOUZHpU7ZDM4m5RmFqTSbz6FX/Qs1ysp", - "seCJ2TDfbwBcSZvILG+Y8ksaEQst957RqwKjl7HAd7YGTanaDY2WkrTrsITXldyWZX073iEGHmS1oT2X", - "libguHJ1WvZLrrz4g4j2ZYnpdc8MWnPBXg5rvUzXAuZ5K8dIu6D6alijG82GJEG5952nu0/2WoLOf5Xr", - "0wBx3Lajcx4vcXA2rNRJGy/a092nz55t7+w+27qWv8pFwDasT1MUbHF9KiXgKz60XYgWG1xrUCYG1j+k", - "hjjY8oBK5dxvPKAvS7ZuU6xBvjeb7jyj4kq6a5eyQ7Sdy3GJtnRQUrnyEvRojUwmBIzKkaFbLx9MJaO3", - "1RgCnOCAqoXHYYIvTVxe9koFtLKNc608WA9JbdsW4ExLLpmO8/yRNdc5+qvxtFd44Wnr2hUyHTd59d9U", - "ezU+/dwHVLwxanFhk5fXrbsLsvlcYlkKPdN/BxAh6mLh63HB5o3lAHrVoE24E7QlWgqBFT7g1cr5Zz8q", - "Ln9lOQte4JKSXKX4siO0eQtey4b2nMgeEzpYnQRUkQ/2ALzZV6NxsarM0rI9pRI0+al7/X5b5DnVIY+z", - "E+z6/RVSO67zYRXeD/jRjsGSPG+7W2KJBm4qRO16zBEekV4W9mBDepFMjX9V73mLGOvJNQku+GRShq3b", - "bYY5hQx+CEt3vWClSJyoLiJXYKaTsIaRadL1h51dOewgLtCwsxkPOxUnoDfRI8ZXI9tBOU17sAx3NKun", - "Vh2kdDMYRzy4MAVTlKBE9tEAxQQziVIGm7/io9wcLPe1dTtJYW0ylE9iLoxrYgvGNCYzPKcALm09VNNS", - "WAu5okpC+A20s49CboAbStXi7Az1ayYNYz+fNBw6mC1sw7pB/R5nLj4ofxfMpQnUqGOfiOBdm3eoJfab", - "Nyddcx0EgRxmYKVoETdRMwItILMuKkjJ+e/+aKxxREYw7iryblynYzF7DvzUgkiipIXizNmhwgQo4ClT", - "VUjeuF1EZzkAvn4kpQxCJ+xlGkCs2N5tjdmQBLaOd929VGL0GzB3JQrTUtoXhrntY2HYFOCZ83ve31r3", - "enUABlOkUOjRtFOMkjM+15FU3FYGyXb1iFwFhIRV7C7/K20jD+2X3sjD37FN989qMNq3IXqsPrv+3YWi", - "w1ibqF2MkGSc9SDR2C2pTQo2KD827bzMaCUU0ULy7ciHlOZ7oU1uGLlaTuvX5EoB1GmYRga/xs+6VlTZ", - "w2gVxW+cg9G0oblYXbL4DirQmOi9G9WgsYF/D1GGxr51J6VnaqtzRpR798yyUXMB4hJme8lB6MIn3Svl", - "G0vDSl1kD3i0Ga9XWHBn5veKWHy9ltkkDMdklAgyoVdLmMe8YAzjcgJ2vpFKRaolWovxFdp5goIZFrIy", - "dkanMxUtyteZOx5YhK+qzySIIkxdo5x3vpruw3rsgF3OYus+5fisAGLgLzNOwtEyCMzD7DV3O5vgBXhx", - "Gl2uT7Z3BoPtrcGNMDBvq/p5oZ2mBIXCd/bapBTzU2whSwerl8i7FBSyyjIySSUIjvchjDnBAUERmQDe", - "TVaadOXZUet6+eCtQmVT5zP+dwtl183daJTR77OuLHyom0bHhUeV0RKKz+vDXgKKk4mZoIaO48lg2O4N", - "9t5tbu/v7u1vbt4FbmVGpKbY2SefNi+fRFt4shM9XTz5c3P2ZLoVb3vtsDsotF/OCK3W3bdzSIio1j6s", - "1gyVJKKM9GQWb7466WOJLDAxGyv3//X8/GYGS3WHs/IkiyoEVjlxSpz1MEghdjJL7y6qszk+Wj6LG8Vz", - "Vwfi57fqUIC92g0GwKY3v7asdcpaHkPvCy+2PoiW5hisOop8KaGw072r3EBxH3uX5GRpwy07wOuHnMd3", - "OuWCqlm8/LTIXssQQSEK7ZNUYRmZoo+OpwwKnxZ/zoIOikaU/rjT7USfdsp7xv7eHqPEgglmDGiXuqgV", - "tLiUh7q6y6kAr+SGhzBxgdpW12P+ZbO3+QxC46JPO78Mes/66O+FEL2uoVaRfJvu7dKvgzY0LFbzcVUg", - "Np9dK37N0XMZB/1GfbVo8nPZwgxaHs/LTLqjw6UvlRY4f1xb4wrmQKMC+rWanj3cRkWlKSQRXvhgZguO", - "WlmxHotMhsZkSpls47fdHmSO29142OmjA4v1CbZsXlS41DyUky3wCY1jElKtYxrTvzkedKulL65qS1wP", - "Ztx95dHW+n517dnqhNNV4eurjsn+V2Q3fZX1287iXZYLDX41Z6IC2gm82EV0gjCr1Bqzlb1tWiKkmQDG", - "zr6DlMlZ1soAmeuBzk/SRVOuUJ6Q2NLflrJmv2A2fnIF/tYlGciGIbZuJb08gzqhy8TX8RFKBA/TIM/G", - "iWDQef60SCtgMkuU/NUBTnfp34A0twkXaLV/o8mh0c4/2bTeFd+kZtjmpd4crF7qO3GKdDtpEq6WYeal", - "dhLsWiCsKxI6PC6aMtkrmmBhMh9bSPS3RQrWbV7jSw60SpQm7oJF81SdkzzXLXDF4AMePCIR0cdUvRHE", - "ozCPOaUyl6KrRerm3tNZ0xUn3EjVB/IbIYm2VQBNAvqLMVt4B1atXI3WBq46oDQXXj1TecBSqzy4Jys1", - "scalal8EvOLzNtnnxRLsGRrp7VYAt1+6sLdG//FduOUeUkl7Y68eKgBzDv4ww3F1/Zt8GgCoZFHlvN7x", - "Xc/72OKdtYybQPOqOTVFr/NB7x/Gy4xG/f2NX/72//Y+/tXrba7YzZKIXkgmEGh0QRY9U0dA2+j9MmQc", - "gBhrZXpqWYXgGHxIAMdqN2NxvLuDTGgsXuO4NgWI0CoUAdhcOaG//aU5vqlAxvcgJ1ey7FdjfN9FLTTF", - "3XG0FhMx1XYhdSHykBk2ZBDpf0EWEhVKi1iVxjHqTzL7RKvo4MPEETo3amCfsPk5GlOo1SSHTFu1OAhI", - "oq0Ji1ZPTcFRDtJHEBwV27ElTlwanb2QNPEEBH04qeENvnn/7vmb96+PRm9OX7w+OB799uK/IcTjsmd6", - "CHua93Z292yZ0SIlNz1L/BWIyF8F+edjNwMc5uEvSFmB2q0ehZlKSDh14QaFl9EaiRO1cHXLXObL+vWA", - "zA6yBr3BbrcMTz94dhslpN4vrRk151FPa9QNMMJeB6ahhTdYG5oyQfCdJj/3dOxRG603cUqn2OPa9lZZ", - "vo1ST25AKxHmauvfWPPCHzp/VMVYNtLAkKqCCVyxS6XqNUfWx1qRGuX1bMvxGimzySe0EM5VzjSJmdqw", - "Jdl8Ca8hB6TPZelG+S5z8EY9+Gh1Fs1SVb4ws8JImtfmxGmsFZ16CYFONWkuZ0SQwkLABzl27TVJZlNB", - "WqRRm5I2CRF5mKTLI9GKENxwSrSWORscCbJ0oboHdjk28Qm+ynoA7z2WtSsvmEdePGDz1XPAkX3rCibR", - "iWsChlGxJ/yoqWUuWkYTx1X1xShyVX3e5n3vxrOyaon0a9pbFebM+yixpo8f/46peskFWCDNSct3Dr4K", - "1k1IBIC4VKFVW+GS0piEo6wofdP+d3XoTcZyVlQrL8rmrC0MTKyF3OoaRC6tNh9DndKaHCRIBVWLMyhe", - "beKHCRZ5JTRXA9v+nHcM1ae+fAE/5cSTo/CKMCJoALW99H6MMQMlHX04KZR4MdV+aoBroF6+OTy2Fq7D", - "7AOLhSpgPRfqd3B63Ol25kQYK68z6G/3B7CZE8JwQjv7ne3+Zn/QMVXLYYobUI+VCPiHTUDMbKXj0GpC", - "z91L+kuBY6Lgiz88qXwQ6mZfB60XTwt2S4KpsIZLEkGKoWEYqr8GLF53oO6bU9mWHm/tppNqYRMuSPLG", - "Lu5HUCph78A0twYDi0yq7PEL6SImRn3jnzZgMe+3lVZnSeSBpq1ZFk63zEj/pdvZGWxea0zLhgJ719fx", - "e4ZTNeOCfiJgEO5ekxA36vSYmUwwZCDGbAROcccBIxX32h8f9ZrJNI6xWDiCFamVcNmkGBOJsHvXACYq", - "iQItKgBAv4/eMGKLEGOFsAmWFSmDwnzuQ82h5V1g2naLnKEMPOfh4tZIWOrDmcVfyuJMb5cvNX6+Pd7J", - "2Li+kPaRg8g0XHsPDPQcZ1UlH2yn7Aye3X2nh5xNIhoo1MsY2IbAUglRJhEAfjrwAC7QnylXGGUR5I9o", - "S1uddZyxWzc/ijY+0/CL2d4R8XleT4mIMTPx+OadFZu+tp2NFzzfzktPNcf4gM0NJ5XLojcHFShy5S1a", - "PLaqymD9ONppKBlOhJ1e+ICMv3MPO9xONiss9pBbDqpVoVSSx7Sd7K3OOFdCvLrcK6K+FZ4f3OeRZVGA", - "v8Nd9FgY+BXJNLx8tWqHwkYiUmYMYK8G+DbPkbPf/VRW/t7lTwqBGeBK100DHrUyV3k4XPSRo6kx+tUC", - "4LWFLVldP1ZO9fC+lR22dR87DGacXU78OKZ+HFPLdrnhFjcF2JiFXd7CB3EtD8T353+4tvfhh++hve+h", - "leeBkUvrXfgnH/eRDYKEqr62Uv+YIAOx48IdFBb96SeERTCjczJk9rYgTiNFEywgmCFGIVbYXNs2OiaW", - "uiWy5jZ0cz0X+pYTuAqdIMkISieNQjolvknnQW+UMRIi/YktMWc/8dXaNHvf62DPGsyPRnQ54xKuNiSV", - "CirkZKc5ZNRKYx1Ds/0he2eR2jQBIX7XyRpJIsCbW+L/4QzhIbMf/OxEiIs9kjjOJRcWRLMUVI4goVmW", - "ejaVHulIBtwH7/KOMMxUTyYkoBMa2GldkIUNIfQ22Kpwgh6wG+eHkyxHAG2t+yHCoNCsH13vKHuGLCeV", - "728YxN0GURrml1wOtQaLMY4iL7L2NOJjHI0MfS6I507wFbxhiVKsyetukxgPiamvmizUjDPzdzpOmUrN", - "32PBLyURw85635bTt7QmYTdXENElVGKJoaQ9RGpBnxtmiBufL8jiS3/IDsKYMscR8AmOJEfkCr6DAhUA", - "02CkVwM/mN3kvwc/TKXisd09kMXj+M4Mk6cqSZVNYpBEmSmY17VKmsoZCYdMcfRZkCmVSiy+bHzOe/wC", - "l8UEh5pPCq+YKYFu3TRqOcJ69iN41XPdToAAw44+SIcd/fdUYKZgTKAYSlCvp8UlXcvgjfUmXa9SOMAM", - "JTwx0NDAVDOsWa7UBsAD4ChCCraS+1Yr7rCSDfOxaG/xuBHqzWBzVbYRZejkeWEzDXae+veTJIEgvoiS", - "/zx78xrBqazXwLyWRwiZLAKmFQYUpnB16mTaCxzMkLmogmpAww4Nh53sOjdch7Gm0ibM93pwp/iLHtov", - "ppsuDX/p93VT5rpyH/3x2bSyr/dSEo8UvyBs2PnSRYUHU6pm6Th79tFP0CbErLOSIEBr5phbB0mCKYCb", - "FE58c0RiFiJuT4FogTDKJVAxcGVMGRaLZblrHtJbCvKJCZ4rEOPzEILlhp39oQuXG3a6ww5hc/jNxtQN", - "O1/8FLC3ls2lZ+A8yy43MybaGwzWV0NZWvp67ixbXAzcsg3YaBVldbP0Cv6ZkvS7ux/4t7Y/s6sfzHTn", - "ORqOMfyd8/0RXkAUNPaiJeq5gqio3ZgFJHJq92pHz/1fHujFCkgU3TeDPhR7ZtdjtsDiI7sPg8XKt9FS", - "9/0Dc9zgvg6Vktv+Yfj30fnPPd5z6zsncxfq7AcaB9gTa0oj8zLCEp3BmHpn2vh+Ab/27X+d7QcwfucR", - "n57vG9MdRXyKIspsCHohUFmrB5aW8JFBPsm+s0AorsrLmtEk/vU//wuDomz6r//5X21XmL9gu28YRC8o", - "Enk+I1ioMcHqfB/9RkjSwxGdEzcZKONG5kQs0PbA+vzhUbH0t9XS5JAN2VuiUsEKofqm4Iq0DdqrAj0f", - "ylIiLXKMfpFOLBq8iW30+G3cXjakvNcd3fUg8MEMChPQp6LjAYAvo6ZSprVEO36XqZlzyWlaDdOsBeut", - "li+KXCnDvT0zwGsKGCCxb9/BAztptHZ29mK9j8DaMlwBiP9gO+TNWDOi/0MmrZZJRqKUBQpQ2cgmA6e0", - "3Ol/ZN9p5/W3LX5Pbn9bBOYafn/j/AFQRbcCP+4AWtwB+Onm7gN8Tvkjhxd2d8GCposHihV0vFenuXlS", - "INlDOAPQmgNiAIcqF+j08BjhMBREyvV/b1eBnqnh0vzoQJxBUYCHuLW2Y4EC4jHJTLUygzwWcfDWjhph", - "N69qba3i+bZRKhXReNJlVSPyI+/uT49Kp9c5RvL6Xzmv/ThJVsbpURlw/W2BW3oBToCQTn3J9mmRi1Y5", - "pEwEYHbkLFWXrHg+PnIb8v5cU7brlFXPhnsQikcVgfiAgrCcpVmsmPeYuPl9tooOG3WJ5+rbYs3B/WlB", - "9+3F8rH5Y3JjhRWyaSlo8AQaD9BXRBkUgc4dLrTtwTPxMyLcrnYFTmHW2bTMp8jAIcCE4Gp+ue17bF5p", - "Z/qa9r4nyxfIcx2NxZL8h4rSwtjNabXMwDVLcJf2LfRwLfP29m68LYN5iAxhN2PnsRaKhGgNywUL1n9c", - "et86R5uQqNyIFSgry4ySCCuIjgQglszO0mPbuge97q2No0ICK2Ljhh5jMt5pGkXuamZOhEJvDo+NCCge", - "VhufIZJstRHixMLSc+v92997hAUcQgezsDe/tmef3LIpYjirlGB3//z8CJPMqDt4m1Sxr1h/E+GJTFBq", - "n/L/2HoZ0bHAYvEfWy9xlFBG/mP7IMKKSLV+Z8wyuK8z5L5Ng0fMfNoyoGWigWhiU4AOXKFKZ2+11Kbd", - "+9+VQm0mfS2VOqPrD626jVZdJNdSxdouxZ2q1qaPB7o7ypjNR2149ANn4h7ckZYjCzgTpfuZHGlixqWC", - "R48v6dBGetKM44rHRku/er4hlx4fjnWPj7pASKgsCsjmNqfnnrzsbhz3rtzafu/fxX4Qj+k05akspgvF", - "WAUzIm0qXUTKAvixqd358dyoeH/DXDq4z6Pj3vXqH3x/Rxp/dUGN8DZXZat0fvdWW53fvq91fgMzaNMN", - "Lfx615XmWG+IfnRAg23ZuITHWI/K9I3LZ4ug99pQyc0FBBbE/pD9H21//KEIjj/+4vKa0sFgaw9+J2z+", - "8ReX2sROHKsQBjXDIcX14PUR3E9OAaERii3lWZTVcZjarcB6Dl76385Ayq9o21tIjgt/WEitLKQCuZZb", - "SHYt7tZEKkPU37uN5PjNR3AL9Pt9Wknf8MXDvVtwMp1MaEAJA6B/yBaVtUg7Y8n9uBm5YZYgszd9hTCd", - "kibS2ozMpNYKDT2vLXrvIVrHeTGV+7YeXRnTx5ntwBNbF9Daa7m20GywfWv8MLjf0+v+DbXHzGLGIqqT", - "LtFKt6dchylUE6cKwktzgB+I30XCmDVZi310mOV1yzRJuFDSFLsBC8GUw5xpC8FXGKdc68ZX3AYKulAi", - "u0MG5U71Y4NPsXFBFqaUDeUsq1qTzdRWhPFl0ZVLCT3oNrp9JdRfJ6mVEnrP29hWvns4JfTBRMe9qHvH", - "pYKia9nGAIt7TLKdzLM0TfqJsun6o4olNsIqm1sBjsyjam3gVHFXFn9jxg0ykR+c7TTCAWCz6dcMbJDN", - "+zU4YcWmIJlX8CgiwsBBJalydbOGLBscZYW6wLZIxblufpQyRaPzrommgZx+iTBbWEyUISt1hpUicaIF", - "m0X5gREKkpgRVwqG6UFTnkp4q4skL3WJcHSJF3LIBJlEJLBzg+KKggQGOS2K+uhXDonUCE8xZTa3V79p", - "ym39JIfsnIYRGdk86HNEJZIzLhRhJEQxnxNZ7pdgEVEiYBKHWFNOohgvAJDIYLMZ+vCEGNCfUrY11//G", - "LKRQjEr3nE15f8gw2hoMUEwwk4hCQq7EE6K/sm0gGERpQD8jjHYGz+xXlXUD0ExH/jW9X4Qgcx7gcbRA", - "RHMxnIhqHRYw219Q11Ev34QKadYrcy/aqj+lhaXS1acMuyhlAZRPTIX+FxcoZfZ41S0KyDGHedpLOEJF", - "VnbMJsSPSYA1PRkv9wNQZDwIUuE7HPVSFwrj/TsqmYXpnQGpfNJJ0wHBngphzRlXM9jTHLbS+s8NXJUz", - "1fdxyHg3CRcIowJf5w4FqCHNpmgNoLvO85JbzFVtPF//2e0dvX2tIHDb34BnPZbzCZiITyalDbj6aDIb", - "eFniQp2Fv9d9euhqLRZFXEjxlHGpaOCEYbUa8A/jsbXxuJyyXm6ecHFR1K3K/PuSi4u21teZK3H/qIyw", - "4gy/wXsAPTwAX3346wBwRhtDRTPNvRtoVf7KdikoXVRJF2fMUcTZVO+i3Cl+7177ikUXRCmo5c6Uc04Q", - "bYSM7I+mXKOejC2GBx7+wLb60LJI934Pd0GvuUI0TiISEyjn2DPMphc706pNtWUq0Syro3c9Wal3VTEp", - "19iC0lz/d506BHzlFmwNtPf6cnmFasSnq4G4ss4d6pQHiWvITDVo4kpHn6NMBmuF1sBeo8sZDWaAygV2", - "q27fgHbhJDnPAEnX99Er2MhFXFbofM2AXWtekzwiBmxrHsfn+/WChR9OTuAjA8hlShOe7yNXpDA7P6R+", - "q4iypWcRYanQa4sdtpYZ47Ci5wprezOb37rF38oBY4fMh8XFyKVtkE7QeQGW67wBl8vJ29/59MGUsW4z", - "zLeZi+LImo7Am4SFnaYYCxr5Ebk2BwMf+mxLdDAzjDsGB6sN5nc+zSDGS6yMk6Qt+9phAhfP43gJD6O1", - "XIIgqUKeqr9JFRIh4GPL3U3MjdZwYMvL4AvNqMxIJbex14H9vJFEBvPXSyotVDvdDmFp3Nn/w/5rHsed", - "bseOp4AVfA3lfgXKWrXBesSLXpkClNoPtfw6IGllYV9ASaucHNacbtbI35oXvvubReeze0A2BP2g4sT9", - "llTQwnjLDh/GkWQ4kTOuHhcuk3U1VbS2ZleNm2VPDy9MXQ2MNiEcZ/bTM/flN2D9rorscGNGbrr3HuJR", - "H8FjzoSVtdlMuKiC+ayK/fjmGen2lqQ21TYc8oM3r+/na8WYia8av1ua0NREwqniMVY0gHocwYxzWWD7", - "MZnhOeX2qtTdWWWcCc4NY2faEPpzzarn1hF8bhX5feu0Qrj4yPbRh89t4L3/C/co/+JlwS7PJH7XKd+A", - "WQ0FgwUlE5TgVBKtV6UxQaYUvy3AQnAwQwFOVCoI1JYiKKaMxmlccDVow0nMcYSoROeb8XkXjVOFIiym", - "YBeZhyacXpCAxzFhIQEP2ZDNCJ5TbdQJFGFFWLDoSQI1Keckr/SvjXwbhWNqWgmiOZBy1kUxUTjECoOq", - "ca53/Mhk8ZxnZSqNYc3IVc4N4ZCJlP1scLZ1s+duoOeISIXHEZWzrJxZgEPCAi+I9dm3LcZu3xt8RlR1", - "og8Ul3MjWfqQgTpFr6cbzrcRw/PIgpG5sMvYRswvUXplsxFZTn9wbPTvuaXNXN0cH+iKJyPxsl38bdzt", - "ZEz3zdzvPPwFDhcoTE13hV0JbP693spkAqUY7gSplWYZb3o1k9Vtysh8LZm38dn9eXwDb9o3Igm7jYZ9", - "U4WQfNLfgsi1VL2RzH0gN6L1JRW8Yg8ogl1M1YOpT1wUpNxjcXdagW22Zia3i9JJCQzWF2c/xHZVbNuQ", - "g5uKbeebrV2qFwQ5ZT2I0vRLcOvGbRTV1nXwb5oLUpldQWQ+uIjM7w7uTSweZ4LQiMYELyKOw+8hTHfJ", - "DU7AhTD4D4Ao8ZjwRwtew2KAPvjmupmE6Lrcyg8nJ+tNUkKopTJCqEcsIco1+oPYV0R/ToSgoSsOfnhy", - "ZANmqUQiZX30JqZQsfuCkCTPKQEgj76en0PCqJc5LkFedDuEKbFIOGVq5SjyV+9mMF9uVBz5nuWkhYr+", - "cSHd+kIaPPuPT5yBlIGsCTOB5ZapwqoxFNCFxlFmap9rvQyPeapb1zJIk0mv5xROwQmNiFxIRWITFzhJ", - "I9huUHbAVqW035lV7kJUrN45JmEtISKmUlLO5JDZbI2ECN23/ly3Xwhx8l4IKJzJ11MjJL+N8Dk9GBMx", - "hlUT1QCzCGrCd/Y7GzhJNkKscEOIlh3eVwzpJcTDIbmIxzyiAYoou5BoLaIXxjxBc4ki/cf60oC6EXx3", - "2zU3b76zNKWP2YR7y5IZns2Y+bvKq7JizV1MPjqx9ooUN4uTP7DQfrEmV8o1QXDUUzQmGXINShWN6Ccj", - "6nQjVCoamKSfHLLgw0mOWjBkJ0QJ/Q6G5LIoIoFyDpuNRPBgY5gOBttBQgH+bJvA4EDgNT+OocfD0/cm", - "EZTEXCy6Q6b/AQ2/Ozg1t7sTbL0JhYEyoi65uEDHG29WhBifAZn+jWP0zASXYgd4F/zHleD1EUEa95Bs", - "2KI8WWYq8eS7DyK1GtwPv8Lj9CsAJFM2m7WpwAEoxXKWqpBfMr8PYc6jNNb/MH8crwL2UjiYfYBXvxlt", - "1wxnZTdugo9iU9o5hcSUTXyQSw9DsMcas6oJ56YASkwpGtB7Chyo75G7b999X6TjN3jdaSnqSpJ+M3vr", - "vk8+OwaHcVGkx2PZ5obT3EwUX+59usS02fv0POLBhbRgKEW3obbbAGBc/5gDQtsrQlATIDcTWRAhRK4S", - "KgD5reKANJg7EmGkiIgpw9EGzNk0AtDWzouF55xCinQQUUhSoyGgFkWATnc5Iwzp2YCjyjVQuNGVtrRU", - "8Z3iZaTiaEwCHhMH973uM93+jql6yUUZu/tbkYvvCvTX89FT1fNcAVfe3ONXwZef4CsIlQ5Te6HsRrT2", - "iuc/GldQF8HaDDvbAznsdNGwsxUPO3oFDjG4ULFCuyimLFVE9tGR8W9BEuzeAEkScBZKhzruPHjbA9mU", - "EmvYsiG/cg++u0+1x3IVkPKt7cQnHvR7SH8PSTtorbjh7J4Mu7DpQsRTZdz9dl/Zt0KiwD2yfu93tYU9", - "8sO2byPJ/263b0lGwSprcVlYeiPZM+znlV43l6gxM+Bw1mkQ4AQHVC26CEcRD3LvQSqz24FeNpSxIPhC", - "21D9IXuboU7b5Ap0ePq+65xmKKTywrRg/WJ99GZOhEzH2eAQSAPjwYPFIOGQKY4CHAVppPmWTCYkgLwI", - "AJOWDX61bCh3WQg678SLfF2IMEofXcENP0/A6uVsISsct2GWekOQIMI0boZitKovXP6C23esG+X6GJ5E", - "9norEFxKZJvqkYhO6TiylzWyj95plQPHZMiSCDNGBEqliVDSQ+8lgkiZmmQb3QBAlhmO6qIcZiURXFk3", - "ccS5kMazqzn8wwmSiiRL2OytafkE5nxH9QdM47anBzIYKmNoPpbsK0gviOEUQ3DNR/qYfoCwIDOgh65T", - "8Fg2/jtBp1Mi9K7ARsiaq1GzrR05zaYvZY80Ft85y95qV3wna7UQIV6Inl4KkzHKkQfDzvVuYD2dX9BG", - "JBX76HoZHb/pj1r2Xc4c8A/CPvrKWX4vNU3PCgHbbUv25Bz+2KrnFEZe2qqlpIfVEAetsxzuMuugNZbB", - "g0EYPGbkAlxKZWiCKPj2GGFwvxl3912e4nHzVgl5oFSxryH9ajW26DfBgXcDKvrAGac3ABX9pnKgAPXx", - "4XJRvRv1oXKaSn5AV/nru8cFvatUJgMOCtAYTalMRurZQIKlhtIH+047M8m2+D1p8Pbu+Rr6uyP7D6u/", - "hclQIJbfZWfyrR0WDIkTtXCXi3xSuQCU9BOkbfjAJLIYgrvDcLjB9frtsYfj08bL9e+zUOeD3N/bQipU", - "ouMjTwXMR4b3UtxzpYNlQ586PSyCGZ2TZqd7eQdbEiWC9BKewOVKaAhm6eHOMoVFf/oJ2eYt/pX9F1Ti", - "AeBSEqKQChKoaGGqImmJYPr4SSLBtSUAz7lY+JzpxZ37UvD4wM5mxXlo95R1huV3vvGiF2KFe3MnbZa4", - "0L7ipt3dbWuBhyhDr56jNXKlhMH7RRNt+SA6yUhqSp9K4Mn14oA3Bw2eTfqJjKbjNqNcgtz8xiJjoyCV", - "isdu7Y+P0BpUgpgSptdCq/oT0GQTwec0NBXOc6LOeWSoutlA0Ov6XbVSkZXxcMaFGdyD6DBtDqTpJ5qU", - "xYIJXejsd8aUYRjcSozk8p4yCVW6P0yZLTLm1siN4scRZi2/NWfsaE6EqkSWiIpzA7e3/uOYe8zHXDEw", - "1Z1ppdOuXYnldrGqLUNI7wKEN4tjvl+39YdvJ7ySykcZWWld5/PMIG1ym39bLDi4v/Phvt3lHx5xOP4r", - "4ozvgqscGtAt+hjmdx7gCIVkTiKeQPVl826n20lF1NnvzJRK9jc2Iv3ejEu1v/PsyXbny8cv/38AAAD/", - "/8y2rwGivAEA", + "R3NDJND7QP+saioFDfHWsT6LV3AuS9hwaIuruNyQ9GZWUGaWRvPJkjn50kZGebBgsVW5kBMLdme1Z+xU", + "IANKm0Ei5BfdciE3xqncSAK6YXNSNgCv4ingVex4E4pDMh+lqc8a0o8cQMn798dHaA1+AaBNSCss8yzG", + "e5tPB0+f9Z6ON/d6O+Fgs4c3t/d6W7t4MNkOnmw3FKxvn2F286Qxr5Hsq0DuIrlHLqLcF+jbFM9fUUds", + "jPIlZSG/LJ143qDRYu82IHVV9/Vw89ZD8CapQDFcaKlBeJ3AuU4C3bYJ0q7Up25w/u29G2yucP6tFGMw", + "uIZj4Z1ImblqNHn3mVs9Lgy4uFjlcd5MysOAXDLIKmoVO29PtMH+7rP93a8lmktoWDXGKjvd4+I2RUk5", + "QNpKxoTL2iu4jhyAY8eqQcbTbRMsOt1OlgMCf8P5X4kvzh63Smxq2rBdvxhZdqw0JBQfl0wUiN4wuHTh", + "vlZOnAkCsO9Z2rrWeg4jnoao4H4zMF1wN3VcMFd0M3BVZL1yBm/RJChoswYAfgFenjItiOFOTjdis4/3", + "0St4Fx7h2FhydhCmyEHxOgqHCxNDoveX69rYVcuHfGZNKvhG21dI/wumrclgvbTLmzAK2T56zeGbzMBj", + "vOruNa+DZVV/veoaXrPYxQ5NAjqz2uU+eplplJlOanXQNUnsnyMrsHIQl/VSKr1d8Y7mlnzlCmnh3Y6h", + "aKfbcYSC9PF6Ivn7nOtr+6/Iir7gKoIjU6M7y5tNFY0sZjHMhEpFA2kTKvTiNqk9tv4KCUfGXmqK0zTJ", + "mNamyj5yWtWHE7QG6IR/Q9aXrf+1nsV0ls66rWc7z/aebD3ba4VBlA9wtTZ8CKnC9cGtVI2DJB1Zd0jT", + "1A9P3xt3R2AcCVk8yIeTIuRDIrgWPXrmrsFi58/6z4rQSyFPx1Hhss3itEEGoFkwL7pYJosaYgP/pNGc", + "Tibsz0/BxdY/BY03r/bk1tjrPM468nvajosX7jW3NBn3TC0VPzoOMJSQjQBSb4mEGaAzohDwTw/hACya", + "LMPXspyDmbIU9zLWzvb29tMnu1ut+MqOrrBxRuB38xzKdgSFLQZvorW3Z2doo8Bwpk2HswBwz8xau/59", + "hmw90UFZIe1vDrZ9XNJwcOdcY9uex40k/2AtRjspS3RIVM6sydou91J7e3vwZGf36W67bWw9wiNxtVzC", + "uDQeQx6LSl5c+TXQJt8dnCJIkp3goOzOcVFT1xqVutaoAFHfIGFfY2BPn+zt7mxvbbZDQvMFYliMv9KG", + "Lcsuz6bzMIVnNTykqIvebtNp4VOnDIO9JUGEaXwQuLSDyuljEK9HwryWL0Kbg8Fa4LWDq8W3rfxZmZfK", + "JK0Y1YALlLKszkJ/9S3srVymNkttczysluq+VBWmqWche0ydpRuQMhFkTnkqb6Ehrkwe6STiXFzr2yaD", + "5S2RaaSMz4ZK9OHkJ5ApmteQVCQp21CWG5cAG91wctfazyUW8TN5E7FarUabpV824W7Dru0uA5koSYNG", + "OLFQS66UrQ6IPMRRkEJlEZytp54V4GJBmnySRAsT7x5FnDMUzDCDexFRQAFCMx6FfW90qH4ymngDKfgl", + "irgBQr4gJLFFN8wg9GdahaFzgtYKafrIsFKlOOJubISMLatQ5sbd2F/lDUtfAleWHq7piRUvYPSaT0qe", + "0IhPJRiFCiL5+1Vo+AQLE6CPmSkiM4+NLekJQvYMsSLMfSeqOUn5xBq4VuWA5GtDSRwILiUiEZ1CwZIP", + "J5Wc3iV5YFlm7+ogx/JgW7CuucP0HGUGIap1rSnf+ejJcfmaExJ4GPLoloQPOudkjFkKZTgKjEyuEioM", + "e7QLEZxxqUYZVtM1ByvVCEospILkCHJZJnrmD3LveM9FJ9puQi4bi3ujr2tc5W+qaYDNMtVLUT+1uhkP", + "+ti4jla1FCArR9yqwitdB8Atx+SnElqlBSgvtAZ5IAWxVIBlW28TF+M3WXU/NWvVlhD8fWdw1hbqbDmy", + "2SlWs2M24R48jGtcljpPtI3gTIiIKRQZQSFhlITOlsxuTa2rC5KoI0lQmBJLOaOfCmwJjs32hjsr5nxk", + "lE0rsr7aYRv3sBnD8goM0K99sU2Ek/Qnmb4TKdDKhChKhPN001Zxn1SO/NdZ9YYFmaYRFsiCFLYZslzE", + "EWUXbVqXi3jMIxog/UH1KnzCo4hfjvQj+QvMZb3V7PQHozzxpHK1bQZnc+XMglT6zafwi57leiVTFzwx", + "G+b7DbgYbRMw5o2efkkjYhHv3jN6VWD0MkT5ztagKYO8odFS7ngdLfG6ktuyrG/HOyDDg6xktefS0sRB", + "V65Oy37JlRd/EGi/LF++7plBay4GzUHAl+lagGJv5RhpF+tfjbZ0o9mQJCj3vvN098leSyz8r3J9GnyQ", + "23Z0zuMlDs6GlTpp40V7uvv02bPtnd1nW9fyV7nA3Ib1aQrOLa5PpTJ9xYe2C0Fsg2sNyoTm+ofUEJ5b", + "HlCpyvyNB/RlydZtijXI92bTnWdUXEl37VJ2iLZzOS7Rlg5KKldeGR+tkcmEgFE5MnTr5YOpJBq3GkOA", + "ExxQtfA4TPClCRfMXqlgabZxrpUH6yGpbdvirmnJJdNxntay5jpHfzWe9govPG1dUkOm4yav/ptqr8an", + "n/uAijdGLS5s8qq/dXdBNp9LLEsRcfrvAAJXXYh+PVzZvLEc168aSwp3grZyTCGwwocHWzn/7EfF5a8s", + "Z8ELXFKSqxRfdoQ2b8Fr2dCeE9ljQgerc5Mq8sEegDf7ajQuFrtZWk2oVBknP3Wv32+L9Ks6EnN2gl2/", + "v0LGyXU+rKIOAj/aMViS5213SyzRwE2FYGKPOcIj0svCHmykMZKp8a/qPW+BbD0pMMEFn0zKaHq7zeir", + "ACwA0fKuF6wUiRPVReQKzHQS1qA7DYrAsLMrhx3EBRp2NuNhp+IE9OafxPhqZDsoZ48PlsGhZmXeqoOU", + "bgbjiAcXpo6LEpTIPhqgmGAmUcpg81d8lJuD5b62bicprE0GPkrMhXFNbMGYxmSG5xQwr62HaloKayFX", + "VEkIv4F29lHIDZ5EqYidnaF+zWSH7OeThkMHs4VtWDeo3+PMxQfl74K5NIHSeewTEbxr0yG1xH7z5qRr", + "roMgkMMMrBQt4iZqRqAFZNZFBcA5/90fjTWOyAjGXQUEjut0LCb1gZ9aEEmUtAihOTtUmAAFPGWqihQc", + "t4voLMfl14+klEHohL1MA+QX27stfRuSwJYXr7uXSox+A+auRGFaSvvCMLd9LAybAjxzfs/7W+terw7A", + "QJ0U6k+adopRcsbnOpKK24Il2a4ekauAkLAKKeZ/pW3kof3SG3n4O7YoBFlpSPs2RI/VZ9e/uwh5GGsT", + "tYsRkoyzHuQ/uyW1ucoGfMhmw5cZrQRuWsgJHvkA3HwvtElZI1fLaf2aXClAYA3TyMDq+FnXiip7GK2i", + "+I1TQ5o2NBerKynfQWEcE713o9I4NvDvIarj2LfupCJObXXOiHLvnlk2aq6LXIKSLzkIXfike6V8Y2lY", + "qYvsAY824/UKC+7M/F4RC/vXMsmF4ZiMEkEm9GoJ85gXjGFczgvPN1KpdrZEazG+QjtPUDDDQlbGzuh0", + "pqJF+Tpzx4PW8FVlowRRhKlrVBnPV9N9WI8dsMtZbN2nHJ8VsBX81c9JOFqGzHmYveZuZxO8AC9Oo8v1", + "yfbOYLC9NbgRNOdtFWUvtNOUoFD4zl6blGJ+ii1kWWr1yn2XgkKyW0YmqQTB8T6EMSc4ICgiE4DhydKD", + "Vp4dta6XD94qVDYnKeN/t1B23dyNRhmUP+vKopq6aXRceFQZxKH4vD7sJVg9mZgJaqA9ngyG7d5g793m", + "9v7u3v7m5l3AaWZEaoqdffJp8/JJtIUnO9HTxZM/N2dPplvxttcOu4P6/+VE1bJIQ2t2DgkR1ZKM1VKm", + "kkSUkZ7M4s1XJ30skQUmZmPl/r+en9/MYKnucFaeZFGFwConTomzHgbAxE5m6d1FdTbHR8tncaN47upA", + "/PxWHQqwV7vBAAb25tdW205Zy2PofeHF1gfR0hyDVUeRL1MVdrp3lRso7mPvkpwsbbhlB3j9kPP4Tqdc", + "UDWLl58W2WsZUClEoX2SKiwDZvTR8ZRBPdbiz1nQQdGI0h93up3o0055z9jf20OnWIzDjAHtUhe1ghaX", + "8lDudzkV4JXc8BAmLlDb6nrMv2z2Np9BaFz0aeeXQe9ZH/29EKLXNdQqkm/TvV36ddCGhsUiQ644xeaz", + "a8WvOXou46DfqK9ETn4uW/RDy+N59Ut3dLj0pdIC549ra1yBQmhUQL9W07OH26ioNIUkwgsf+m3BUSsr", + "1mORydCYTCmTbfy224PMcbsbDzt9dGAhSMGWzWsdl5qHKrcFPqFxTEKqdUxj+jfHg2619MVVbYnroZ+7", + "rzzaWt+vrj1bnXC6Knx91THZ/4rspq+yfttZvMtyocGv5kxUAGGBF7uIThBmlRJotuC4TUuENBOA/tl3", + "SDc5y1oZIHM90PlJumjKFcoTElv621LW7BfMxk+uwN+6JAPZMMTWraSXZwgsdJn4Oj5CieBhGuTZOBEM", + "Os+fFmkF42aJkr86wOku/RuQ5jbhAq32bzQ5NNr5J5vWu+Kb1AzbvNSbg9VLfSdOkW4nTcLVMsy81E6C", + "XQsbdkVCh8dFUyZ7RRMsTOZjC4n+tkjBus1rfMmBVonSxF2waJ6qc5LnugWuGHx4iEckIvqYqjeCeBTm", + "MadU5lJ0tUjd3Hs6a7rihBup+kB+IyTRtgqgSUB/MWYL78CqBbXR2sAVLZTmwqtnCiJYapUH92SlJta4", + "VO1rk1d83ib7vFgZPgNJvd3C5PZLF/bW6D++C7fcQyppb+zVQwX3zqEyZvCyrn+TTwO4mSyqnNc7vut5", + "H1u8s5ZxE5ZfNaem6HU+6P3DeJnRqL+/8cvf/t/ex796vc0Vu1kS0QvJBAKNLsiiZ8obaBu9X0ayA2xl", + "rUxPLasQHIMPCVBi7WYsjnd3kAmNxWsc16YAEVqF2gSbKyf0t780xzcVyPge5ORKlv1q6PG7KNGmuDuO", + "1mIiptoupC5EHjLDhgwi/S/IQqJCxROr0jhG/Ulmn2gVHXyYOELnRg3sEzY/R2MKJaTkkGmrFgcBSbQ1", + "YUH0qamDykH6CIKjYju28opLo7MXkiaegKAPJzUYxDfv3z1/8/710ejN6YvXB8ej3178N4R4XPZMD2FP", + "897O7p6tflqk5KZnib8CqPmrkAh97GbwzDz8BSkrUFLWozBTCQmnLtyg8DJaI3GiFq6cmst8Wb8evtpB", + "1qA32O2WUfMHz26jstX7paWs5jzqaY26Ad3Y68A0tPAGa0NTJgi+0+Tnno49aqP1Jk7pFHtc297iz7dR", + "gcoNaCXwXW39G0tx+EPnj6rQz0YaGFJVoIordqlUvebI+lgrUg3IeYAGaJJPaCGcq5xpEjO1YSvF+RJe", + "Qw4ApMvSjfJd5uCNevDR6iyapap8YWaFkTSvzYnTWCs69RICnWrSXM6IIIWFgA9ySN1rksymgrRIozaV", + "dhIi8jBJl0eiFSG44ZRoLXM2OBJk6UJ1D+xyyOQTfJX1AN57LGtXXjCPvKbB5qvnAG/71tVxohPXBAyj", + "Yk/4wVzLXLSMJo6r6otR5Kr6vM373o1nZdUS6de0tyrMmfdRYk0fP/4dU/WSC7BAmpOW7xwTFqybkAgA", + "cakivraCS6UxCUdZrfym/e/K45uM5azWV14rzllbGJhYC7nVpZFcWm0+hjqlNTlIkAqqFmdQU9vEDxMs", + "8gJtrjS3/TnvGIpiffkCfsqJJ0fhFWFE0ABKjun9GGMGSjr6cFKoPGOKENUA10C9fHN4bC1ch9kHFgtV", + "wHou1O/g9LjT7cyJMFZeZ9Df7g9gMyeE4YR29jvb/c3+oGOKqcMUN6BMLBHwD5uAmNlKx6HVhJ67l/SX", + "AsdEwRd/eFL5INTNvg5aL54W7JYEU2ENlySCFEPDMFR/DRDB7kDdN6eyrYje2k0n1cImXJDkjV3cj6BU", + "wt6BaW4NBhaZVNnjF9JFTIz6xj9twGLebyutzpLIg5hbsyycbpmR/ku3szPYvNaYlg0F9q6v4/cMp2rG", + "Bf1EwCDcvSYhbtTpMTOZYMhAjNkInOKOA0Yq7rU/Puo1k2kcY7FwBCtSK+GySTEmEmH3rgFMVBIFWlQA", + "rn8fvWHE1kbGCmETLCtSBvUC3YeaQ8u7wLTtFjlDGXjOw8WtkbDUhzOLv5TFmd4uX2r8fHu8k7FxfSHt", + "IweRabj2HhjoOc6KXT7YTtkZPLv7Tg85m0Q0UKiXMbANgaUSokwiAPx04AFcoD9TrjDKIsgf0Za2Ous4", + "Y7dufhRtfKbhF7O9I+LzvJ4SEWNm4vHNOys2fW07Gy94vp2XnmqO8Y+POvakcln05qACRa68RYvHVlUZ", + "rB9HOw2VzImw0wsfkPF37mGH28lm9c4ecstBES2USvKYtpO91RnnSohXl3tF1LfC84P7PLIsCvB3uIse", + "CwO/IpmGl69W7VDYSETKjAHs1QDf5jly9rufysrfu/xJITADXOm6acCjVuYqD4eLPnI0NUa/WgC8trCV", + "tOvHyqke3reyw7buY4fBjLPLiR/H1I9jatkuN9zipgAbs7DLW/ggruWB+P78D9f2PvzwPbT3PbTyPDBy", + "ab0L/+TjPrJBkFBsWM54GoVoTJCB2HHhDgqL/vQTwiKY0TkZMntbEKeRogkWEMwQoxArbK5tGx0TS90S", + "WXMburmeC33LCVyFTpBkBBWdRiGdEt+k86A3yhgJkf7EVr6zn/hKgJq973WwZw3mRyO6nHEJVxuSSgUV", + "crLTHDJqpbGOodn+kL2zSG2agBC/62SNJBHgzS3x/3CG8JDZD352IsTFHkkc55ILC6JZCipHkNAsSz2b", + "So90JAPug3d5RxhmqicTEtAJDey0LsjChhB6G2xVOEEP2I3zw0mWI4C21v0QYVD/1o+ud5Q9Q5aTyvc3", + "DOJugygN80suh1qDxRhHkRdZexrxMY5Ghj4XxHMn+AresEQplgp2t0mMh8SUfU0WasaZ+Tsdp0yl5u+x", + "4JeSiGFnvW+r/Ftak7CbK4joEiqxxFBpHyK1oM8NM8SNzxdk8aU/ZAdhTJnjCPgER5IjcgXfQYEKgGkw", + "0quBH8xu8t+DH6ZS8djuHsjicXxnhslTlaTKJjFIoswUzOtaJU3ljIRDpjj6LMiUSiUWXzY+5z1+gcti", + "gkPNJ4VXzJRAt24atRxhPfsRvOq5bidAgGFHH6TDjv57KjBTMCZQDCWo19Pikq5l8MZQ8qxK4QAzlPDE", + "QEMDU82wZrlSGwAPgKMIKdhK7lutuMNKNszHor3F40aoN4PNVdlGlKGT54XNNNh56t9PkgSC+CJK/vPs", + "zWsEp7JeA/NaHiFksgiYVhhQmMLVqZNpL3AwQ+aiCqoBDTs0HHay69xwHcaaSpsw3+vBneIvemi/mG66", + "NPyl39dNmevKffTHZ9PKvt5LSTxS/IKwYedLFxUeTKmapePs2Uc/QZsQs85KggCtmWNuHSQJpgBuUjjx", + "zRGJWYi4PQWiBcIol0DFwJUxZVgsluWueUhvKcgnJniuQIzPQwiWG3b2hy5cbtjpDjuEzeE3G1M37Hzx", + "U8DeWjaXnoHzLLvczJhobzBYXw1laenrubNscTFwyzZgo1WU1c3SK/hnStLv7n7g39r+zK5+MNOd52g4", + "xvB3zvdHeAFR0NiLlqjnCqKidmMWkMip3asdPfd/eaAXKyBRdN8M+lDsmV2P2QKLj+w+DBYr30ZL3fcP", + "zHGD+zpUSm77h+HfR+c/93jPre+czF2osx9oHGBPrCmNzMsIS3QGY+qdaeP7Bfzat/91th/A+J1HfHq+", + "b0x3FPEpiiizIeiFQGWtHlhawkcG+ST7zgKhuCova0aT+Nf//C8MirLpv/7nf7VdYf6C7b5hEL2gSOT5", + "jGChxgSr8330GyFJD0d0TtxkoIwbmROxQNsD6/OHR8WK5FZLk0M2ZG+JSgUrhOqbgivSNmivCvR8KEuJ", + "tMgx+kU6sWjwJrbR47dxe9mQ8l53dNeDwAczKExAn4qOBwC+jJpKmdYS7fhdpmbOJadpNUyzFqy3Wr4o", + "cqUM9/bMAK8pYIDEvn0HD+yk0drZ2Yv1PgJry3AFIP6D7ZA3Y82I/g+ZtFomGYlSFihAZSObbDX1pU7/", + "I/tOO6+/bfF7cvvbIjDX8Psb5w+AKroV+HEH0OIOwE83dx/gc8ofObywuwsWNF08UKyg4706zc2TAske", + "whmA1hwQAzhUuUCnh8cIh6EgUq7/e7sK9EwNl+ZHB+IMigI8xK21HQsUEI9JZqqVGeSxiIO3dtQIu3lV", + "a2sVz7eNUqmIxpMuqxqRH3l3f3pUOr3OMZLX/8p57cdJsjJOj8qA628L3NILcAKEdOpLtk+LXLTKIWUi", + "ALMjZ6m6ZMXz8ZHbkPfnmrJdp6x6NtyDUDyqCMQHFITlLM1ixbzHxM3vs1V02KhLPFffFmsO7k8Lum8v", + "lo/NH5MbK6yQTUtBgyfQeIC+IsqgCHTucKFtD56JnxHhdrUrcAqzzqZlPkUGDgEmBFfzy23fY/NKO9PX", + "tPc9Wb5AnutoLJbkP1SUFsZuTqtlBq5Zgru0b6GHa5m3t3fjbRnMQ2QIuxk7j7VQJERrWC5YsP7j0vvW", + "OdqEROVGrEBZWWaURFhBdCQAsWR2lh7b1j3odW9tHBUSWBEbN/QYk/FO0yhyVzNzIhR6c3hsREDxsNr4", + "DJFkq40QJxaWnlvv3/7eIyzgEDqYhb35tT375JZNEcNZpQS7++fnR5hkRt3B26SKfcX6mwhPZIJS+5T/", + "x9bLiI4FFov/2HqJo4Qy8h/bBxFWRKr1O2OWwX2dIfdtGjxi5tOWAS0TDUQTmwJ04ApVOnurpTbt3v+u", + "FGoz6Wup1Bldf2jVbbTqIrmWKtZ2Ke5UtTZ9PNDdUcZsPmrDox84E/fgjrQcWcCZKN3P5EgTMy4VPHp8", + "SYc20pNmHFc8Nlr61fMNufT4cKx7fNQFQkJlUUA2tzk99+Rld+O4d+XW9nv/LvaDeEynKU9lMV0oxiqY", + "EWlT6SJSFsCPTe3Oj+dGxfsb5tLBfR4d965X/+D7O9L4qwtqhLe5Klul87u32ur89n2t8xuYQZtuaOHX", + "u640x3pD9KMDGmzLxiU8xnpUpm9cPlsEvdeGSm4uILAg9ofs/2j74w9FcPzxF5fXlA4GW3vwO2Hzj7+4", + "1CZ24liFMKgZDimuB6+P4H5yCgiNUGwpz6KsjsPUbgXWc/DS/3YGUn5F295Cclz4w0JqZSEVyLXcQrJr", + "cbcmUhmi/t5tJMdvPoJboN/v00r6hi8e7t2Ck+lkQgNKGAD9Q7aorEXaGUvux83IDbMEmb3pK4TplDSR", + "1mZkJrVWaOh5bdF7D9E6zoup3Lf16MqYPs5sB57YuoDWXsu1hWaD7Vvjh8H9nl73b6g9ZhYzFlGddIlW", + "uj3lOkyhmjhVEF6aA/xA/C4SxqzJWuyjwyyvW6ZJwoWSptgNWAimHOZMWwi+wjjlWje+4jZQ0IUS2R0y", + "KHeqHxt8io0LsjClbChnWdWabKa2Iowvi65cSuhBt9HtK6H+OkmtlNB73sa28t3DKaEPJjruRd07LhUU", + "Xcs2BljcY5LtZJ6ladJPlE3XH1UssRFW2dwKcGQeVWsDp4q7svgbM26QifzgbKcRDgCbTb9mYINs3q/B", + "CSs2Bcm8gkcREQYOKkmVq5s1ZNngKCvUBbZFKs5186OUKRqdd000DeT0S4TZwmKiDFmpM6wUiRMt2CzK", + "D4xQkMSMuFIwTA+a8lTCW10kealLhKNLvJBDJsgkIoGdGxRXFCQwyGlR1Ee/ckikRniKKbO5vfpNU27r", + "Jzlk5zSMyMjmQZ8jKpGccaEIIyGK+ZzIcr8Ei4gSAZM4xJpyEsV4AYBEBpvN0IcnxID+lLKtuf43ZiGF", + "YlS652zK+0OG0dZggGKCmUQUEnIlnhD9lW0DwSBKA/oZYbQzeGa/qqwbgGY68q/p/SIEmfMAj6MFIpqL", + "4URU67CA2f6Cuo56+SZUSLNemXvRVv0pLSyVrj5l2EUpC6B8Yir0v7hAKbPHq25RQI45zNNewhEqsrJj", + "NiF+TAKs6cl4uR+AIuNBkArf4aiXulAY799RySxM7wxI5ZNOmg4I9lQIa864msGe5rCV1n9u4Kqcqb6P", + "Q8a7SbhAGBX4OncoQA1pNkVrAN11npfcYq5q4/n6z27v6O1rBYHb/gY867GcT8BEfDIpbcDVR5PZwMsS", + "F+os/L3u00NXa7Eo4kKKp4xLRQMnDKvVgH8Yj62Nx+WU9XLzhIuLom5V5t+XXFy0tb7OXIn7R2WEFWf4", + "Dd4D6OEB+OrDXweAM9oYKppp7t1Aq/JXtktB6aJKujhjjiLOpnoX5U7xe/faVyy6IEpBLXemnHOCaCNk", + "ZH805Rr1ZGwxPPDwB7bVh5ZFuvd7uAt6zRWicRKRmEA5x55hNr3YmVZtqi1TiWZZHb3ryUq9q4pJucYW", + "lOb6v+vUIeArt2BroL3Xl8srVCM+XQ3ElXXuUKc8SFxDZqpBE1c6+hxlMlgrtAb2Gl3OaDADVC6wW3X7", + "BrQLJ8l5Bki6vo9ewUYu4rJC52sG7FrzmuQRMWBb8zg+368XLPxwcgIfGUAuU5rwfB+5IoXZ+SH1W0WU", + "LT2LCEuFXlvssLXMGIcVPVdY25vZ/NYt/lYOGDtkPiwuRi5tg3SCzguwXOcNuFxO3v7Opw+mjHWbYb7N", + "XBRH1nQE3iQs7DTFWNDIj8i1ORj40GdbooOZYdwxOFhtML/zaQYxXmJlnCRt2dcOE7h4HsdLeBit5RIE", + "SRXyVP1NqpAIAR9b7m5ibrSGA1teBl9oRmVGKrmNvQ7s540kMpi/XlJpodrpdghL487+H/Zf8zjudDt2", + "PAWs4Gso9ytQ1qoN1iNe9MoUoNR+qOXXAUkrC/sCSlrl5LDmdLNG/ta88N3fLDqf3QOyIegHFSfut6SC", + "FsZbdvgwjiTDiZxx9bhwmayrqaK1Nbtq3Cx7enhh6mpgtAnhOLOfnrkvvwHrd1VkhxszctO99xCP+gge", + "cyasrM1mwkUVzGdV7Mc3z0i3tyS1qbbhkB+8eX0/XyvGTHzV+N3ShKYmEk4Vj7GiAdTjCGacywLbj8kM", + "zym3V6XuzirjTHBuGDvThtCfa1Y9t47gc6vI71unFcLFR7aPPnxuA+/9X7hH+RcvC3Z5JvG7TvkGzGoo", + "GCwomaAEp5JovSqNCTKl+G0BFoKDGQpwolJBoLYUQTFlNE7jgqtBG05ijiNEJTrfjM+7aJwqFGExBbvI", + "PDTh9IIEPI4JCwl4yIZsRvCcaqNOoAgrwoJFTxKoSTkneaV/beTbKBxT00oQzYGUsy6KicIhVhhUjXO9", + "40cmi+c8K1NpDGtGrnJuCIdMpOxng7Otmz13Az1HRCo8jqicZeXMAhwSFnhBrM++bTF2+97gM6KqE32g", + "uJwbydKHDNQpej3dcL6NGJ5HFozMhV3GNmJ+idIrm43IcvqDY6N/zy1t5urm+EBXPBmJl+3ib+NuJ2O6", + "b+Z+5+EvcLhAYWq6K+xKYPPv9VYmEyjFcCdIrTTLeNOrmaxuU0bma8m8jc/uz+MbeNO+EUnYbTTsmyqE", + "5JP+FkSupeqNZO4DuRGtL6ngFXtAEexiqh5MfeKiIOUei7vTCmyzNTO5XZROSmCwvjj7IbarYtuGHNxU", + "bDvfbO1SvSDIKetBlKZfgls3bqOotq6Df9NckMrsCiLzwUVkfndwb2LxOBOERjQmeBFxHH4PYbpLbnAC", + "LoTBfwBEiceEP1rwGhYD9ME3180kRNflVn44OVlvkhJCLZURQj1iCVGu0R/EviL6cyIEDV1x8MOTIxsw", + "SyUSKeujNzGFit0XhCR5TgkAefT1/BwSRr3McQnyotshTIlFwilTK0eRv3o3g/lyo+LI9ywnLVT0jwvp", + "1hfS4Nl/fOIMpAxkTZgJLLdMFVaNoYAuNI4yU/tc62V4zFPdupZBmkx6PadwCk5oRORCKhKbuMBJGsF2", + "g7IDtiql/c6scheiYvXOMQlrCRExlZJyJofMZmskROi+9ee6/UKIk/dCQOFMvp4aIflthM/pwZiIMaya", + "qAaYRVATvrPf2cBJshFihRtCtOzwvmJILyEeDslFPOYRDVBE2YVEaxG9MOYJmksU6T/WlwbUjeC72665", + "efOdpSl9zCbcW5bM8GzGzN9VXpUVa+5i8tGJtVekuFmc/IGF9os1uVKuCYKjnqIxyZBrUKpoRD8ZUacb", + "oVLRwCT95JAFH05y1IIhOyFK6HcwJJdFEQmUc9hsJIIHG8N0MNgOEgrwZ9sEBgcCr/lxDD0enr43iaAk", + "5mLRHTL9D2j43cGpud2dYOtNKAyUEXXJxQU63nizIsT4DMj0bxyjZya4FDvAu+A/rgSvjwjSuIdkwxbl", + "yTJTiSfffRCp1eB++BUep18BIJmy2axNBQ5AKZazVIX8kvl9CHMepbH+h/njeBWwl8LB7AO8+s1ou2Y4", + "K7txE3wUm9LOKSSmbOKDXHoYgj3WmFVNODcFUGJK0YDeU+BAfY/cffvu+yIdv8HrTktRV5L0m9lb933y", + "2TE4jIsiPR7LNjec5mai+HLv0yWmzd6n5xEPLqQFQym6DbXdBgDj+sccENpeEYKaALmZyIIIIXKVUAHI", + "bxUHpMHckQgjRURMGY42YM6mEYC2dl4sPOcUUqSDiEKSGg0BtSgCdLrLGWFIzwYcVa6Bwo2utKWliu8U", + "LyMVR2MS8Jg4uO91n+n2d0zVSy7K2N3filx8V6C/no+eqp7nCrjy5h6/Cr78BF9BqHSY2gtlN6K1Vzz/", + "0biCugjWZtjZHshhp4uGna142NErcIjBhYoV2kUxZakiso+OjH8LkmD3BkiSgLNQOtRx58HbHsimlFjD", + "lg35lXvw3X2qPZargJRvbSc+8aDfQ/p7SNpBa8UNZ/dk2IVNFyKeKuPut/vKvhUSBe6R9Xu/qy3skR+2", + "fRtJ/ne7fUsyClZZi8vC0hvJnmE/r/S6uUSNmQGHs06DACc4oGrRRTiKeJB7D1KZ3Q70sqGMBcEX2obq", + "D9nbDHXaJlegw9P3Xec0QyGVF6YF6xfrozdzImQ6zgaHQBoYDx4sBgmHTHEU4ChII823ZDIhAeRFAJi0", + "bPCrZUO5y0LQeSde5OtChFH66Apu+HkCVi9nC1nhuA2z1BuCBBGmcTMUo1V94fIX3L5j3SjXx/Akstdb", + "geBSIttUj0R0SseRvayRffROqxw4JkOWRJgxIlAqTYSSHnovEUTK1CTb6AYAssxwVBflMCuJ4Mq6iSPO", + "hTSeXc3hH06QVCRZwmZvTcsnMOc7qj9gGrc9PZDBUBlD87FkX0F6QQynGIJrPtLH9AOEBZkBPXSdgsey", + "8d8JOp0SoXcFNkLWXI2abe3IaTZ9KXuksfjOWfZWu+I7WauFCPFC9PRSmIxRjjwYdq53A+vp/II2IqnY", + "R9fL6PhNf9Sy73LmgH8Q9tFXzvJ7qWl6VgjYbluyJ+fwx1Y9pzDy0lYtJT2shjhoneVwl1kHrbEMHgzC", + "4DEjF+BSKkMTRMG3xwiD+824u+/yFI+bt0rIA6WKfQ3pV6uxRb8JDrwbUNEHzji9AajoN5UDBaiPD5eL", + "6t2oD5XTVPIDuspf3z0u6F2lMhlwUIDGaEplMlLPBhIsNZQ+2HfamUm2xe9Jg7d3z9fQ3x3Zf1j9LUyG", + "ArH8LjuTb+2wYEicqIW7XOSTygWgpJ8gbcMHJpHFENwdhsMNrtdvjz0cnzZern+fhTof5P7eFlKhEh0f", + "eSpgPjK8l+KeKx0sG/rU6WERzOicNDvdyzvYkigRpJfwBC5XQkMwSw93liks+tNPyDZv8a/sv6ASDwCX", + "khCFVJBARQtTFUlLBNPHTxIJri0BeM7FwudML+7cl4LHB3Y2K85Du6esMyy/840XvRAr3Js7abPEhfYV", + "N+3ublsLPEQZevUcrZErJQzeL5poywfRSUZSU/pUAk+uFwe8OWjwbNJPZDQdtxnlEuTmNxYZGwWpVDx2", + "a398hNagEsSUML0WWtWfgCabCD6noalwnhN1ziND1c0Ggl7X76qViqyMhzMuzOAeRIdpcyBNP9GkLBZM", + "6EJnvzOmDMPgVmIkl/eUSajS/WHKbJExt0ZuFD+OMGv5rTljR3MiVCWyRFScG7i99R/H3GM+5oqBqe5M", + "K5127Uost4tVbRlCehcgvFkc8/26rT98O+GVVD7KyErrOp9nBmmT2/zbYsHB/Z0P9+0u//CIw/FfEWd8", + "F1zl0IBu0ccwv/MARygkcxLxBKovm3c73U4qos5+Z6ZUsr+xEen3Zlyq/Z1nT7Y7Xz5++f8DAAD//2zG", + "qBw5vQEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/openapi.yaml b/openapi.yaml index 02bf078e..daba162d 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1565,8 +1565,12 @@ components: example: "L40S-1Q" mdev_uuid: type: string - description: mdev device UUID + description: mdev device UUID (mdev hosts only) example: "aa618089-8b16-4d01-a136-25a0f3c73123" + device_path: + type: string + description: sysfs path of the assigned vGPU device + example: "/sys/bus/pci/devices/0000:82:00.4" GPUProfile: type: object