diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 12861a6a7..7873aa35c 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -4,6 +4,8 @@ import ( "bytes" "context" "os" + "path/filepath" + "strings" "testing" "time" @@ -21,21 +23,23 @@ import ( "github.com/stretchr/testify/require" ) -// TestVGPU is an integration test that verifies vGPU (SR-IOV mdev) support works. +// TestVGPU is an integration test that verifies vGPU (SR-IOV) support works +// on the host's framework: mdev or NVIDIA's vendor-specific VFIO. // // This test automatically detects vGPU availability and skips if: -// - No SR-IOV VFs are found in /sys/class/mdev_bus/ +// - No vGPU framework (mdev or vendor VFIO) is discovered // - No vGPU profiles are available -// - Not running as root (required for mdev creation) +// - Not running as root (required for sysfs vGPU assignment) // - KVM is not available // // To run manually: // // sudo go test -v -run TestVGPU -timeout 5m ./integration/... // -// Note: This test verifies mdev creation and PCI device visibility inside the VM. -// It does NOT test nvidia-smi or CUDA functionality since that requires NVIDIA -// guest drivers pre-installed in the image. +// Note: This test verifies vGPU assignment, release on stop, reacquisition on +// start, and PCI device visibility inside the VM. It does NOT test nvidia-smi +// or CUDA functionality since that requires NVIDIA guest drivers pre-installed +// in the image. func TestVGPU(t *testing.T) { t.Parallel() if testing.Short() { @@ -84,9 +88,15 @@ func TestVGPU(t *testing.T) { // Cleanup any orphaned instances and mdevs t.Cleanup(func() { - if instanceID != "" { - t.Log("Cleanup: Deleting instance...") - instanceManager.DeleteInstance(ctx, instanceID) + if instanceID == "" { + return + } + if _, err := instanceManager.StopInstance(ctx, instanceID); err != nil { + t.Logf("Cleanup: stop instance: %v", err) + } + t.Log("Cleanup: Deleting instance...") + if err := instanceManager.DeleteInstance(ctx, instanceID); err != nil { + t.Errorf("cleanup: delete instance: %v", err) } }) @@ -159,9 +169,18 @@ func TestVGPU(t *testing.T) { instanceID = inst.Id t.Logf("Instance created: %s", inst.Id) - // Verify mdev UUID was assigned - require.NotEmpty(t, inst.GPUMdevUUID, "Instance should have mdev UUID assigned") - t.Logf("mdev UUID: %s", inst.GPUMdevUUID) + // Verify the assignment matches the host's framework + require.NotEmpty(t, inst.GPUDevicePath, "Instance should have a vGPU device path assigned") + switch inst.GPUFramework { + case devices.VGPUFrameworkMdev: + require.NotEmpty(t, inst.GPUMdevUUID, "mdev instance should have a UUID assigned") + t.Logf("mdev UUID: %s", inst.GPUMdevUUID) + case devices.VGPUFrameworkVendorVFIO: + require.Empty(t, inst.GPUMdevUUID, "vendor VFIO instance should not have an mdev UUID") + t.Logf("vendor VFIO VF: %s", inst.GPUDevicePath) + default: + t.Fatalf("unexpected vGPU framework %q", inst.GPUFramework) + } // Step 5: Check GPU resources AFTER creating instance t.Run("ResourcesDecrementedAfterCreation", func(t *testing.T) { @@ -180,12 +199,9 @@ func TestVGPU(t *testing.T) { assert.Less(t, availableAfter, availableBefore, "available instances should decrease after creating VM") }) - // Step 6: Verify mdev was created in sysfs - t.Run("MdevCreated", func(t *testing.T) { - mdevPath := "/sys/bus/mdev/devices/" + inst.GPUMdevUUID - _, err := os.Stat(mdevPath) - assert.NoError(t, err, "mdev device should exist at %s", mdevPath) - t.Logf("mdev exists at: %s", mdevPath) + // Step 6: Verify the assignment exists in sysfs + t.Run("VGPUAssignedInSysfs", func(t *testing.T) { + assertVGPUAssigned(t, inst.GPUFramework, inst.GPUDevicePath) }) // Step 7: Wait for guest agent to be ready @@ -225,13 +241,68 @@ func TestVGPU(t *testing.T) { require.NoError(t, err) assert.Equal(t, profile, actualInst.GPUProfile, "GPU profile should match") - assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set") - t.Logf("Instance GPU: profile=%s, mdev=%s", actualInst.GPUProfile, actualInst.GPUMdevUUID) + assert.Equal(t, inst.GPUFramework, actualInst.GPUFramework, "framework should match") + assert.NotEmpty(t, actualInst.GPUDevicePath, "device path should be set") + if inst.GPUFramework == devices.VGPUFrameworkMdev { + assert.NotEmpty(t, actualInst.GPUMdevUUID, "mdev UUID should be set") + } + t.Logf("Instance GPU: profile=%s, framework=%s, device=%s", actualInst.GPUProfile, actualInst.GPUFramework, actualInst.GPUDevicePath) + }) + + t.Log("Step 10: Stopping instance...") + _, err = instanceManager.StopInstance(ctx, inst.Id) + require.NoError(t, err, "stop should succeed") + + t.Run("VGPUReleasedOnStop", func(t *testing.T) { + stopped, err := instanceManager.GetInstance(ctx, inst.Id) + require.NoError(t, err) + assert.Empty(t, stopped.GPUDevicePath, "assignment metadata should be cleared on stop") + assertVGPUReleased(t, inst.GPUFramework, inst.GPUDevicePath) + }) + + t.Log("Step 11: Starting instance to reacquire a vGPU...") + started, err := instanceManager.StartInstance(ctx, inst.Id, instances.StartInstanceRequest{}) + require.NoError(t, err, "start should succeed") + + t.Run("VGPUReacquiredOnStart", func(t *testing.T) { + require.NotEmpty(t, started.GPUDevicePath, "start should assign a vGPU") + assert.Equal(t, inst.GPUFramework, started.GPUFramework, "framework should match") + assertVGPUAssigned(t, started.GPUFramework, started.GPUDevicePath) }) t.Log("✅ vGPU test PASSED!") } +func assertVGPUAssigned(t *testing.T, framework devices.VGPUFramework, devicePath string) { + t.Helper() + switch framework { + case devices.VGPUFrameworkMdev: + _, err := os.Stat(devicePath) + assert.NoError(t, err, "mdev device should exist at %s", devicePath) + case devices.VGPUFrameworkVendorVFIO: + data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type")) + require.NoError(t, err, "VF should expose current_vgpu_type") + assert.NotEqual(t, "0", strings.TrimSpace(string(data)), "VF should have a vGPU type assigned") + default: + t.Fatalf("unexpected vGPU framework %q", framework) + } +} + +func assertVGPUReleased(t *testing.T, framework devices.VGPUFramework, devicePath string) { + t.Helper() + switch framework { + case devices.VGPUFrameworkMdev: + _, err := os.Stat(devicePath) + assert.True(t, os.IsNotExist(err), "mdev device should be gone from %s", devicePath) + case devices.VGPUFrameworkVendorVFIO: + data, err := os.ReadFile(filepath.Join(devicePath, "nvidia", "current_vgpu_type")) + require.NoError(t, err, "VF should expose current_vgpu_type") + assert.Equal(t, "0", strings.TrimSpace(string(data)), "VF assignment should be released") + default: + t.Fatalf("unexpected vGPU framework %q", framework) + } +} + // checkVGPUTestPrerequisites checks if vGPU test can run. // Returns (skipReason, profileName) - skipReason is empty if all prerequisites are met. func checkVGPUTestPrerequisites() (string, string) { @@ -245,10 +316,18 @@ func checkVGPUTestPrerequisites() (string, string) { return "vGPU test requires root (sudo) for mdev creation", "" } - // Check for vGPU mode (SR-IOV VFs present) - mode := devices.DetectHostGPUMode() - if mode != devices.GPUModeVGPU { - return "vGPU test requires SR-IOV VFs in /sys/class/mdev_bus/", "" + // Check for a vGPU framework (mdev or vendor VFIO) + framework, _, err := devices.DiscoverVGPU() + if err != nil { + return "vGPU test failed to discover vGPU framework: " + err.Error(), "" + } + if framework == devices.VGPUFrameworkNone { + return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", "" + } + if framework == devices.VGPUFrameworkVendorVFIO { + // CreateVGPU rejects vendor VFIO until the instance lifecycle + // integration lands. + return "vGPU test requires the vendor VFIO instance lifecycle integration", "" } // Check for available profiles diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 54a19c472..0cba075fa 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -8,16 +8,17 @@ hypeman supports two GPU modes, automatically detected based on host configurati | Mode | Description | Use Case | |------|-------------|----------| -| **vGPU (SR-IOV)** | Virtual GPUs via mdev on SR-IOV VFs | Multi-tenant, shared GPU resources | +| **vGPU (SR-IOV)** | Virtual GPUs on SR-IOV VFs via mdev or vendor VFIO | Multi-tenant, shared GPU resources | | **Passthrough** | Whole GPU VFIO passthrough | Dedicated GPU per instance | The host's GPU mode is determined by the host driver configuration: -- If `/sys/class/mdev_bus/` contains VFs → vGPU mode -- If NVIDIA GPUs are available for VFIO → passthrough mode +- If `/sys/class/mdev_bus/` contains VFs → mdev vGPU mode +- If VFs expose `/sys/bus/pci/devices//nvidia/current_vgpu_type` → vendor VFIO vGPU mode +- If NVIDIA GPUs are available for whole-device VFIO → passthrough mode ## vGPU Mode (Recommended) -vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs), each capable of hosting an mdev (mediated device) representing a vGPU. +vGPU mode uses NVIDIA's SR-IOV technology to create Virtual Functions (VFs). Hosts on older kernels represent each vGPU as an mdev. Hosts using NVIDIA's vendor VFIO framework assign the profile directly to the VF through `current_vgpu_type`. ### How It Works @@ -74,7 +75,7 @@ curl -X POST http://localhost:4973/instances \ }' ``` -The response includes the assigned mdev UUID: +On an mdev host, the response also includes the assigned mdev UUID: ```json { @@ -87,19 +88,23 @@ The response includes the assigned mdev UUID: } ``` -### Ephemeral mdev Lifecycle +### Ephemeral vGPU Lifecycle -mdev devices are **ephemeral**: created on instance start, destroyed on instance delete. +vGPU assignments are created on instance start and released on stop or delete. Hypeman creates/removes an mdev on mdev hosts and writes the profile ID/`0` to `current_vgpu_type` on vendor VFIO hosts. ``` -Instance Create → Create mdev → Attach to VM → Instance Running -Instance Delete → Stop VM → Destroy mdev → VF available again +Instance Create → Assign profile to VF → Attach VF to VM → Instance Running +Instance Stop/Delete → Release profile → VF available again ``` -This ensures: -- **Security**: No VRAM data leakage between instances -- **Clean state**: Fresh vGPU for each instance -- **Automatic cleanup**: Orphaned mdevs cleaned up on server restart +Hypeman reconciles orphaned assignments on server restart while preserving devices held open by a running VMM. + +### Hypervisor Support + +Hypervisor selection for vGPU instances is caller policy; hypeman does not enforce it. In practice **QEMU is the only hypervisor with working vGPU support**: + +- **QEMU**: fully supported and validated on both mdev and vendor VFIO hosts. +- **Cloud Hypervisor**: vendor VFIO vGPUs are known broken upstream ([cloud-hypervisor#7572](https://github.com/cloud-hypervisor/cloud-hypervisor/issues/7572)) — the VM boots and the VF attaches, but VFIO region reads fail, the guest driver cannot initialize, and the vGPU is non-functional. Do not place vGPU instances on Cloud Hypervisor. ## Passthrough Mode @@ -241,7 +246,8 @@ To upgrade the NVIDIA driver version: 1. Check host GPU mode detection: ```bash - ls /sys/class/mdev_bus/ # Should show VFs for vGPU mode + ls /sys/class/mdev_bus/ + find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type' ``` 2. Verify NVIDIA drivers are loaded on host: @@ -265,17 +271,18 @@ curl -s http://localhost:4973/resources | jq '.gpu.profiles' curl http://localhost:4973/instances//logs?source=app ``` -### mdev creation fails +### vGPU assignment fails -1. Check if VFs are available: - ```bash - ls /sys/class/mdev_bus/ - ``` +Check the files for the framework detected on the host: -2. Verify mdev types: - ```bash - cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances - ``` +```bash +# mdev +cat /sys/class/mdev_bus/*/mdev_supported_types/*/available_instances + +# vendor VFIO +cat /sys/bus/pci/devices/*/nvidia/creatable_vgpu_types +cat /sys/bus/pci/devices/*/nvidia/current_vgpu_type +``` ## Performance Tuning diff --git a/lib/devices/gpu_mode.go b/lib/devices/gpu_mode.go deleted file mode 100644 index 40b3b2ba0..000000000 --- a/lib/devices/gpu_mode.go +++ /dev/null @@ -1,30 +0,0 @@ -package devices - -import ( - "os" -) - -// DetectHostGPUMode determines the host's GPU configuration mode. -// -// Returns: -// - GPUModeVGPU if /sys/class/mdev_bus has entries (SR-IOV VFs present) -// - GPUModePassthrough if NVIDIA GPUs are available for VFIO passthrough -// - GPUModeNone if no GPUs are available -// -// Note: A host is configured for either vGPU or passthrough, not both, -// because the host driver determines which mode is available. -func DetectHostGPUMode() GPUMode { - // Check for vGPU mode first (SR-IOV VFs present) - entries, err := os.ReadDir("/sys/class/mdev_bus") - if err == nil && len(entries) > 0 { - return GPUModeVGPU - } - - // Check for passthrough mode (physical GPUs available) - gpus, err := DiscoverAvailableDevices() - if err == nil && len(gpus) > 0 { - return GPUModePassthrough - } - - return GPUModeNone -} diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 1427a5095..4274063ed 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -12,10 +12,9 @@ func SetGPUProfileCacheTTL(ttl string) { // No-op on macOS } -// DiscoverVFs returns an empty list on macOS. -// SR-IOV Virtual Functions are not available on macOS. -func DiscoverVFs() ([]VirtualFunction, error) { - return []VirtualFunction{}, nil +// DiscoverVGPU reports no vGPU framework on macOS. +func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) { + return VGPUFrameworkNone, nil, nil } // ListGPUProfiles returns an empty list on macOS. @@ -24,7 +23,7 @@ func ListGPUProfiles() ([]GPUProfile, error) { } // ListGPUProfilesWithVFs returns an empty list on macOS. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { return []GPUProfile{}, nil } @@ -59,6 +58,10 @@ func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { return nil } +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + return nil +} + // ReconcileMdevs is a no-op on macOS. func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { return nil diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 1a398a418..61e55b599 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -89,20 +89,23 @@ func getCachedProfiles(firstVF string) []profileMetadata { return cachedProfiles } -// DiscoverVFs returns all SR-IOV Virtual Functions available for vGPU. -// These are discovered by scanning /sys/class/mdev_bus/ which contains -// VFs that can host mdev devices. -func DiscoverVFs() ([]VirtualFunction, error) { - entries, err := os.ReadDir(mdevBusPath) +// discoverMdevVFs returns all SR-IOV Virtual Functions available for vGPU, +// discovered by scanning /sys/class/mdev_bus/. +func discoverMdevVFs() ([]VirtualFunction, error) { + return discoverMdevVFsWith(mdevBusPath, pciDevicesPath, ListMdevDevices) +} + +func discoverMdevVFsWith(busPath, pciPath string, listMdevs func() ([]MdevDevice, error)) ([]VirtualFunction, error) { + entries, err := os.ReadDir(busPath) if err != nil { if os.IsNotExist(err) { - return nil, nil // No mdev_bus means no vGPU support + return nil, nil // No mdev_bus means no mdev vGPU support } return nil, fmt.Errorf("read mdev_bus: %w", err) } // List mdevs once and build a lookup map to avoid O(n*m) performance - mdevs, _ := ListMdevDevices() + mdevs, _ := listMdevs() mdevByVF := make(map[string]bool, len(mdevs)) for _, mdev := range mdevs { mdevByVF[mdev.VFAddress] = true @@ -111,10 +114,27 @@ func DiscoverVFs() ([]VirtualFunction, error) { var vfs []VirtualFunction for _, entry := range entries { vfAddr := entry.Name() + types, err := os.ReadDir(filepath.Join(busPath, vfAddr, "mdev_supported_types")) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read mdev supported types for VF %s: %w", vfAddr, err) + } + usable := false + for _, typ := range types { + if typ.IsDir() { + usable = true + break + } + } + if !usable { + continue + } // Find parent GPU by checking physfn symlink // VFs have a physfn symlink pointing to their parent Physical Function - physfnPath := filepath.Join("/sys/bus/pci/devices", vfAddr, "physfn") + physfnPath := filepath.Join(pciPath, vfAddr, "physfn") parentGPU := "" if target, err := os.Readlink(physfnPath); err == nil { parentGPU = filepath.Base(target) @@ -133,20 +153,9 @@ func DiscoverVFs() ([]VirtualFunction, error) { return vfs, nil } -// ListGPUProfiles returns available vGPU profiles with availability counts. -// Profiles are discovered from the first VF's mdev_supported_types directory. -func ListGPUProfiles() ([]GPUProfile, error) { - vfs, err := DiscoverVFs() - if err != nil { - return nil, err - } - return ListGPUProfilesWithVFs(vfs) -} - -// ListGPUProfilesWithVFs returns available vGPU profiles using pre-discovered VFs. -// This avoids redundant VF discovery when the caller already has the list. -// Uses parallel sysfs reads for fast availability counting. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +// listMdevGPUProfilesWithVFs returns available vGPU profiles using +// pre-discovered VFs. Uses parallel sysfs reads for fast availability counting. +func listMdevGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { if len(vfs) == 0 { return nil, nil } @@ -305,7 +314,7 @@ func countAvailableForSingleProfile(freeVFsByParent map[string][]VirtualFunction // findProfileType finds the internal type name (e.g., "nvidia-556") for a profile name (e.g., "L40S-1Q") func findProfileType(profileName string) (string, error) { - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil || len(vfs) == 0 { return "", fmt.Errorf("no VFs available") } @@ -531,7 +540,7 @@ func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevic } // Discover all VFs - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return nil, fmt.Errorf("discover VFs: %w", err) } @@ -697,7 +706,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log := logger.FromContext(ctx) _ = instanceInfos - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return fmt.Errorf("discover managed VFs: %w", err) } diff --git a/lib/devices/types.go b/lib/devices/types.go index 809d669fe..31ebd90ef 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -63,12 +63,13 @@ type GPUMode string type VGPUFramework string const ( - VGPUFrameworkNone VGPUFramework = "" - VGPUFrameworkMdev VGPUFramework = "mdev" + VGPUFrameworkNone VGPUFramework = "" + VGPUFrameworkMdev VGPUFramework = "mdev" + VGPUFrameworkVendorVFIO VGPUFramework = "vendor-vfio" // GPUModePassthrough indicates whole GPU VFIO passthrough GPUModePassthrough GPUMode = "passthrough" - // GPUModeVGPU indicates SR-IOV + mdev based vGPU + // GPUModeVGPU indicates vGPU mode GPUModeVGPU GPUMode = "vgpu" // GPUModeNone indicates no GPU available GPUModeNone GPUMode = "none" @@ -76,9 +77,10 @@ const ( // VirtualFunction represents an SR-IOV Virtual Function for vGPU type VirtualFunction struct { - PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" - ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" - Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" + ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" + Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + ProfileType string `json:"profile_type,omitempty"` } // VGPUAssignment identifies an existing vGPU assignment to release. @@ -86,6 +88,7 @@ type VGPUAssignment struct { Framework VGPUFramework DevicePath string MdevUUID string + InstanceID string } type VGPUDevice struct { @@ -97,6 +100,16 @@ type VGPUDevice struct { MdevUUID string } +// VGPUCreateCleanupPendingError reports a failed create whose assignment could +// not be rolled back. Device identifies the assignment that still needs release. +type VGPUCreateCleanupPendingError struct { + Device VGPUDevice + Err error +} + +func (e *VGPUCreateCleanupPendingError) Error() string { return e.Err.Error() } +func (e *VGPUCreateCleanupPendingError) Unwrap() error { return e.Err } + // MdevDevice represents an active mediated device (vGPU instance) type MdevDevice struct { UUID string `json:"uuid"` // e.g., "aa618089-8b16-4d01-a136-25a0f3c73123" diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go new file mode 100644 index 000000000..566a0d772 --- /dev/null +++ b/lib/devices/vendor_vfio_linux.go @@ -0,0 +1,533 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + + "github.com/kernel/hypeman/lib/logger" +) + +const ( + pciDevicesPath = "/sys/bus/pci/devices" + vfioDevicesPath = "/dev/vfio/devices" +) + +type vendorVFIOSysfs struct { + pciDevicesPath string + procPath string + vfioDevicesPath string + owners map[string]string + framebufferByType map[string]int +} + +var ( + hostVendorVFIO = vendorVFIOSysfs{ + pciDevicesPath: pciDevicesPath, + procPath: procPath, + vfioDevicesPath: vfioDevicesPath, + owners: make(map[string]string), + framebufferByType: make(map[string]int), + } + vendorVFIOMu sync.Mutex +) + +func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) { + entries, err := os.ReadDir(s.pciDevicesPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read PCI devices: %w", err) + } + + vfs := make([]VirtualFunction, 0) + var vfErrs []error + for _, entry := range entries { + vfPath := filepath.Join(s.pciDevicesPath, entry.Name()) + nvidiaPath := filepath.Join(vfPath, "nvidia") + if _, err := os.Stat(filepath.Join(nvidiaPath, "creatable_vgpu_types")); err != nil { + if os.IsNotExist(err) { + continue + } + vfErrs = append(vfErrs, fmt.Errorf("stat creatable vGPU types for VF %s: %w", entry.Name(), err)) + continue + } + + currentType, err := readCurrentVGPUType(filepath.Join(nvidiaPath, "current_vgpu_type")) + if err != nil { + vfErrs = append(vfErrs, fmt.Errorf("read current vGPU type for VF %s: %w", entry.Name(), err)) + continue + } + + parentGPU := "" + if target, err := os.Readlink(filepath.Join(vfPath, "physfn")); err == nil { + parentGPU = filepath.Base(target) + } + vfs = append(vfs, VirtualFunction{ + PCIAddress: entry.Name(), + ParentGPU: parentGPU, + Allocated: currentType != "0", + ProfileType: currentType, + }) + } + if len(vfErrs) > 0 { + // One unreadable VF must not blank out the host's GPU capacity: skip + // it and keep the readable inventory. A skipped VF is never selected + // for placement and never reconciled, both safe directions. Only when + // no VF is readable does discovery fail, so a wholesale sysfs outage + // cannot demote a vGPU host to passthrough while assignments exist. + if len(vfs) == 0 { + return nil, errors.Join(vfErrs...) + } + slog.Default().Warn("skipping unreadable vendor VFIO VFs", "error", errors.Join(vfErrs...)) + } + + sort.Slice(vfs, func(i, j int) bool { return vfs[i].PCIAddress < vfs[j].PCIAddress }) + return vfs, nil +} + +// listProfiles counts each free VF advertising a type as one creatable +// instance, matching the driver-reported units that mdev sums through +// available_instances. This is a best-effort snapshot because creating on one +// VF may revoke the type from siblings that share its GPU framebuffer. +func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) { + profilesByType := make(map[string]profileMetadata) + creatableVFs := make(map[string]int) + for _, vf := range vfs { + creatable, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + // Mirror discoverVFs: one unreadable VF must not blank the host's + // advertised capacity. Skipping only underreports availability, + // the safe direction for status and admission. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue + } + for _, profile := range creatable { + profilesByType[profile.TypeName] = profile + if !vf.Allocated { + creatableVFs[profile.TypeName]++ + } + } + } + + metadata := make([]profileMetadata, 0, len(profilesByType)) + for _, profile := range profilesByType { + metadata = append(metadata, profile) + } + sort.Slice(metadata, func(i, j int) bool { return metadata[i].Name < metadata[j].Name }) + + profiles := make([]GPUProfile, 0, len(metadata)) + for _, profile := range metadata { + profiles = append(profiles, GPUProfile{ + Name: profile.Name, + FramebufferMB: profile.FramebufferMB, + Available: creatableVFs[profile.TypeName], + }) + } + return profiles, nil +} + +func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + + vfs, err := s.discoverVFs() + if err != nil { + return nil, err + } + metadata, err := s.profileMetadata(vfs) + if err != nil { + return nil, err + } + + var requested profileMetadata + found := false + for _, profile := range metadata { + if profile.Name == profileName { + requested = profile + found = true + break + } + } + if !found { + if len(metadata) == 0 && len(vfs) > 0 { + return nil, fmt.Errorf("no creatable vGPU profiles on any VF, GPUs may be at capacity: profile %q", profileName) + } + return nil, fmt.Errorf("profile %q is not creatable on any VF (unknown profile or insufficient capacity)", profileName) + } + + targetVF, err := s.selectLeastLoadedVF(vfs, requested.TypeName) + if err != nil { + return nil, err + } + if targetVF == "" { + return nil, fmt.Errorf("no available VF for profile %q", profileName) + } + + currentTypePath := filepath.Join(s.pciDevicesPath, targetVF, "nvidia", "current_vgpu_type") + if err := os.WriteFile(currentTypePath, []byte(requested.TypeName), 0200); err != nil { + return nil, fmt.Errorf("create vGPU on VF %s: %w", targetVF, err) + } + device := VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: targetVF, + ProfileType: requested.TypeName, + ProfileName: profileName, + SysfsPath: filepath.Join(s.pciDevicesPath, targetVF), + } + currentType, err := readCurrentVGPUType(currentTypePath) + if err != nil { + verifyErr := fmt.Errorf("verify vGPU on VF %s: %w", targetVF, err) + return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) + } + if currentType != requested.TypeName { + verifyErr := fmt.Errorf("verify vGPU on VF %s: type is %s, want %s", targetVF, currentType, requested.TypeName) + return nil, s.rollbackCreate(currentTypePath, targetVF, instanceID, device, verifyErr) + } + s.owners[targetVF] = instanceID + + logger.FromContext(ctx).InfoContext(ctx, "created vendor VFIO vGPU", + "profile", profileName, + "vf", targetVF, + "instance_id", instanceID, + ) + return &device, nil +} + +func (s vendorVFIOSysfs) destroy(ctx context.Context, vfAddress, instanceID string) error { + return s.destroyWithOpenPaths(ctx, vfAddress, instanceID, nil) +} + +func (s vendorVFIOSysfs) destroyWithOpenPaths(ctx context.Context, vfAddress, instanceID string, openPaths map[string]struct{}) error { + vendorVFIOMu.Lock() + defer vendorVFIOMu.Unlock() + + log := logger.FromContext(ctx) + currentTypePath := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type") + currentType, err := readCurrentVGPUType(currentTypePath) + if err != nil { + if os.IsNotExist(err) { + delete(s.owners, vfAddress) + return nil + } + return fmt.Errorf("read current vGPU type for VF %s: %w", vfAddress, err) + } + if currentType == "0" { + delete(s.owners, vfAddress) + return nil + } + + if owner, ok := s.owners[vfAddress]; ok { + if instanceID == "" { + return fmt.Errorf("cannot release vendor VFIO vGPU on VF %s without instance ID", vfAddress) + } + if owner != instanceID { + log.WarnContext(ctx, "skipping vendor VFIO vGPU release owned by another instance", + "vf", vfAddress, + "owner_instance_id", owner, + "requesting_instance_id", instanceID, + ) + return nil + } + } + + if openPaths == nil { + if openPaths, err = s.openVFIOPaths(); err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) + } + } + inUse, err := s.vfioDeviceInUse(vfAddress, openPaths) + if err != nil { + return fmt.Errorf("check vendor VFIO vGPU usage for VF %s: %w", vfAddress, err) + } + if inUse { + return fmt.Errorf("vendor VFIO vGPU on VF %s is still in use", vfAddress) + } + + if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + return fmt.Errorf("destroy vGPU on VF %s: %w", vfAddress, err) + } + delete(s.owners, vfAddress) + log.InfoContext(ctx, "destroyed vendor VFIO vGPU", "vf", vfAddress) + return nil +} + +func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + vfs, err := s.discoverVFs() + if err != nil { + return err + } + log := logger.FromContext(ctx) + protectedVFs := make(map[string]struct{}, len(protectedDevicePaths)) + for path := range protectedDevicePaths { + protectedVFs[filepath.Base(path)] = struct{}{} + } + var openPaths map[string]struct{} + for _, vf := range vfs { + if !vf.Allocated { + continue + } + if _, ok := protectedVFs[vf.PCIAddress]; ok { + log.DebugContext(ctx, "skipping vendor VFIO vGPU held by a live instance", "vf", vf.PCIAddress) + continue + } + if openPaths == nil { + if openPaths, err = s.openVFIOPaths(); err != nil { + return fmt.Errorf("scan open VFIO handles: %w", err) + } + } + inUse, err := s.vfioDeviceInUse(vf.PCIAddress, openPaths) + if err != nil { + log.WarnContext(ctx, "failed to check vendor VFIO vGPU usage", "vf", vf.PCIAddress, "error", err) + continue + } + if inUse { + continue + } + if err := s.destroyWithOpenPaths(ctx, vf.PCIAddress, "", openPaths); err != nil { + log.WarnContext(ctx, "failed to destroy orphaned vendor VFIO vGPU", "vf", vf.PCIAddress, "error", err) + } + } + return nil +} + +func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) { + usageByGPU := make(map[string]int) + unknownUsageByGPU := make(map[string]bool) + freeByGPU := make(map[string][]VirtualFunction) + for _, vf := range vfs { + if vf.Allocated { + // framebufferByType only covers currently creatable profiles, so + // after a restart an allocated type can be missing when its + // capacity is exhausted. Prefer GPUs whose load is fully known + // instead of rejecting placement outright; the kernel driver + // still enforces real capacity through creatable_vgpu_types. + framebuffer, ok := s.framebufferByType[vf.ProfileType] + if !ok { + unknownUsageByGPU[vf.ParentGPU] = true + continue + } + usageByGPU[vf.ParentGPU] += framebuffer + continue + } + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + // An unreadable free VF is just not a placement candidate. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue + } + for _, profile := range profiles { + if profile.TypeName == profileType { + freeByGPU[vf.ParentGPU] = append(freeByGPU[vf.ParentGPU], vf) + break + } + } + } + + gpus := make([]string, 0, len(freeByGPU)) + for gpu := range freeByGPU { + gpus = append(gpus, gpu) + } + sort.Slice(gpus, func(i, j int) bool { + if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] { + return !unknownUsageByGPU[gpus[i]] + } + if usageByGPU[gpus[i]] == usageByGPU[gpus[j]] { + return gpus[i] < gpus[j] + } + return usageByGPU[gpus[i]] < usageByGPU[gpus[j]] + }) + if len(gpus) == 0 { + return "", nil + } + return freeByGPU[gpus[0]][0].PCIAddress, nil +} + +func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetadata, error) { + profilesByType := make(map[string]profileMetadata) + for _, vf := range vfs { + profiles, err := s.readCreatableProfiles(vf.PCIAddress) + if err != nil { + // Mirror listProfiles: an unreadable VF must not fail placement + // while /resources still advertises the remaining capacity. + slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err) + continue + } + for _, profile := range profiles { + profilesByType[profile.TypeName] = profile + s.framebufferByType[profile.TypeName] = profile.FramebufferMB + } + } + profiles := make([]profileMetadata, 0, len(profilesByType)) + for _, profile := range profilesByType { + profiles = append(profiles, profile) + } + sort.Slice(profiles, func(i, j int) bool { return profiles[i].Name < profiles[j].Name }) + return profiles, nil +} + +func (s vendorVFIOSysfs) readCreatableProfiles(vfAddress string) ([]profileMetadata, error) { + path := filepath.Join(s.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read creatable vGPU types for VF %s: %w", vfAddress, err) + } + return parseCreatableVGPUTypes(string(data)) +} + +func (s vendorVFIOSysfs) vfioDeviceInUse(vfAddress string, openPaths map[string]struct{}) (bool, error) { + devicePaths := make([]string, 0, 2) + probeErrs := make([]error, 0, 2) + + vfioDevices, err := os.ReadDir(filepath.Join(s.pciDevicesPath, vfAddress, "vfio-dev")) + if err != nil { + if !os.IsNotExist(err) { + probeErrs = append(probeErrs, fmt.Errorf("read VFIO devices for VF %s: %w", vfAddress, err)) + } + } else { + for _, device := range vfioDevices { + devicePaths = append(devicePaths, filepath.Join(s.vfioDevicesPath, device.Name())) + } + } + + target, err := os.Readlink(filepath.Join(s.pciDevicesPath, vfAddress, "iommu_group")) + if err != nil { + if !os.IsNotExist(err) { + probeErrs = append(probeErrs, fmt.Errorf("read IOMMU group for VF %s: %w", vfAddress, err)) + } + } else { + devicePaths = append(devicePaths, filepath.Join(filepath.Dir(s.vfioDevicesPath), filepath.Base(target))) + } + + for _, path := range devicePaths { + if _, ok := openPaths[path]; ok { + return true, nil + } + } + if len(probeErrs) > 0 { + return false, errors.Join(probeErrs...) + } + return false, nil +} + +// openVFIOPaths hard-fails on any unreadable /proc entry, unlike mdev's +// isVFIOGroupInUse which skips them. That is deliberate: this scan authorizes +// clearing current_vgpu_type on a VF path that is reused across assignments, +// so an incomplete scan must fail the release (which retains metadata for a +// later retry) rather than risk a false "not in use" answer. +func (s vendorVFIOSysfs) openVFIOPaths() (map[string]struct{}, error) { + processes, err := os.ReadDir(s.procPath) + if err != nil { + return nil, err + } + prefix := filepath.Dir(s.vfioDevicesPath) + string(filepath.Separator) + open := make(map[string]struct{}) + for _, process := range processes { + if _, err := strconv.Atoi(process.Name()); err != nil { + continue + } + fdPath := filepath.Join(s.procPath, process.Name(), "fd") + fds, err := os.ReadDir(fdPath) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read process %s file descriptors: %w", process.Name(), err) + } + for _, fd := range fds { + target, err := os.Readlink(filepath.Join(fdPath, fd.Name())) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, fmt.Errorf("read process %s file descriptor %s: %w", process.Name(), fd.Name(), err) + } + if strings.HasPrefix(target, prefix) { + open[target] = struct{}{} + } + } + } + return open, nil +} + +func parseCreatableVGPUTypes(value string) ([]profileMetadata, error) { + profiles := make([]profileMetadata, 0) + for lineNumber, line := range strings.Split(value, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + typeID, name, found := strings.Cut(line, ":") + typeID = strings.TrimSpace(typeID) + name = strings.TrimSpace(name) + if typeID == "ID" { + continue + } + if !found || name == "" { + return nil, fmt.Errorf("parse creatable vGPU types line %d: %q", lineNumber+1, line) + } + if _, err := strconv.Atoi(typeID); err != nil { + return nil, fmt.Errorf("parse vGPU type ID %q: %w", typeID, err) + } + profiles = append(profiles, profileMetadata{ + TypeName: typeID, + Name: name, + FramebufferMB: framebufferFromProfileName(name), + }) + } + return profiles, nil +} + +// framebufferFromProfileName parses the framebuffer size from names like +// "NVIDIA L40S-12Q". NVIDIA's 512 MB 0Q/0B profiles parse as 0 and would read +// as free VRAM in least-loaded placement; no supported GPU exposes them today, +// so placement only skews if such a profile ever appears. +func framebufferFromProfileName(name string) int { + series := strings.LastIndexAny(name, "ABCQ") + if series <= 0 { + return 0 + } + dash := strings.LastIndex(name[:series], "-") + if dash < 0 { + return 0 + } + gb, err := strconv.Atoi(name[dash+1 : series]) + if err != nil { + return 0 + } + return gb * 1024 +} + +func (s vendorVFIOSysfs) rollbackCreate(currentTypePath, vfAddress, instanceID string, device VGPUDevice, verifyErr error) error { + if err := os.WriteFile(currentTypePath, []byte("0"), 0200); err != nil { + s.owners[vfAddress] = instanceID + return &VGPUCreateCleanupPendingError{ + Device: device, + Err: errors.Join(verifyErr, fmt.Errorf("roll back vGPU on VF %s: %w", vfAddress, err)), + } + } + return verifyErr +} + +func readCurrentVGPUType(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + value := strings.TrimSpace(string(data)) + if _, err := strconv.Atoi(value); err != nil { + return "", fmt.Errorf("invalid current vGPU type %q", value) + } + return value, nil +} diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go new file mode 100644 index 000000000..4f2e40844 --- /dev/null +++ b/lib/devices/vendor_vfio_linux_test.go @@ -0,0 +1,573 @@ +//go:build linux + +package devices + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const testCreatableTypes = `ID : vGPU Name +1147 : NVIDIA L40S-1Q +1148 : NVIDIA L40S-2Q +1159 : NVIDIA L40S-48Q +` + +func TestParseCreatableVGPUTypes(t *testing.T) { + t.Parallel() + + profiles, err := parseCreatableVGPUTypes(testCreatableTypes) + require.NoError(t, err) + require.Len(t, profiles, 3) + assert.Equal(t, profileMetadata{TypeName: "1147", Name: "NVIDIA L40S-1Q", FramebufferMB: 1024}, profiles[0]) + assert.Equal(t, profileMetadata{TypeName: "1159", Name: "NVIDIA L40S-48Q", FramebufferMB: 48 * 1024}, profiles[2]) +} + +func TestVendorVFIODiscoverSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + // Replace one VF's current_vgpu_type with a directory so reading it fails. + badType := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "current_vgpu_type") + require.NoError(t, os.Remove(badType)) + require.NoError(t, os.Mkdir(badType, 0755)) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.Equal(t, "0000:82:00.4", vfs[0].PCIAddress) +} + +func TestVendorVFIODiscoverFailsWhenNoVFIsReadable(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + badType := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type") + require.NoError(t, os.Remove(badType)) + require.NoError(t, os.Mkdir(badType, 0755)) + + // With every VF unreadable, discovery must fail rather than report an + // empty inventory that would demote the host to passthrough. + _, err := sysfs.discoverVFs() + require.Error(t, err) +} + +func TestVendorVFIOListProfilesSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + badCreatable := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Remove(badCreatable)) + require.NoError(t, os.Mkdir(badCreatable, 0755)) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + require.NotEmpty(t, profiles, "readable VFs must keep advertising capacity") + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-1Q"), + "only the readable VF counts toward availability") +} + +func TestVendorVFIOCreateSkipsUnreadableVF(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + badCreatable := filepath.Join(sysfs.pciDevicesPath, "0000:82:00.5", "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Remove(badCreatable)) + require.NoError(t, os.Mkdir(badCreatable, 0755)) + + // Create places on the readable VF instead of failing the placement. + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "inst-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.4", device.VFAddress) +} + +func TestVendorVFIOCreateAndDestroy(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + require.Len(t, vfs, 1) + assert.False(t, vfs[0].Allocated) + assert.Equal(t, "0000:82:00.0", vfs[0].ParentGPU) + + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-2Q")) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, VGPUFrameworkVendorVFIO, device.Framework) + assert.Equal(t, "0000:82:00.4", device.VFAddress) + assert.Equal(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4"), device.SysfsPath) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "instance-1")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIODestroySkipsAssignmentOwnedByAnotherInstance(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "stale-instance")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") + + require.NoError(t, sysfs.destroy(context.Background(), device.VFAddress, "instance-1")) + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIODestroyRejectsMissingInstanceIDForOwnedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.NoError(t, err) + + err = sysfs.destroy(context.Background(), device.VFAddress, "") + require.ErrorContains(t, err, "without instance ID") + assertFileValue(t, filepath.Join(device.SysfsPath, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIODestroyRetainsAssignmentInUse(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio42") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(activeDevice, filepath.Join(fdDir, "5"))) + + err := sysfs.destroy(context.Background(), vfAddress, "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "still in use") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIODestroyReleasesUnboundVF(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + unbind func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) + }{ + { + name: "missing vfio device directory", + unbind: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.RemoveAll(filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev"))) + }, + }, + { + name: "missing iommu group symlink", + unbind: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.Remove(filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group"))) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + tt.unbind(t, sysfs, vfAddress) + + require.NoError(t, sysfs.destroy(context.Background(), vfAddress, "instance-1")) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "0") + }) + } +} + +func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + remove func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) + open func(sysfs testVendorVFIOSysfs) string + }{ + { + name: "missing iommu group", + remove: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.Remove(filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group"))) + }, + open: func(sysfs testVendorVFIOSysfs) string { + return filepath.Join(sysfs.vfioDevicesPath, "vfio42") + }, + }, + { + name: "missing vfio device directory", + remove: func(t *testing.T, sysfs testVendorVFIOSysfs, vfAddress string) { + require.NoError(t, os.RemoveAll(filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev"))) + }, + open: func(sysfs testVendorVFIOSysfs) string { + return filepath.Join(filepath.Dir(sysfs.vfioDevicesPath), "42") + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + tt.remove(t, sysfs, vfAddress) + + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(tt.open(sysfs), filepath.Join(fdDir, "5"))) + + err := sysfs.destroy(context.Background(), vfAddress, "instance-1") + require.ErrorContains(t, err, "still in use") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") + }) + } +} + +func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "0", testCreatableTypes) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"), + "each free VF advertising the type counts as one creatable instance") +} + +func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "ID : vGPU Name\n") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "1159", "") + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + profiles, err := sysfs.listProfiles(vfs) + require.NoError(t, err) + assert.Empty(t, profiles) + + _, err = sysfs.create(context.Background(), "NVIDIA L40S-2Q", "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "GPUs may be at capacity") +} + +func TestVendorVFIOCreateReportsAmbiguousMissingProfile(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "ID : vGPU Name\n1148 : NVIDIA L40S-2Q\n") + + _, err := sysfs.create(context.Background(), "NVIDIA L40S-48Q", "instance-1") + require.Error(t, err) + assert.ErrorContains(t, err, "not creatable on any VF") + assert.ErrorContains(t, err, "unknown profile or insufficient capacity") +} + +func TestVendorVFIOSelectsLeastLoadedGPU(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOSelectsLeastLoadedGPUWithConsumedType(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", testCreatableTypes) + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", testCreatableTypes) + + vfs, err := sysfs.discoverVFs() + require.NoError(t, err) + _, err = sysfs.profileMetadata(vfs) + require.NoError(t, err) + for _, vfAddress := range []string{"0000:82:00.5", "0000:e3:00.4"} { + creatableTypesPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "creatable_vgpu_types") + require.NoError(t, os.Chmod(creatableTypesPath, 0644)) + require.NoError(t, os.WriteFile(creatableTypesPath, []byte("1147 : NVIDIA L40S-1Q\n"), 0444)) + } + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress) +} + +func TestVendorVFIOPlacementPrefersKnownLoadWhenAllocatedTypeIsUnknown(t *testing.T) { + t.Parallel() + + // Simulates a restart: type 1159 is allocated but no longer creatable + // anywhere, so its framebuffer size is unknown. + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147 : NVIDIA L40S-1Q\n") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "44", "0", "1147 : NVIDIA L40S-1Q\n") + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:e3:00.4", device.VFAddress, "the GPU with unknown load should be picked last") +} + +func TestVendorVFIOPlacesOnGPUWithUnknownLoadWhenItHasTheOnlyCapacity(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1159", "") + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", "1147 : NVIDIA L40S-1Q\n") + + device, err := sysfs.create(context.Background(), "NVIDIA L40S-1Q", "instance-1") + require.NoError(t, err) + assert.Equal(t, "0000:82:00.5", device.VFAddress) +} + +func TestVendorVFIOReconcile(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + activeDevice := filepath.Join(sysfs.vfioDevicesPath, "vfio43") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(activeDevice, filepath.Join(fdDir, "5"))) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcileSkipsProtectedVF(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + protected := map[string]struct{}{ + filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4"): {}, + } + require.NoError(t, sysfs.reconcile(context.Background(), protected)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "1148") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "0") +} + +func TestVendorVFIOReconcilePreservesLegacyGroupFD(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1148", "") + sysfs.addVF(t, "0000:e3:00.0", "0000:e3:00.4", "43", "1148", "") + + legacyGroup := filepath.Join(filepath.Dir(sysfs.vfioDevicesPath), "43") + fdDir := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdDir, 0755)) + require.NoError(t, os.Symlink(legacyGroup, filepath.Join(fdDir, "5"))) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:82:00.4", "nvidia", "current_vgpu_type"), "0") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, "0000:e3:00.4", "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenVFIODeviceProbeFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + vfioDevPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "vfio-dev") + require.NoError(t, os.RemoveAll(vfioDevPath)) + require.NoError(t, os.WriteFile(vfioDevPath, nil, 0644)) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenIOMMUGroupProbeFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + iommuGroupPath := filepath.Join(sysfs.pciDevicesPath, vfAddress, "iommu_group") + require.NoError(t, os.Remove(iommuGroupPath)) + require.NoError(t, os.WriteFile(iommuGroupPath, nil, 0644)) + + require.NoError(t, sysfs.reconcile(context.Background(), nil)) + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenProcFDDirectoryScanFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + processPath := filepath.Join(sysfs.procPath, "123") + require.NoError(t, os.MkdirAll(processPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(processPath, "fd"), nil, 0644)) + + err := sysfs.reconcile(context.Background(), nil) + require.Error(t, err) + assert.ErrorContains(t, err, "read process 123 file descriptors") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestVendorVFIOReconcilePreservesVFWhenProcFDLinkScanFails(t *testing.T) { + t.Parallel() + + sysfs := newTestVendorVFIOSysfs(t) + const vfAddress = "0000:82:00.4" + sysfs.addVF(t, "0000:82:00.0", vfAddress, "42", "1148", "") + + fdPath := filepath.Join(sysfs.procPath, "123", "fd") + require.NoError(t, os.MkdirAll(fdPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(fdPath, "5"), nil, 0644)) + + err := sysfs.reconcile(context.Background(), nil) + require.Error(t, err) + assert.ErrorContains(t, err, "read process 123 file descriptor 5") + assertFileValue(t, filepath.Join(sysfs.pciDevicesPath, vfAddress, "nvidia", "current_vgpu_type"), "1148") +} + +func TestParseCreatableVGPUTypesHeaderOnly(t *testing.T) { + t.Parallel() + + profiles, err := parseCreatableVGPUTypes("ID : vGPU Name\n") + require.NoError(t, err) + assert.Empty(t, profiles) +} + +func TestParseCreatableVGPUTypesRejectsMalformedLine(t *testing.T) { + t.Parallel() + + _, err := parseCreatableVGPUTypes("NVIDIA") + require.Error(t, err) + + _, err = parseCreatableVGPUTypes("not-an-id : NVIDIA L40S-1Q") + require.Error(t, err) +} + +func TestRollbackVendorVFIOCreate(t *testing.T) { + t.Parallel() + + verifyErr := errors.New("verification failed") + device := VGPUDevice{ + Framework: VGPUFrameworkVendorVFIO, + VFAddress: "0000:82:00.4", + SysfsPath: "/sys/bus/pci/devices/0000:82:00.4", + } + t.Run("preserves verification error", func(t *testing.T) { + currentTypePath := filepath.Join(t.TempDir(), "current_vgpu_type") + require.NoError(t, os.WriteFile(currentTypePath, []byte("1148"), 0644)) + sysfs := vendorVFIOSysfs{owners: make(map[string]string)} + + err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) + require.ErrorIs(t, err, verifyErr) + assertFileValue(t, currentTypePath, "0") + assert.Empty(t, sysfs.owners) + }) + + t.Run("retains assignment when rollback fails", func(t *testing.T) { + currentTypePath := filepath.Join(t.TempDir(), "missing", "current_vgpu_type") + sysfs := vendorVFIOSysfs{owners: make(map[string]string)} + + err := sysfs.rollbackCreate(currentTypePath, device.VFAddress, "instance-1", device, verifyErr) + require.ErrorIs(t, err, verifyErr) + assert.ErrorContains(t, err, "roll back vGPU on VF 0000:82:00.4") + var pending *VGPUCreateCleanupPendingError + require.ErrorAs(t, err, &pending) + assert.Equal(t, device, pending.Device) + assert.Equal(t, "instance-1", sysfs.owners[device.VFAddress]) + }) +} + +func profileAvailability(profiles []GPUProfile, name string) int { + for _, profile := range profiles { + if profile.Name == name { + return profile.Available + } + } + return -1 +} + +type testVendorVFIOSysfs struct { + vendorVFIOSysfs +} + +func newTestVendorVFIOSysfs(t *testing.T) testVendorVFIOSysfs { + t.Helper() + root := t.TempDir() + pci := filepath.Join(root, "sys", "bus", "pci", "devices") + proc := filepath.Join(root, "proc") + vfio := filepath.Join(root, "dev", "vfio", "devices") + require.NoError(t, os.MkdirAll(pci, 0755)) + require.NoError(t, os.MkdirAll(proc, 0755)) + require.NoError(t, os.MkdirAll(vfio, 0755)) + return testVendorVFIOSysfs{vendorVFIOSysfs{ + pciDevicesPath: pci, + procPath: proc, + vfioDevicesPath: vfio, + owners: make(map[string]string), + framebufferByType: make(map[string]int), + }} +} + +func (s testVendorVFIOSysfs) addVF(t *testing.T, parent, address, vfioID, currentType, creatableTypes string) { + t.Helper() + parentPath := filepath.Join(s.pciDevicesPath, parent) + vfPath := filepath.Join(s.pciDevicesPath, address) + nvidiaPath := filepath.Join(vfPath, "nvidia") + require.NoError(t, os.MkdirAll(parentPath, 0755)) + require.NoError(t, os.MkdirAll(nvidiaPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte(currentType), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte(creatableTypes), 0444)) + require.NoError(t, os.Symlink(parentPath, filepath.Join(vfPath, "physfn"))) + vfioName := "vfio" + vfioID + require.NoError(t, os.MkdirAll(filepath.Join(vfPath, "vfio-dev", vfioName), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(s.vfioDevicesPath, vfioName), nil, 0600)) + require.NoError(t, os.Symlink(filepath.Join("..", "..", "..", "kernel", "iommu_groups", vfioID), filepath.Join(vfPath, "iommu_group"))) +} + +func assertFileValue(t *testing.T, path, expected string) { + t.Helper() + value, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, expected, string(value)) +} diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index eaf210b42..72fe3b944 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -8,31 +8,118 @@ import ( "path/filepath" ) +// DiscoverVGPU returns the host's active vGPU framework and virtual functions. +func DiscoverVGPU() (VGPUFramework, []VirtualFunction, error) { + return discoverVGPUWith(discoverMdevVFs, hostVendorVFIO.discoverVFs) +} + +func discoverVGPUWith(discoverMdev, discoverVendorVFIO func() ([]VirtualFunction, error)) (VGPUFramework, []VirtualFunction, error) { + vfs, err := discoverMdev() + if err != nil { + return VGPUFrameworkNone, nil, fmt.Errorf("discover mdev VFs: %w", err) + } + if len(vfs) > 0 { + return VGPUFrameworkMdev, vfs, nil + } + + vfs, err = discoverVendorVFIO() + if err != nil { + return VGPUFrameworkNone, nil, fmt.Errorf("discover vendor VFIO VFs: %w", err) + } + if len(vfs) == 0 { + return VGPUFrameworkNone, nil, nil + } + return VGPUFrameworkVendorVFIO, vfs, nil +} + +// ListGPUProfiles returns available vGPU profiles with availability counts. +func ListGPUProfiles() ([]GPUProfile, error) { + framework, vfs, err := DiscoverVGPU() + if err != nil { + return nil, err + } + return ListGPUProfilesWithVFs(framework, vfs) +} + +// ListGPUProfilesWithVFs returns available profiles for discovered VFs. +func ListGPUProfilesWithVFs(framework VGPUFramework, vfs []VirtualFunction) ([]GPUProfile, error) { + switch framework { + case VGPUFrameworkMdev: + return listMdevGPUProfilesWithVFs(vfs) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.listProfiles(vfs) + default: + return nil, nil + } +} + func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { - mdev, err := CreateMdev(ctx, profileName, instanceID) + framework, _, err := DiscoverVGPU() if err != nil { return nil, err } - return &VGPUDevice{ - Framework: VGPUFrameworkMdev, - VFAddress: mdev.VFAddress, - ProfileType: mdev.ProfileType, - ProfileName: mdev.ProfileName, - SysfsPath: mdev.SysfsPath, - MdevUUID: mdev.UUID, - }, nil + switch framework { + case VGPUFrameworkMdev: + mdev, err := CreateMdev(ctx, profileName, instanceID) + if err != nil { + return nil, err + } + return &VGPUDevice{ + Framework: VGPUFrameworkMdev, + VFAddress: mdev.VFAddress, + ProfileType: mdev.ProfileType, + ProfileName: mdev.ProfileName, + SysfsPath: mdev.SysfsPath, + MdevUUID: mdev.UUID, + }, nil + case VGPUFrameworkVendorVFIO: + // The instance lifecycle does not yet persist vendor VFIO assignments + // durably or guard their release against live claims, so keep the + // backend out of the create path until that integration lands. + return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle") + default: + return nil, fmt.Errorf("vGPU framework not available") + } } func DestroyVGPU(ctx context.Context, assignment VGPUAssignment) error { - if assignment.Framework != VGPUFrameworkNone && assignment.Framework != VGPUFrameworkMdev { - return fmt.Errorf("unknown vGPU framework %q", assignment.Framework) + framework := assignment.Framework + if framework == VGPUFrameworkNone && assignment.MdevUUID != "" { + framework = VGPUFrameworkMdev + } + + switch framework { + case VGPUFrameworkMdev: + mdevUUID := assignment.MdevUUID + if mdevUUID == "" { + mdevUUID = filepath.Base(assignment.DevicePath) + } + return DestroyMdev(ctx, mdevUUID) + case VGPUFrameworkVendorVFIO: + return hostVendorVFIO.destroy(ctx, filepath.Base(assignment.DevicePath), assignment.InstanceID) + case VGPUFrameworkNone: + return nil + default: + return fmt.Errorf("unknown vGPU framework %q", framework) } - mdevUUID := assignment.MdevUUID - if mdevUUID == "" { - if assignment.DevicePath == "" { +} + +// ReconcileVGPUs releases orphaned vGPU assignments. +func ReconcileVGPUs(ctx context.Context, protectedDevicePaths map[string]struct{}) error { + framework, _, err := DiscoverVGPU() + if err != nil { + return err + } + + switch framework { + case VGPUFrameworkMdev: + return ReconcileMdevs(ctx, nil) + case VGPUFrameworkVendorVFIO: + if protectedDevicePaths == nil { return nil } - mdevUUID = filepath.Base(assignment.DevicePath) + return hostVendorVFIO.reconcile(ctx, protectedDevicePaths) + default: + return nil } - return DestroyMdev(ctx, mdevUUID) } diff --git a/lib/devices/vgpu_linux_test.go b/lib/devices/vgpu_linux_test.go new file mode 100644 index 000000000..40b987394 --- /dev/null +++ b/lib/devices/vgpu_linux_test.go @@ -0,0 +1,75 @@ +//go:build linux + +package devices + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiscoverVGPUWithPropagatesMdevError(t *testing.T) { + t.Parallel() + + discoveryErr := errors.New("mdev discovery failed") + vendorCalled := false + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return nil, discoveryErr + }, + func() ([]VirtualFunction, error) { + vendorCalled = true + return []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, nil + }, + ) + + require.ErrorIs(t, err, discoveryErr) + assert.Equal(t, VGPUFrameworkNone, framework) + assert.Nil(t, vfs) + assert.False(t, vendorCalled) +} + +func TestDiscoverVGPUWithFallsBackFromTypelessMdevBus(t *testing.T) { + t.Parallel() + + root := t.TempDir() + busPath := filepath.Join(root, "sys", "class", "mdev_bus") + require.NoError(t, os.MkdirAll(filepath.Join(busPath, "0000:82:00.4", "mdev_supported_types"), 0755)) + + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return discoverMdevVFsWith(busPath, filepath.Join(root, "sys", "bus", "pci", "devices"), func() ([]MdevDevice, error) { + return nil, nil + }) + }, + func() ([]VirtualFunction, error) { + return []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, nil + }, + ) + + require.NoError(t, err) + assert.Equal(t, VGPUFrameworkVendorVFIO, framework) + assert.Equal(t, []VirtualFunction{{PCIAddress: "0000:82:00.4"}}, vfs) +} + +func TestDiscoverVGPUWithPropagatesVendorVFIOError(t *testing.T) { + t.Parallel() + + discoveryErr := errors.New("vendor VFIO discovery failed") + framework, vfs, err := discoverVGPUWith( + func() ([]VirtualFunction, error) { + return nil, nil + }, + func() ([]VirtualFunction, error) { + return nil, discoveryErr + }, + ) + + require.ErrorIs(t, err, discoveryErr) + assert.Equal(t, VGPUFrameworkNone, framework) + assert.Nil(t, vfs) +} diff --git a/lib/instances/create.go b/lib/instances/create.go index ed3dde691..63b198cf4 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -301,6 +301,7 @@ func (m *manager) createInstance( Framework: gpuDevice.Framework, DevicePath: gpuDevice.SysfsPath, MdevUUID: gpuDevice.MdevUUID, + InstanceID: id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) diff --git a/lib/instances/start.go b/lib/instances/start.go index 7e7855eac..a6c832450 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -177,6 +177,7 @@ func (m *manager) startInstance( Framework: device.Framework, DevicePath: device.SysfsPath, MdevUUID: device.MdevUUID, + InstanceID: id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index cffe2ac1d..a8ca6aceb 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -27,6 +27,7 @@ func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { Framework: stored.GPUFramework, DevicePath: path, MdevUUID: stored.GPUMdevUUID, + InstanceID: stored.Id, } if err := devices.DestroyVGPU(ctx, assignment); err != nil { return err diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 05d7482f5..a604d52e9 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -701,11 +701,7 @@ type CreateInstanceRequestNetworkEgressEnforcementMode string // CreatePushRequest defines model for CreatePushRequest. type CreatePushRequest struct { - // Credentials Registry credentials borrowed for this push only, docker-style: the - // caller's registry login (e.g. from the client's ~/.docker/config.json) - // rides along with the request instead of living on the server. Never - // persisted or logged; a push interrupted by a restart fails instead of - // retrying without them. When omitted, the server's own registry + // Credentials Registry credentials borrowed for this push only. When omitted, the server's own registry // credentials are used. Credentials *PushCredentials `json:"credentials,omitempty"` @@ -867,7 +863,7 @@ type GPUConfig struct { // GPUProfile Available vGPU profile type GPUProfile struct { - // Available Number of instances that can be created with this profile + // Available Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer. Available int `json:"available"` // FramebufferMb Frame buffer size in MB @@ -1381,7 +1377,7 @@ type PathInfo struct { // Push defines model for Push. type Push struct { - // Bytes Total compressed layer bytes pushed (only when status is pushed) + // Bytes Total compressed layer bytes pushed Bytes *int64 `json:"bytes,omitempty"` CompletedAt *time.Time `json:"completed_at"` CreatedAt time.Time `json:"created_at"` @@ -1389,7 +1385,7 @@ type Push struct { // Digest Cached manifest digest being pushed Digest string `json:"digest"` - // Error Error message (only when status is failed) + // Error Error message Error *string `json:"error"` // Id Push job identifier @@ -1398,10 +1394,10 @@ type Push struct { // Image Hypeman image name (normalized ref) Image string `json:"image"` - // Layers Number of layers pushed (only when status is pushed) + // Layers Number of layers pushed Layers *int `json:"layers,omitempty"` - // QueuePosition Position in the push queue (only when status is queued) + // QueuePosition Position in the push queue QueuePosition *int `json:"queue_position"` Status PushStatus `json:"status"` @@ -1409,17 +1405,13 @@ type Push struct { Target string `json:"target"` } -// PushCredentials Registry credentials borrowed for this push only, docker-style: the -// caller's registry login (e.g. from the client's ~/.docker/config.json) -// rides along with the request instead of living on the server. Never -// persisted or logged; a push interrupted by a restart fails instead of -// retrying without them. When omitted, the server's own registry +// PushCredentials Registry credentials borrowed for this push only. When omitted, the server's own registry // credentials are used. type PushCredentials struct { // Password Registry password or access token Password *string `json:"password,omitempty"` - // RegistryToken Bearer token sent as-is in the Authorization header + // RegistryToken Bearer token for an Authorization header RegistryToken *string `json:"registry_token,omitempty"` // Username Registry username @@ -18477,349 +18469,346 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+z9+3IbOZI3DN8Kgt9stDRDUtTRtjo6nk8tud3atmytZHuenaZfCqwCSbSqgGoARYl2", - "+P1zLmAuca7kDSSAOqLIkmxJ1tixG9MyqwqHRCKRmcj85cdOwOOEM8KU7Ox/7MhgRmIMfx4ohYPZOx6l", - "MTkjf6ZEKv1zInhChKIEXop5ytQowWqm/xUSGQiaKMpZZ79zitUMXc2IIGgOrSA542kUojFB8B0JO90O", - "ucZxEpHOfmcjZmojxAp3uh21SPRPUgnKpp1P3Y4gOOQsWphuJjiNVGd/giNJupVuT3TTCEukP+nBN1l7", - "Y84jglnnE7T4Z0oFCTv7vxen8T57mY//IIHSnR+kip8rzMLx4pRHNFjUJ/uSsvQaekM4VTzGigZImm9Q", - "Ah+hMZYkRJwhHCg6J4iyMU9ZiN4cnqKAM0YC3ZgcMj6WRMxJiCaCx0jNCJpxqeAdJXBwiRQeR6Q/ZJ1u", - "ZT0I00/C1VT6+4yoGRGewVKJbCtowgVSMyoRZfppQPrFBVMiJXXKdjs0jMhI0ZjwVNUJ9Su/QhFnU5iW", - "axfFqVRohucEfSCCoz9THNHJgrJpM5HGZMIFQb8uEhJjhpIIB0QiqhBlirvZGBrlPLYb+5iLThkXZBQS", - "qSjDuv1RwoXZEeXRv4Y/cIQK78LQ4H2kZlg5LmdcoUtCkvJE8RW+LJPx962t7rPBYPC+26GKxGZb4Wsa", - "p3Fnf293d3u324kpM//ezEZPmSJTIvTw7S9YCLwoTEfyVARkFNBQLJtJEFHCFDo8Pjq75QQ6m4M+/N/G", - "0063s/lsq7+59xT+vbnXKU6rRvjyyD8t33rnCqtU1mWQ2U0jyyijApPUZ/0qjcdEID5BQSoEYSpaINhS", - "JGzBdKVpD3xLEXA2odNUuC3o23Ilcs6wRJgZodGryIu8sVb7LtBCLORXbCRIjCnTNK4N4sw9QnqHIruJ", - "9JACzpTgUaSFglIkTpR0u6irxThDOEkiGoDoKW2qnXggO90OS6NIP6yMMF9tEtEphRdakYbKwiK5b5Hi", - "iDBFRLbD25CmJBabOs7J7V2NXC626nLGo3CUMkWjeq9v9c+WpsWeqEQzEoWITyZdRCcII92K/tnweJns", - "W4Otvd5gpzfYe7P5ZH/wbH+w+49OtzPhIsaqs98JsSI9vcpt1gaEtqQs8K8Oq7JIrA8kQQKzOtmBVVrA", - "MQl4TJBueunIdz5/5I3HzWG+qPpFZF/MT1bPUvcrR0Y77o6wVJkQgvWiajHCnjG9oTGRCseJlkN6DAVi", - "Nkkh12B1HRzllxJ487MIzMi1GlkKeefj4w9ynZBAn4jcSZNMwdDtWfbORNa98LggWOoBazVJH6a/d1Im", - "00Qf3SQcJRFWul2tUwEbjGIqpf40+yGk0siRbscx+YhxNRIpY+ZFRtQVF5fFN20rI5p0up0ZlqP5NEk7", - "3WXHVpmpoQsS4URCe3bFxYgIwUXHqMaL0YQLt0j6zM1JuKSpGoVkdsR6KNTpdkoEyMS5m4sbd7aq3sFB", - "L8BLwlgVxgyAydQHXmyrPtxsaMsFuzlFjBLtlhnZj2VZAoQUTxmXigaylZgH5UEvb8xDj+g8yppDNCRM", - "0QklwurVBAl9PsQEuUaQbgRRhlJZ2QeZ6j8ic22rjeY7IxUkdaJUDJvi4hV0k/xELJzK2fJnO2UFk5bn", - "7jWc5pjCnjwic2qOlrLuZpdmFAo6J8IjvjMFwIhC8x5a03tdixDGGVkvUYrNaUhxG3EQwphG1MM9p4fH", - "yDxGx0dobUauy51sPRk/7TQ3yXDs4YVf0xiznt4QeliufXi32PbLHa+JwuM4HU0FT5N6y8evT07eIniI", - "GGi4xRafbvk01SSgIxyGgkjpn797WBzbYDAY7OOt/cGgP/CNck5YyEUjSc1jP0k3ByFZ0mQrktr2ayR9", - "9e746PgAHXKRcAE228qNUyRPcV5Ftimvio//f05pFNa5fqx/Jg10IkJSqbS8+tm8hgQxphy6mnFJUICD", - "GUFjY7SAeQ7ttWH5rGN3evlGcERlwkHOo3cnuTYHpiG5JkGqSv3+iEKqbeHAKlWFybUYUcD1Gunz16dZ", - "AAWQfUeb2sqpTrdWAgJB8Iru9ButOqvv/9Sw1yiWTa27V7SYj2kUUUkCzkJZ7IMytbfTPJnCLjbHZq2r", - "5/pnFBMp8ZSgNXBLgblhJLzWtiaYRiRcb6dhN03mDz4unGulPQds0MPjYHNr2yvQYjwlo5BOrV+xyoP6", - "d60f63YUgrf9EwENo908oEtBJvX+foHzBDoRZEIE0Rz/md0lgs8Jw9ak+gv02/n/beQO1w3rbd0AYp7m", - "r3/qdv5MSUpGCZfUjLAmJuwTzUZAagRf+McMj5atdYGjpMJi+f6AN77ATsyVzZW0sa4frW/h6cpP3uh3", - "qgIdxFGm4BSkQKPcfq41LY/KwpmyDyouYD5FEWXGDNLy26wFKHuLhPwU8el654vRISN/ffPrcd9CeJkf", - "GlrTz7qZVRDxaZGaM4KFGpMSMRvOVdtQPrpG8p+Wtk/lAMWSjJZLkFPKGAnB5243tnlT69Ze2wd20SVV", - "o7k+gX17Dob1G1XIvtHYVMSDywmNyGiG5cw6KcOQGofraWkmHhWydJmBwUngGgTVBozq818Ptnb3kO3A", - "Q0Pr/dUv1GdS+Fo3b9ULhcUYR5GXN5rZ7eZndJ1D/ByQO3ybzp6MAx1jGknXsatpjfdUzsxfILv1qODs", - "02JAs1ek/64xbrdz3dOt9uZYANV184Vh/Y/rqfDbz3mnhV9Ps/4LP57ZoRR++sWNqvDbYT5ARxVjJnld", - "4DBpv1bJKVM9ymAJtAJuPU/m6MjdrNaTgfQyZhoenOyfpVgd1lQqtHb2y+H29vaz9fbKFZWXI0k/kNF0", - "vFRvtoO2irL+DOnP9FE5pVM8Xigi++gQM8YVGhMUzDCbkhDhidJf2bGWDPDdwapLmEYliYgmFUl92MHP", - "nl5fY/Vsj17JZx/isZj+se29/wTvopY3zR44bFdTv4oEZp5FvN1BHeNrbfkbLvFdfp6Y6yqUvWTXQFaH", - "0Ee5QT+h15rmCm2WKO0lrt/6y66wGGe9lNE/U1jtJMKLug2oCI57wBE+8hq5YTaQXL19pN4/5hs30ytM", - "ld452Z2pnXIX8SjU586ECqk67W/DbqQYEPGlVKTSLvNrTHWOqBKw8VDPxtm4VyI6IcEiiEhduDvxnYjU", - "ultDos8c+DNzILYV424oJUnsfjzNuij9fJT3V/r9uen8U9dIOmIfNoZNLBdlh0vl1uuYKq0BpNLc2xlv", - "MLLX/V9AauU3wziKiOjJNEkiChcoTox1jdN0ShgRmiuMK04hbfLQsBLXke+83qZv79355r7RhqjxrVlR", - "48BsXNAVc5hGXGtWC+Qmkvv++ui4TLsuwvAArkTKVM4urQr+ObRG+tN+Fw07SUB7g8Fg0MNbvcGgNxh2", - "yh62aKdnbh4SrBQReoD/z++49+Gg949B79n7/M9Rv/f+b3/xUbKt09CpF3aea07KdJEbbNGTWB3oKi/j", - "rQVccfjvG5f6WFsLN13pw+O6+8DMNeTBJRF9yjciOhZYLDbYlLLr/QgrIlV55svf9a6Iu7Wq3zJiMSWF", - "+w4sEZcbWASz3zfmWFDM1HszRjTsRJSl1xs4Dvd2hp31LoqxCrTSio5gRKjXc+1YAeSuMEp3Kn30iiuE", - "7dGuxe4+vFTvdypwHGOheXyOIxoCextJ1pM0JAizEFEGz/QbKZFIEJUKhnYGA/dkVBiUNrW5dFMZdtAV", - "VTOku0UwLcQFwiLe2zFXjQFJwI3JQ1y57y2Qor5X9Eb520b215r7s//+b+v/5y9flFmB0ZZwKZtqPr4h", - "n1Yc5iBn1iJ+RUSgDeWI6NnKrraVqZJdWIYQbEzEWbT4EQVGazY+IC4QYaGlNLxXZud40cMJ7VEzVKM3", - "vCRsqmad/b1tL3HX7B+99391PzUQVqQR8UihM56CHgaPixFsbgyZ/rVsRRx10wi8cTFlx+azzbqa9nkr", - "7CaybKWNB75xqfUpkV2vrhhIPZRRm26xTw2YEyH0VjRy7fDkCK1F9JJYgaZNRTRMB4PtAF6AP4n9JeBx", - "jFloflsv6yzWGeNRV37vkGDGwZ8URfwmkWOgnoJ2gqOl7pZlpPFS+zBrt+6c+ZVL1Ysxw9p4zAeAxoJf", - "Ej1QE09CiUSXZKFNhQWa6kZ7cyohmImwOZpjc2PVH7I3cL8Dr7hHEuJC6JygmAeXJspxxkFTNmKxi65m", - "NDKqoCA4yqVljCkbMi2gezLgidbSmH0NpoYuCJtfoBgnsM2xILDHtewnguKIfjDRqhChQkKqZfSQEdgY", - "KMF6zwcBFyEEk3FEcDArUOEHiS6MX+kCmr+gTLP1hdmYlbjMj53Xb9/8/Prtq6PR69Pnrw6OR789/1/9", - "s/mos//7x46JSs4s2J8JFkSgv3yE+X4yXkhwjnQOUjXjgn4wN30QiyUVKP44oX2eEIZpP+Bxp9v5a/Gf", - "7z+9d34zEwIx19vAM7BPXkXRKDsekXTkbpIlsreTLi5Gk0yLqBenbze0+pRgKdVM8HQ6K28Mq7vdaEuA", - "kUH5aJxI7yXfJTreeI20ZokiqjdopkluDgYnP2/IYUf/Y9f9Y72PjsyuheFrGcSFVXDlTLNPFuB8ePoW", - "4Sjigb3qmjTFMrqufAKeMCUWiba8Vwqn/NW6jOr18qc3EEUbY8o2pF6GXnAzugPf3Nrj+5zNqeAsJkwh", - "UJfGEZHlvfLq9dHz0fNX7zr7+iAI08DeaJ++PnvT2e9sDwaDjo9BNQetkIEvTt+aiDnYNgRHajYKZiS4", - "XPXhr/DuIbwKO04lUToFG7e+egcZaVBMYi7MJYn9Bq3NykqK2fII1nXY2X7xs+HLzRfAkm49bWhS1opp", - "uBJN9uJnH6PNFgkRcyp9N6m/Zs/MjTsLHfPUg+PRnyRO9TNpIgtFMKOKBCoVpMcwhPzDKY1FiMYci/BH", - "+KIX00DweWy+/J/nJ29/kAjMdhwhyCuwGix8Y1QyTiQYjTa+B+UBK7JbIUFXC53YRPlihgidzhSaU71z", - "e3FMufuuvLutV8Ft29wjan0xQcTTsFegXLczoYIEEEut/6VnZv/jJtjpduYfyjFenmb817atTK4VthSO", - "EsrIEmPqK9GZr7i4jDgOwV3yJVVmG53oCYs3D8rLndl4jsFr2SZjzMIrGqrZKORXTA/Zc9DYJyh7OTtt", - "rvVMcPTvf/7r3UnuGdh8MU7s0bO5tfuZR0/lsNFNe+/+somkiX8abxP/JN6d/Puf/3IzedhJGM3sVlqu", - "Xf/npoVqwLzNIzK3Aw1hlpk6k8cLcesCgs+R472VMZW+k4vPiYjwonCc2DF1Ngcg0yujEhQypJD9ruBL", - "XX646Nac1vOi6pbaGviPj+8umK/cBSMIiN5RkuXuLdsgZ+bt3Dr2MJ2H537WZ4HVZtpwWsZom1sn9s+t", - "Osv5OU5e0mQEZuIIT7NYlGXpfueXNLG2J3xhtmkUGUEfpmCtjjlX/SEz4fx6bwLnkGsSAC9IhRU6OD2W", - "6IpGEagfcGjUFSFtyRbyQOB1qfT/ipR10ThV2jzliiDrKLBxhXos8PKYoJRhFzxcMRbtBOux2ECWSyIY", - "iUbGGJQtKWM+QvajRuLAVCdY2vQjodKkTK+j307O0drRguGYBug30+oJD9OIoHOjqq2XqdcdskRATLfu", - "RLMjtf3yCeKp6vFJTwlC3BBjaCzz+tvI1vmL07c2Nlqu94fsjGjCEhbaJE6nUdgEv5CzH7REJmG52WL/", - "FaI3xb9LhhM5420317l9Pd9d7b1n3c48SNLykm51GxP8tIqb4kifpSX7yRsNbRKUPXayyX8u2uv2XMsT", - "IlU5YrSti9G0DNnK3lRIj6fQaMKtPYUF31XNZ+gcKx/bDXZF+8fMDWSppzT3rXxGX+emkVqmg/m562Z2", - "CyodZzSp+Fe/DHkOZMEX1Sqx2Jzs5viTaO0CJ7Rv+bgf8Piiiy7+WvpB731nEGv98QoZaoA8YfqnYvtV", - "L9xK/9iNUnmLi4Pl7dfjQDamhaD5JlICM2kSemY4IX30KwhxpEisVRPIo5Moy4NBjF/9iLhRWt2nQ6aH", - "JtGalpWWHJmXVNIpo2y6rs04fTDhMDSu1EmqLXskyJzKnJpl1nHuyppmaEZno+oh+52yIEpDgi6cS/Oi", - "rBTVHZ51R4b1gNYsWEMSsFzBtlcbcap093rCmaLJU2WybOzUyxlQFbfqqkBPO5Ys5ugW63+eiYsqyMHc", - "Y8LqydlrZ/CDFxzyTX5vq6j4ffKXZAFL7vzvuOaBL7re/Q5yQSSP5sQeu0Xn/RhgHLhRnHK/vfHAW6e7", - "3v5VAAOfO3rVUmh6tSZ/2RT0wDdI1XOTzTnGWncugdZJIT050x94oSQB4oNpuY9AHbvoGluYgOWBmGaW", - "CIVUkEDVmqdsOmQQ235hf+nb1i70Jtc6yhcBxYAcc1Dai0uLCivr1D5oRk+Nx1QpEnbLusElIYlcPSmt", - "XtubGs91kiBXgjpB5jIwW6pnhE24CEhsjYTPcww8LzS2JFSmbRP1UHND38KYXe49thFIsPZmPeBeoZSS", - "X0WmCStWmwltLnd5gaPoAq3Zl9aRIH9A2rJdK8ZZzuxvDk8dC2SBOO9OupojtRS4mCmVjPT/yJHexRfV", - "xuy3bofnqCFPB2Bf7exs21W1PlYz4EqzZf+pN1y7eWlOUzlrvEauXKEu4xXdzmHh9WYvrYNnMZ5adwIl", - "qZyhNYWneuvYuHx9Tqx/rgeXMkmCVJDVe/0A1CMbGw5jijBlvV/fvDlFgkypbrMYWV/M+QW3TkMSkTVs", - "8zAgN13Fy268re2d3b0nT5/1w0vRJ4Hop7JHsFS9zT6O8QfO8JXUatdGvMBJsj/fXCnnzRJk42sW+c4K", - "a2YFHmuR7NIg2lh0h/kn+Q3SJWVh2wZ+0++uDgd0Budd++ETQXppMhUY0lK/pBf+1vEiQM3mVV0B1bU8", - "1DOVisfFHPW15RGeZWLNedQLscI3CfA0w63HcsYL05Qxyb3ur6bA2fNqtGwpsNwL2PO5wTtuLL5laUp6", - "N44EEo4U9+RyO93h+AgwMuy7rbJ7IUdf8dF8QvlySAQbmFmC2DFaiXVv6CZ6SUCtVwlU3WBmEiANEcB2", - "eHdSjFnoD1kPtLB9dJR1kDWbNYnBxMChufZd46IwCAN+gMaLdYTRu5M+epON9geJ7C2qRQGYYYnGhDCU", - "wg0TKEU9o5IVB5BK0J1U9XPrQjOAAesQmsHts34GKwbOugwkDWJ4x7QyHwPkAwtlzxJ94OXO0FbOy2Xp", - "M2fmPFqWQlOCTNntDTZ7m7tvNgf7A/3//2ifY/PlMRF8bR2UZYuNii5Kn8O3x0db1me+fuusmS+OmuAX", - "ZUd5ODdaSyURPScm4crDE8RdiJVuCNK+dez1HYVS5/mhy941lHij37wLOAhfworNKL05YENVYK7MCi5M", - "ru7QWSTgfsh3SUGRt2H3AfXkx3YhFOtnQfAlwNzUz22t1cmROc/8cVypNMGF5No6uQTnaiKN0l12fm/u", - "PNl5ur2383Qw8AAO1BmeB3QEiRWtBvD68BhFeJHl3q3BvXaIxhEflxl9d3vv6ZPBs82ttuMwN7nt6JDZ", - "3+4rtGYp8jcHReqelAa1tfVkb3t7e7C3t7XTalT22qDVoNwVQ0klebL9ZGfz6dZOKyr47LrnDgCiqsD7", - "0IgODICf/ldPJiSgExogSJxC+gO0FsMRRrJLy/KeHOPQYTb5zw6FaSSXhnuZzuybxt8ap5GiSUTMM1iQ", - "VlcSMPMjaMkLgslYhpF1s5YsbMbKQCA3l+wVUGdCMk6nU5uTlpHuxICAFZQnSqJwH2Um21I5B6uZD+x9", - "Ex/YObTkhpfadOpFZE6iIhOYo8ugUQmCMj4xi1aalYsBoCxJvSzRSMpfUgG6qGkU4TFPlblttqBmeSfH", - "xnMAt68pC9u5O37h4nJl3Lw+iTPstpYOg4n12MEpjrN8aZtBX1D6slthc3dun0t0Zr4wjsL85yQtA9d2", - "oSfrUGRIEKk4SFLrN7bNtNUu/XoL+MxdGInpL5ed9xTi1puYqKAva2GLKQHMQrVSY9GcYuJ0zuH11mk4", - "+sOVjpQWdGfk6j6IDolkPc22PclwcjcUXxZJm/ka8pfgFBY0JH0EuwuC3xx8TWWnnSueJCTM/D/9IbN5", - "LNlP0lyk6Q8NHdSMUIG4oFNa7rjsZ73HWNabcKZjrltzZ/HDptg0mV0D+mSAQUSYmNBQA9RVTMi2a9Lp", - "ds4zcEcrmMqkOcsAMmsUySPOa0N8cfr2phGpieAT6kMYhggZ+9Qaai5W8+XO4Ly3+T8mmlyzH2hslJmo", - "mpiHFSxG+367g+jF6dvTpjFl6IioOLranLI4qGVw1o4i9qrR3lVbg8btBn3OZJ3kqvgzn2o7ETgm43Qy", - "IWIUe3xtv+jnyLyQpamf/FxWb7Ua3daIPi0tDljRExxQNl1vTX2Pf64yjW6Bmu/9y3Vm0faagAr0UmWI", - "fAajoI9eZXiU6MXpW4ny2DWP466CBdCUNnQ6W0ga4Mi0aMCkKCv624A5WyvMp/mH1jPpUZv9MKZuI6C1", - "+TRJYRuen/WOX7/biEMy75bGBPFmMx4RPe71grSYu+zzPMepJCTmTY4Pwxiy7QYq0Crbwa2JVNivHuoo", - "rnA0khH3hfC80Q8RPERr734xyad6BF2UlJZS/16gQom/97w7BoBfGro9hw6rHtTSBveakuX6F8bbUphe", - "qVPfVjEZP3WVp44hzC/LC80vV+PWmkaa+z10SUkVH7fVK5HJXUKQu+SiEpD51DixradEkgQLrEi0MIpG", - "dvSVcUhI/YKZXJPgBllRz/Xrnwz2VCrISM0EkTMelcMStrt1/HIJobFzYiEbzZwKfnjFUYzFJRyMTq9G", - "KTMUKEdWb6/CA5kpldxgUr++eXNqjG1FxBxH1dh8WbtwPyIRXqAxUVeEMDcVLBFGL3iGjVlNEZQNsHlC", - "jRIiKC/TsLPt6ffchOuiqcABQeYrV/3ALomEU7MtKW0vHkzqICBSNqzv5rL1tZ9O0qjdGvuGtbmyWEhw", - "kwV+c3jq4N8yeH1H5q06lU+J6Jkt53D2ly/tllwOROi6YpyRemeCjwkgE9qEiGK+ogs3AuRF/XkpR7Ag", - "HGQxq832A7vAkKpr9vn7Vspedbv77tVjzEJfmQITB28SsadprBdED1mkcJVEQxOLZHLDjVpeDOcXBIeU", - "ESkrWaxBKqJOt9Ob2Fntb2xEPMDRjEu1v7O9+XRjeVTn0nBeG700Cukyc8/FOJkoGJfsacoFwKTLLLGB", - "k6SFQ8zQccX5AOKpHj4IpQn02VbQ8Fw8+WBQS8i9xoFyYKrgISvdgOLitk00S5bmAw1m5Xx2nz0r7s+B", - "90o6r6bl2H+jxvsQpgIhhppHjPehQkfD5B+8GhUXPsRMLpRNTBsTF4OYnYcuws/esZQ6ezp4Wpxlq6JF", - "IGwq29zuO89Uzdvl1FtSILfdv7YBiJYq6xxuSy8H/dZ0WcFTWiLWWGo5QXlC2I3oubuzvXUzeradyLGL", - "0qrIJR8AyOHJkdGJAs4UpowIFBOFbYW2gpAB15KWMgDJhkkM2SuTH5eLloZwhiKix23xJL/UZXgDtOyZ", - "ieoNUYwZnWiBbN8s9ixneGt3b98AYIdksrO71+/3b4pz8DwHNmi1FBsmZq4AedCXs89bhzuAM2gzl4+d", - "04M3v2pBlkphDq0NOaZsv/Dv7J/5A/jD/HNMmRcGoRVmOp3UsNLL4WHa4De/7xdKhDm9p1VdHb9vGIKF", - "AXvFCypWCpq8O/SwjMfNAG6TwrokpVOrK6+hmmMDbW4Nf54XBlEF2PNi8lcLCHT6Yfl1t3N3wTu2Tyj1", - "laPD1y+6b4XvL5dCINcQMhPCMtDjKDJ/BZzN9Xb1ISCXjkj3rAWAJhwjFikz67L4Y9Z74cfD4kAKvzsA", - "zsJPFgr5/Q0jVJYqpH+v66GrpZBTR1dsZr/vMTsV2sLOW9Qxb2j2A5+Ft4niKvf+evrff/5fefrkj80/", - "X75797/zF/999Ir+77vo9PVnQXEsB5x7UNS4LwYUZ0qpFNHi2rLSCVaBx0enzb8GCtsnxuOgghkAZaMx", - "2R+yHnpJFRE42kfDTiUXcdhBawQsJfhKq7u6KZtSva4/PjUXjPrjj04N/lRtI7S508IuSAaJIdNxyGNM", - "2fqQDZltC7mJSLAL9F8hCnCiwJ1DGdL27wKNBdSGtTc+eedd9BEnyaf1IbPFbZTQM0iwUBmEqesBmMKO", - "ykSl2tdJho5gLl2GLDutM8Q1c+3Xz5R/iP6opnb5ibLcfrOW09OBD5sO0kv0QkYAjI6yCzQqgdGzvBf0", - "dLBet+dW2BgZDy1hP3svbvIPD1Kfu7gpZ/GIhDQAueLS9mY2sTPLmDScZp14ieDXC1ibM2KLUeJUzbQs", - "MhFPXRRwfklJF5a0iyYCxxCIAV+a6/UZT3rjRW/GkwwLAAsTfILNBXXZyP6/PTvR3jsi6MT25C1jpVnE", - "o3TClrEzMxmAmXehNrE3euEkYVr1mRP7OpUosdD3oY2sVqlgJuZM897B6bEBnci4j/h08h9tIWDpyv/O", - "AF5N6WZ8CGmdQZbBgsdBSCbVf5cjDLZ298CCdf/e3mqdQGpIt4zL0shj08ZO9LWQ2EZMwgCMejByjvAV", - "MUH6BDSSGIGfQnH47zlyDeU7LhMk5nbKpLRJE+ugIllIZlv3Zh3ZbWCRLUbY7qY251FtF5aSmFq0YIIF", - "4LOoBabGc5M8+eblOVJExC6dfS3QqwO7xNZdpVKmWn5SjA4OT56v91uUaoelWprkVB60B/rABg80xUTk", - "Phocky46PoLkVXus5LYYZBv8wgWKzKmYH0b76K0FZy81hUyws2GMaJFHIBi1ZdhZdy0m1eNtH51lJiDO", - "hlIqD18OasgPE2jWxqOZVIha691a4WXhzD97HkPiA1aZLNZqYvP51d7n6BCS9EFV8ZDd+EAqBoU0+r8K", - "a/+lQXS/vI6+fTMd3RXTTGZY+rh7VrzVhJeWFNymlasa0RzCU+p3ZRHpwjGkiD7nSp/7yv/u9jY332zu", - "3Nx9d1P80zJiUwHNLYNAbY9dehcYoHVX1jVVo8ZYb6Qf28hu5xd5d4JmWLIfFDyseEc2t5+0Kjqoe20b", - "JV2Mj+YTM6RMSjn4pyy61wBhXdIoMgqMpFOGI/QMrZ0fv/jt+OXLddRDr1+fVJdi2Re+9WkBhepExYvT", - "t3AzjuXIBRo259bhPD+VXFOpZB0dq1W87udAr5pP2xWPcZM0beQ1ZJbjt/5aAhj1wp2t3wHwqgu2rpHz", - "ATBJHzKp7evDQ12KYPq5MKTWEL8jFNJGme9D8Kxk8O82if/b44neyXBKACg+sVVUAVzG8a0BPLsd6sm2", - "PJBaMpMQHZ/mpWtyb71rvjKnZ1v9zb2n/c3BoL85aFerLFjS98nBYfvOB1tG49jH4/0g3AeL9raXOpax", - "jY6Ooyu8kGjorKhhx1iBBfOvsG2tpdUqAKWOk3o7WNSqntMAfAr6novskbbocZPO0yKdrgryFaeRydgu", - "FkouK5EzLJFMDKKlKUGQqbhDBgPsWmQtODloTBAOApHmvjlXsc4oxGli+X7IBJEJZ1pJ1sYO+o0sJIop", - "3BJm3UNsoERZ1kY4ZGvCZfhkqTwJTiUJ9Q8QL991cdl6aFRBqQX9wZDJWaq0EFvvo0POZBoTYd2WaEzh", - "SmUdydTYfDBeoMZCC0xJQyKGTL/mwcz8mOnv+3uDwWAAdxLG1NnW/x74uOlObwf7FhPW5KgCOh2z6LAA", - "eydShlIWEoHOuCRKYUQMO1RjyG54s/iZYLDu83bqlf0816v8G3MVdmw7UNjPReJcVvn5vFzzubX1uvuP", - "zyoPTdrqrzY7xn41uskVP0EBT6NQm0RjfdoZjxUJrZ9OEpWX04YD8i27ZHqPlqZuY3MVR3+mRCzQu5OT", - "UlyAIBNbLbjFxEFKNKwDT260DFsrnAgrR3NLfNb7wGStaioFDfGLI7AWr+Bc0q7h0BZXcbkh6c2soMws", - "jeaTJXOquLdDMh+lqc8Q0Y8cVMfbt8dHJebAeG/z6eDps97T8eZebyccbPbw5vZeb2sXDybbwZPthnr9", - "7TOrbp8s5bVGfQXYXcj0yIVu+yJqmwLnK+e+DQa+oizkV6WjxRudWezdRn6u6r4e1916CN5sEKgFDC01", - "SIkTOEBJoNs20dCV8twNXra9N4PNFV62lfICBtcgf9+IlJk7PZNvnvmv48KAi4tVHuftxCkMyGVdrKJW", - "sfP2RBvs7z7b3/1cornMgVVjrLLTPS5uUziSw+OtpCa49LiCj8bhV3asvmFcyjaTodPtZMkW8DcctJVA", - "3uxxqwyipg3b9YuRZfK7IZH2uGQLQJiEwWML97UWkOVgjlOFsnRtrV4cRjwNUcHPZeCp4BLouGAX6Gbg", - "Tsa6vwzcpMkE0PYD4BsDuj5lWhDD5ZduxGbd7qMX8C48wrExmewgTI2H4r0PDhcmWEPvL9e1MWCWD/nc", - "2i7wjTZkkP4XTFuTwbpDlzdhNJ999IrDN5klxXjVr2peBxOm/nrVB7tm02EdigJ0ZtW4ffRLprplyp9V", - "9tYksX+OrMDKwUvWSynkdsU7mlvylSukQ3c7hqKdbscRCtKm6wnUb3Our+2/Iiv6opgIjkyJ8ixBNVU0", - "spDNMBMqFQ2kzVzQi9ukX9jyMyQcGcOkKSDSZD1a4yX7yKkv707QGqDy/Q1Zp7H+13oWPFk667ae7Tzb", - "e7L1bK8V9k4+wNVq5yHk5NYHt1IHDZJ0ZP0OTVM/PH1r/AqBsdizwIt3J0Wog0RwLXr0zF2Dxc6f9Z8V", - "IYdCno6jwq2WxSczCKewYF5UrUwWNQTh/UmjOZ1M2J8fgsutPwSNN6/35NZ4swFK1XTkd2kdF2+2a/5f", - "Mu6ZUjJ+VBhgKCEbgZPOiIQZoHOiEPBPD+EATIcsldaynINXshT3MtbO9vb20ye7W634yo6usHFG4ODy", - "HMp2BIUtBm+itbPzc7RRYDjTpsMXALRrZs1K/z5DtpzqoKyQ9jcH2z4uaTi4c66xbc/jRpK/s6aZnZQl", - "OmQEZ2ZbbZd7qb29PXiys/t0t902tq7XkbheLmFcvowhjwVlL678GmiTbw5OEWSjTnBQ9pu48KQbjUrd", - "aFRQUMAAgd9gYE+f7O3ubG9ttkMA80U8WGy70oYtyy7PpvMwhWc1PKSoi95u02nhU6cMg52RIMI0Pghc", - "fH/l9DGA3yNhXssXoc3BYL39tYOrxbetHEeZO8hkhxjVgAuUsqzMRH/1decXubVsltrmeFgt1X05IUxT", - "z0LVmDJTtyBlIsic8lR+gYa4Mgmbk4hzcaNvmwyWMyLTSJmrRSrRu5MfQKZoXkNSkaRsQ1luXALoc8vJ", - "3Wg/l1jEz+RNxGq1Gm2WftmEuw27trsMzaEkDRphtEItuVK2OvLwEEdBCoVVcLaeelaAgAP56EkSLUxg", - "eRRxzlAwwwwuIIQt48SmCKMZj8K+NwxTPxlNvBEL/ApF3AAAXxKS2JojZhD6M63C0DlBa4V8eGRYqVIb", - "cjc2QsZWlShz427sL3KHpS9TKsvD1vTEihewac0nJZdjxKcSjEIFIfP9KiR6goWJhMfM1NCZx8aW9ET7", - "eoZYEea+E9WcpHxiDVyrckCWs6EkDgSXEpGITqFey7uTSvLskoSrLIV2dTRhebAtWNdcFnqOMjjiZOtS", - "W77z0ZNM8jknJPAwJKwtidNzzskYsxSqkBQYmVwnVBj2aBeLN+NSjTJQpBsOVqoRlBZIBcmR07KU78wf", - "5N7xnotOtN2GXDbo9VZf17jK31TTAJtlqpeifmp1Mx70sXEdFmopElUObVXFMboJcFmORU8ltEoLmFlo", - "DRIuCmKpgKe+3iYAxW+y6n5q1qqtoPhyZ3DeFlNsOYTYKVazYzbhHuCJG9xKOk+0DZVMiIgpFNdAIWGU", - "hM6WzK4nrasLspUjSVCYEks5o58KbAmOzfYG8AjmfGSUTSuyvtphG/ewGcPyygPQr32xTSiR9GdzvhEp", - "0MrEAkqE87zOVgGWVI7811n1hgWZphEWqIqbt2TIchFHlF22aV0u4jGPaID0B9U75wmPIn410o/kTzCX", - "9Vaz0x+MmsrUnJvB2aQ0syCVfvMp/KRnuV5JiQVPzIb5fgPgStpEZnnDlH+hEbHQcm8ZvS4wehmae2dr", - "0JSq3dBoKUm7Dkt4U8ltWda741PpSWxbquW4AjsktHjpRu1JUjnLIg8qZqZ5tt5p5b9waJ3ufvB21z3l", - "lIbPA804NMK8ApmBxgQyUmBu3tDFNjLTSy8jQNfbzNNboCCVM/QHH5ddqW2DcT1lr9ZYjuwgyMQblg68", - "sNSVbd5oySh1vrgJrgJIa00FC63g6yuHWLgJlsGq0mJ5vFSTDDurVdmaEUvsjAKm4laLChIuniPL17e9", - "tk/cr9ZDa6rgsijU9ZNozIUA0OMsRwDIrencRRbIRapFRPZNQGKAo4gI8FTY1iI+pcwG2GWX6SaP8weJ", - "/t+NvmllwwRU9/+QnK0PmaAhkQiDrepwkrPMLq1nERwCr9E5lAU1rGD8+X30isyJGLI86ZQLPY4pCX9E", - "2MwA/JwiTawfFCMbNQd7Uha6GDJBlAB73BnoakbiPvp7wfrtFrr/QSJ+xTICDFmRnto2TaWrjVhBaMJS", - "XnERLlka9wrUbIF7YqT4JWFFiZs14zVvTUMj81U9ys8UcIWnCC4AsOzRLBikVF4VVcunLu86lUT4dd5s", - "dtkrrSJxCruwYGuZ3Q7oIlA2z/4FP+WIIi3wQ/Lm/8c1mf90mjVe/q3yWgEjxEH2HhgvtNejHJiMn0rs", - "UvlicGXkDaSULUOGqV+NoDUXbe1qj5QVm0INkFYne7ustmpegRvNhiRBufedp7tP9loWYfmsu0eDhPWl", - "bxrn8ZIbxoaVOmlzjfV09+mzZ9s7u8+2bnRh5FJQGtanKQ2luD5ojVwrvVmjf//zX+9OKpdYuxCuPbjR", - "oEwSin9IDYko5QG9O/n3P//lRnXrAfkETR1tuyEMoTHoKCqupIt7KN9ItrvzW+KuOCj5PHAmZtAamUwI", - "eHVHhm69fDAVSI12ejtOcEDVwiPI8ZUJjM9eqaBGt7ndKg/Wp6Sbti3CqJZcMh3nCZxrrnP0V3PVXeGF", - "p61rOcl03HSt/rraq7lUzy9hiiEbLSImZFZGvO6vz+ZzhWUp9lv/HYDy4ZLR6ok55o3lCLbVrAkIyrEl", - "ywqRjT7k84qqaj8qLn9lOQvXsCUvVZXi75fsw+YteCMntudE9viwg9VZuBX5YA/A2301GherrC0tY1cq", - "yZafujfvt0Wicb3mQHaC3by/Qm7lTT6s4usCP9oxWJLnbXdLLNHATYW0GY8/kEekl8UdOutApuaCU+95", - "C9nuSfYMLvlkUsaN3W3GGQcIHcgLc71gpUicqC4i184FUwWpNng5w86uHHa0NTDsbMbDTuUWzptpGePr", - "ke2gjJMyWAb8ndUXrQ5SuhmMIx5cmgJiUJe6jwYoJphJlDLY/JVLws3B8suubicprE0Gs01MxFZNbMGY", - "xmSG5xSqO9gromkprpRcUyUh/hXa2UchN8hJpeqpdob6NZMHuZ9PGg4dzBa2Yd2gfo8zF6CbvwvOiAnU", - "bGUfiOBdm/ivJfbr1yddE48BkZRmYKVwTTdRMwItILMuKqUK8t/94dDjiIxg3FXo+7hOx2L6Ohjfgkii", - "pMXCztmhwgQo4ClTVUz8uJ0hV85Aqx9JKYPYRevRAYwz27stvR+SAHakrO/FMqPfgrkraRCW0r48iG0f", - "C8OmgKsx/9X3mb3frg7AeAoKhY9NO8UwdXPpOZKK20pZ2a4ekeuAkLAKnul/pW3ov/3SG/r/Elu8nawm", - "sX0bwrfrs+vfXS4YjLWJ2sUUBcZZD5A+3JJaVA4Ds2dxX8qMVoLxLqBfjHxQpb4X2iRnk+vltH5FrhVg", - "jYdpZADk/KxrRZU9jFZR/NZJkE0bmovVJfzvoCKbCZ+/VU02G3n/EGXZ7Ft3UoqttjrnRLl3zy0bNRfk", - "LxVNKd3QufwF90o5ZMiwUhfZAx5txusVFtyZ+b0iFuC2ZTonwzEZJYJM6PUS5jEvGMO4jICSb6QsIcNg", - "da7F+BrtPEHBDAtZGTuj05mKFuV4oh0PLtFn1SsURBHmHIVtVj5fTfdhPXjPLmexdZ9yfF5AEaqVB7Eq", - "6WgZBvVhfnlow6MSvAAvTuOd55PtncFge2twKxBqN6wbkOsw/8RW9yu305QhWPjOxi2Ugm6LLWT52PWS", - "sVeCQlp3RiapBMHxPuQRJTggKCITAJzLSnWvvgutdr188Fahstg1Gf+7hbLr5kIKyuVnsq4sfrebRsdd", - "h5bhiorPV1zhNoiZoAZP50kh3O4N9t5sbu/v7u1vbt4FcHRGpKbklScfNq+eRFt4shM9XTz5c3P2ZLoV", - "b3vtsEtqquy04dXf9LuNQUP5IVmGPSqJNLRm55AQUa0FXK2hLUlEGenJLOFrddblEllgwglW7v+b+fnN", - "DJbqDuflSRZVCKxy4pQ462Gguuxklt5dVGdzfLR8FrdKqKoOxM9v1aEAe7UbDFR72Ox8JohDyloeQ28L", - "L7Y+iJYm+a06inyX97DTvavcQHEfe5fkZGnDLTvA64ecx3c65YKqWbz8tMheyyC5IQz8g1RhGRqqj46n", - "DAqBF3/Oov6KRpT+uNPtRB92ynvG/t4eJMyi+WYMaJe6qBW0iIqDOvPLqQCv5IaHMIH52lbXY/5ps7f5", - "DGLTow87Pw16z6pRAkCtIvk23dulXwdtaFgsp+fKMG0+u1EAuaPnMg76jfqKweXnssX5tTyel112R4fL", - "Hy4tcP64tsYV0J9GBfRzNT17uI2KSlNIIrzw4bwXHLWyYj0WmQyNyZQy2cZvuz3IHLe78bDTRwcWbBts", - "2bzIfql5KK9e4BMaxySkWsc0pn9zQsZWS19c1Za4WZ0P95VHW+v71bVnqxEfVuWPrTom+5+RXvxZ1m87", - "i3cZGAn41ZyJCnBj8GIX0QnCrFLsk7I5jmhocQEgzxPi5PYdplvOslYGyFwPdH6SLppyhXJEgJb+tpQ1", - "+wWz8ZNr8LcugQAxDLH1RfBdMqwxukx8HR+hRPAwDfJ02AgGnQOYiLSC5rZEyV8dYXyX/g3IM59wgVb7", - "N5ocGu38k03rXfFNaoZtXurNweqlvhOnSLeTJuFqGWZeaifBboSCviKj0uOiKZO9ogkWJvO+hUQ/K1Kw", - "bvMaX3KgVaI0cRcsmqfqnOS5boErBl+w8RGJiD6m6o0gHoV50geVuRRdLVI3957Omq444UaqPpDfCEm0", - "rQJwTtBfjNnCOzBXwjM7S9YGrjyvNBdePVP6x1KrPLgnKzWxxqUqunCbyhEYKV/xeRv4l8uC+zuDAy+q", - "ZnVAFyfwS0raWTOcvv3Shb01+o/vwi33kEraa3v1UEF4dfjDWZC0698ktAJCNIsq5/WO73rexxZvrGXc", - "hFpbTWotep0Pev8wXmY06u9v/PS3/3/v/V+93uaK3SyJ6IVkAoFGl2TRM4V8tI3eL2O2QhUBrUxPLasQ", - "HIMPCfDQ7WYsjnd3kAmNxSsc16YAEVqFKjybKyf0t780xzcVyPgW5ORKlv3sIht3UYxUcXccrcVETLVd", - "SF2OGqRmDxmk2l2ShUSF2l5WpXGM+oPMPilE+aMLowb2CZtfoDGFYolyyLRVi4OAJNqasOViqKn4zUH6", - "CIKjYju2xpgLk7cXkiaegKB3JzXA39dv3/z8+u2ro9Hr0+evDo5Hvz3/XwjxuOqZHsKe5r2d3T1b57tI", - "yU3PEn9GSYLPwtz1sZtB7vTwF+SMQvF0j8JMJSA+uHCDwstojcSJWrjCoS71dP1mSKIHWYPeYLcvXB9m", - "8OxL1HB8u7Ro45xHPa1RN+D4ex2YhhbeYG1oygTBd5r83NOxR2203sQpnWKPa9vnwv4itRbdgFbm/9TW", - "v7HolD90/qha5MBIA0OqCih/xS6VqtccWR9rRWqUF5Qvx2ukzGZ/0kI4VznVM2Zqw9ZE9SFOhBygtpfl", - "++a7zOEL9uCj1WmsS1X5wswKI2lemxOnsVZ06iUEOtWkuZoRQQoLAR/k4PE3JJlNBWmBY2JqyiVE5GGS", - "Lo9EK0JwwynRWuZscCTI8nXrHtjlxQFO8HXWA3jvsaxdecE88uo9my9+BiD3M1exkE5cEzCMij3hhy0v", - "c9Eymjiuqi9Gkavq8zbvezeelVVLpF/T3qowZ95HiTV9/Ph3TNUvXIAF0owacufo52DdhEQAiloV27wV", - "MDiNSTjiqVq+/21BdAsZklW1zKuiOmsLAxMHpTzgJlngcC3yMdQprclBglRQtdDmuVWGx5CE50qRAiGh", - "I/g57xjKP376BH7KiSdH4QVhRNAAimvq/RhjBko6endSqLFmyu3VEE9BvXx9eGwtXAeaCxYLVcB6LtTv", - "4PS40+3MiTBWXmfQ3+4PYDMnhOGEdvY72/3N/qADivwMprgBBdFtHrHNns1spePQakI/u5f0lwLHRMEX", - "v3ty6SHUzb4OWi+eFuyWBFNhDZckghx/wzBUfw1g+O5A3TencteQvbWbDlJgIeGCJK/t4r4HpRL2Dkxz", - "azCw0ODKHr+QLmJi1Df+sAGLeb+ttDpLIg82fM2ycLplRvpP3c7OYPNGY1o2FNi7vo7fMmzzRgkYhLs3", - "JMStOj1mJhPMJuXaCJzijgNGKu6139/rNZNpHGOxcAQrUivhskkxJhJh965BLFYSBVpUQAWbPnrNiHmO", - "sELYBMuKlEFlXPeh5tDyLjBtu0XOYH5+5uHii5Gw1Icziz+VxZneLp9q/PzleCdj4/pC2kcOo9pw7T0w", - "0M84K+v8YDtlZ/Ds7js95GwS0UChXsbANgSWSogyiQBx26H3cIH+TLnCKIsgf0Rb2uqs44zduvlRtPGR", - "hp/M9o6Iz/N6SkSMmYnHN++s2PS17Wy84Pl2XnqqOcaH4hhwUjkYG3NQgSJX3qLFY6uqDNaPox1P/r/t", - "00wvfEDG37mHHW4nm1X2fMgtZ0AWUkke03aytzrjXAnx6nIviPpaeH5wn0eWheH/BnfRY2HgFyTT8PLV", - "qh0KG4lImTGAvRrgWZ4jZ7/7oaz8vcmfFAIzwJWum4aCEMpc5eFw0UeOpsboVwsADBIE5hnWj5VTPbyv", - "ZYdt3ccOgxlnlxPfj6nvx9SyXW64xU0BNmZhl7fwQdzIA/Ht+R9u7H347nto73to5Xlg5Mp6F/7g4z6y", - "QZBQVl/OeBqFaEyQgdhx4Q4Ki/70A8IimNE5GTJ7WxCnkaIJAKFxEaMQK2yubRsdE0vdEllzG7q5ngt9", - "ywlchU6QZARYd6MmkMY86I0yRkKkP7Gwejk4Xq3Ytdn7Xgd71mB+NKKrGZdwtQH4cUwVTnPIqJXGOoZm", - "+0P2xkKlagJC/K6TNZJEAPi6xP/DGcJDZj/40YkQF3skcZxLLiwAY44ahEazLPVsKj3SkQy4D97lDWGY", - "qZ5MSEAnNLDTuiQLG0LobbBV5SI9YDfOdydZjgDa8oM8GtA/P7ztUfYMWU4q398wiLsNojTML7kcag0W", - "YxxF3tIW04iPcTQy9LkknjvBF/CGJUqxKL67TWI8JKbAebJQM87M3+k4ZSo1f48Fv5JEDDvr/SGD2H9L", - "a4cRaHjgCkqhxQnX+0zw2PS5YYa48fGSLD71h+wgjClzHAGf4EhyRK7hO6gQBTANRno18IPZTf578MNU", - "Kh4XEUAd35lh8lQlqbJJDJKorg/DcsgURx8dsuCnjY95j5+KUI2FV8yUQLduGrUcYT37EbzquW4nQIBh", - "Rx+kw47+eyowUwbcMYNGRNPikq5l9QX0Jl2vUjjADCU8MbUZgKlmWLNcqQ2AB8BRhBRsJfetVtxhJRvm", - "Y9He4nEj1JvB5qpsI8rQyc+FzTTYeerfT5IEgvgiSv77/PUrBKeyXgPzWh4hZLIImFYYUJjC1amTac9x", - "MEPmogrK8Q07NBx2suvccB3GmkqbMN/rwZ3iT3poP5luujT8qd/XTZnryn30+0fTyr7eS0lsUCiHnU9d", - "VHgwpWqWjrNn7/0EbULMOi8JArRmjrl1kCSYArhJ4cQ3RyRmIeL2FIgWCKNcAhUDV8aUYbFYlrvmIb2l", - "IJ+Y4LkCMT4OIVhu2NkfunC5Yac77BA2h99sTN2w88lPAXtr2Vz7Dc6z7HIzY6K9wWB9NZa0pa/nzrLF", - "xcAXtgEbraKscKVeQYv8+W3dD/xH25/Z1Q9muvMcDccY/s75/ggvIAoae9ES9VxBVNRuzAISObV7taPn", - "/i8P9GIFJIrum0Efij2z67EMzv5RsSMsVr6NlrrvH5jjBvd1qJTc9g/Dv4/Of+7xnlvfOZm7UGd/pQ+A", - "PbGmNDIvIyzROYypd66N7+fwa9/+19l+AON3EfHpxb4x3VHEpyiizIagFwKVtXpgaQkfGeST7DsLhOLK", - "rK0ZTeLf//wXDIqy6b//+S8LJ/7vf/4LtvuGLd0Azc0IFmpMsLrYR78RkvRwROfETQZg1MmciAXaHlif", - "PzxChWLxVkuTQzZkZ0SlghVC9U3FM2kbtFcFej6UpURa5Bj9Ip3YciwmttHjt3F72ZDyXnd014PABzMo", - "TECfio4HAL6MmlLV1hLt+F2mZs4lp2k1TLMWrLdavihyrQz39swAbyhggMS+fQcP7KTR2vn58/U+AmvL", - "cAWU3AHbIW/GmhH97zJptUwyEqUsUIDKRjYZOKXlTv8j+047r79t8Vty+9sqbDfw+xvnD4AquhX4fgfQ", - "4g7ATzd3H+Bzyh85vLC7CxY0XTxQrKDjvTrNzZMCyR7CGYDWHBADOFS5QKeHxwiHoSBSrv9nuwr0TA2X", - "5kcH4gyKAjzErbUdCxcWosqaamUGeSzi4MyOGmE3r2pxy+L5tlEqFdF40mVVI/Ij7+5Pj0qnNzlG8gKc", - "Oa99P0lWxulRGXD9bYFbegFOgJBOfcn2aZGLVjmkTARgduQsVZeseD4+chvy/lxTtuuUVc+GexCKRxWB", - "+ICCsJylWSxZ+5i4+W22ig4bdYnn6utizcH9aUH37cXysfljcmOFFbJpKWjwBBoP0BdEGRSBzh0utO3B", - "M/FzItyudhXGYdbZtMynyMAhwITgan657XtsXmln+pr2viXLF8hzE43Fkvy7itLC2M1ptczAPba1UO/O", - "voUebmTefrkbb8tgHiJD2M3YeayFIiFaw3LBgvXvl95fnKNNSFRuxAo3bxKiJMIKoiMBiCWzs/TYtu5B", - "r8vqpQqsiI0beozJeKdpFLmrmTkRCr0+PDYioHhYbXyESLLVRogTC0vPrbdnL3uEBRxCB7OwN7+2Z598", - "YVPEcFYpwe7++fkRJplRd/A2qWKfsf62PLUJSu1T/l9bv0R0LLBY/NfWLzhKKCP/tX0QYUWkWr8zZhnc", - "1xly36bBI2Y+bRnQMtFANLEpQAeuUKWzt1pq0+79b0qhNpO+kUqd0fW7Vt1Gqy6Sa6libZfiTlVr08cD", - "3R1lzOajNjz6jjNxD+5Iy5EFnInS/UyONDHjUsGjx5d0aCM9acZxxWOjpV8935BLjw/HusdHXSAkVBYF", - "ZHOb03NPXnY3jntXbm2/9+9iP4jHdJryVBbThWKsghmRNpUuImUB/NjU7vx4blS8v2IuHdzn0XHvevV3", - "vr8jjb+6oEZ4m6uyVTq/e6utzm/f1zq/gRm06YYWfr3rSnOsN0Q/OqDBtmxcwmOsR2X6xuWzRdBbbajk", - "5gICC2J/yP6Ptj9+VwTH739yeU3pYLC1B78TNn//k0ttYieOVQiDmuGQ4nrw6gjuJ6eA0AjFlvIsyuo4", - "TO1WYD0HL/0fZyDlV7TtLSTHhd8tpFYWUoFcyy0kuxZ3ayKVIerv3UZy/OYjuAX6/TatpK/44uHeLTiZ", - "TiY0oIQB0D9ki8papJ2x5L7fjNwyS5DZm75CmE5JE2ltRmZSa4WGntcWvfcQreO8mMp9W4+ujOnjzHbg", - "ia0LaO21XFtoNti+Nn4Y3O/pdf+G2mNmMWMR1UmXaKXbU67DFKqJUwXhpTnAD8TvImHMmqzFPjrM8rpl", - "miRcKGmK3YCFYMphzrSF4CuMU6514ytuAwVdKJHdIYNyp/qxwafYuCQLU8qGcpZVrclmaivC+LLoyqWE", - "HnQbfXkl1F8nqZUSes/b2Fa+ezgl9MFEx72oe8elgqJr2cYAi3tMsp3MszRN+oGy6fqjiiU2wiqbWwGO", - "zKNqbeBUcVcWf2PGDTKRH5ztNMIBYLPp1wxskM37NThhxaYgmVfwKCLCwEElqXJ1s4YsGxxlhbrAtkjF", - "hW5+lDJFo4uuiaaBnH6JMFtYTJQhK3WGlSJxogWbRfmBEQqSmBFXCobpQVOeSniriyQvdYlwdIUXcsgE", - "mUQksHOD4oqCBAY5LYr66FcOidQITzFlNrdXv2nKbf0gh+yChhEZ2TzoC0QlkjMuFGEkRDGfE1nul2AR", - "USJgEodYU06iGC8AkMhgsxn68IQY0J9StjXX/8YspFCMSvecTXl/yDDaGgxQTDCTiEJCrsQTor+ybSAY", - "RGlAPyKMdgbP7FeVdQPQTEf+Nb1fhCBzHuBxtEBEczGciGodFjDbX1DXUS/fhApp1itzL9qqP6WFpdLV", - "pwy7KGUBlE9Mhf4XFyhl9njVLQrIMYd52ks4QkVWdswmxI9JgDU9GS/3A1BkPAhS4Tsc9VIXCuP9JyqZ", - "hemdA6l80knTAcGeCmHNGVcz2NMcttL6jw1clTPVt3HIeDcJFwijAl/nDgWoIc2maA2guy7yklvMVW28", - "WP/R7R29fa0gcNvfgGc9lvMJmIhPJqUNuPpoMht4WeJCnYW/1X166GotFkVcSPGUcalo4IRhtRrwd+Ox", - "tfG4nLJebp5wcVnUrcr8+wsXl22tr3NX4v5RGWHFGX6F9wB6eAC++vDXAeCMNoaKZpp7N9Cq/JXtUlC6", - "qJIuzpijiLOp3kW5U/zevfYViy6IUlDLnSnnnCDaCBnZH025Rj0ZWwwPPPyBbfWhZZHu/R7ugl5xhWic", - "RCQmUM6xZ5hNL3amVZtqy1SiWVZH72ayUu+qYlKusQWluf7vOnUI+Mot2Bpo7/Xl8grViE9XA3FlnTvU", - "KQ8S15CZatDElY6+QJkM1gqtgb1GVzMazACVC+xW3b4B7cJJcpEBkq7voxewkYu4rND5mgG71rwmeUQM", - "2NY8ji/26wUL352cwEcGkMuUJrzYR65IYXZ+SP1WEWVLzyLCUqFXFjtsLTPGYUUvFNb2Zja/dYu/lQPG", - "DpkPi4uRK9sgnaCLAizXRQMul5O3L/n0wZSxbjPMt5mL4siajsCbhIWdphgLGvkRuTYHAx/6bEt0MDOM", - "OwYHqw3mJZ9mEOMlVsZJ0pZ97TCBi+dxvISH0VouQZBUIU/V36QKiRDwseXuJuZGaziw5WXwpWZUZqSS", - "29jrwH7eSCKD+esllRaqnW6HsDTu7P9u/zWP4063Y8dTwAq+gXK/AmWt2mA94kWvTAFK7btafhOQtLKw", - "L6CkVU4Oa043a+Rn5oVv/mbR+ewekA1BP6g4cb8mFbQw3rLDh3EkGU7kjKvHhctkXU0Vra3ZVeNm2dPD", - "C1NXA6NNCMe5/fTcffkVWL+rIjvcmJGb7r2HeNRH8JgzYWVtNhMuqmA+q2I/vnpG+nJLUptqGw75zps3", - "9/O1YszEV43fLU1oaiLhVPEYKxpAPY5gxrkssP2YzPCccntV6u6sMs4E54axM20I/YVm1QvrCL6wivy+", - "dVohXHxk++jD5zbw3v+Fe5R/8UvBLs8kftcp34BZDQWDBSUTlOBUEq1XpTFBphS/LcBCcDBDAU5UKgjU", - "liIopozGaVxwNWjDScxxhKhEF5vxRReNU4UiLKZgF5mHJpxekIDHMWEhAQ/ZkM0InlNt1AkUYUVYsOhJ", - "AjUp5ySv9K+NfBuFY2paCaI5kHLWRTFROMQKg6pxoXf8yGTxXGRlKo1hzch1zg3hkImU/WhwtnWzF26g", - "F4hIhccRlbOsnFmAQ8ICL4j1+dctxr68N/icqOpEHygu51ay9CEDdYpeTzecryOG55EFI3Nhl7GNmF+i", - "9MpmI7Kc/uDY6D9zS5u5ujk+0BVPRuJlu/jruNvJmO6rud95+AscLlCYmu4KuxLY/Fu9lckESjHcCVIr", - "zTLe9momq9uUkflGMm/jo/vz+BbetK9EEnYbDfumCiH5pL8GkWupeiuZ+0BuROtLKnjFHlAEu5iqB1Of", - "uChIucfi7rQC22zNTG4XpZMSGKwvzr6L7arYtiEHtxXbzjdbu1QvCHLKehCl6Zfg1o3bKKqt6+A/NBek", - "MruCyHxwEZnfHdybWDzOBKERjQleRByH30KY7pIbnIALYfAfAFHiMeGPFryGxQB98M11MwnRdbmV705O", - "1pukhFBLZYRQj1hClGv0B7GviP6cCEFDVxz88OTIBsxSiUTK+uh1TKFi9yUhSZ5TAkAefT0/h4RRL3Nc", - "grzodghTYpFwytTKUeSv3s1gPt2qOPI9y0kLFf39Qrr1hTR49h+fOAMpA1kTZgLLLVOFVWMooAuNo8zU", - "Ptd6GR7zVLeuZZAmk17PKZyCExoRuZCKxCYucJJGsN2g7ICtSmm/M6vchahYvXNMwlpCREylpJzJIbPZ", - "GgkRum/9uW6/EOLkvRBQOJOvp0ZIfh3hc3owJmIMqyaqAWYR1ITv7Hc2cJJshFjhhhAtO7zPGNIvEA+H", - "5CIe84gGKKLsUqK1iF4a8wTNJYr0H+tLA+pG8N2Xrrl5+52lKX3MJtxblszwbMbM31RelRVr7mLy0Ym1", - "F6S4WZz8gYX2izW5Uq4JgqOeojHJkGtQqmhEPxhRpxuhUtHAJP3kkAXvTnLUgiE7IUrodzAkl0URCZRz", - "2Gwkggcbw3Qw2A4SCvBn2wQGBwKv+XEMPR6evjWJoCTmYtEdMv0PaPjNwam53Z1g600oDJQRdcXFJTre", - "eL0ixPgcyPQfHKNnJrgUO8C74N+vBG+OCNK4h2TDFuXJMlOJJ998EKnV4L77FR6nXwEgmbLZrE0FDkAp", - "lrNUhfyK+X0Icx6lsf6H+eN4FbCXwsHsHbz61Wi7Zjgru3ETfBSb0s4pJKZs4oNcehiCPdaYVU04NwVQ", - "YkrRgN5T4EB9i9z95d33RTp+hdedlqKuJOlXs7fu++SzY3AYF0V6PJZtbjjNzUTx5d6nK0ybvU8/Rzy4", - "lBYMpeg21HYbAIzrH3NAaHtFCGoC5GYiCyKEyHVCBSC/VRyQBnNHIowUETFlONqAOZtGANraebHwnFNI", - "kQ4iCklqNATUogjQ6a5mhCE9G3BUuQYKN7rSlpYqvlO8jFQcjUnAY+Lgvtd9ptvfMVW/cFHG7v5a5OKb", - "Av31fPRU9TxXwJU39/hZ8OUn+BpCpcPUXii7Ea294PmPxhXURbA2w872QA47XTTsbMXDjl6BQwwuVKzQ", - "LoopSxWRfXRk/FuQBLs3QJIEnIXSoY47D972QDalxBq2bMiv3IPv7lPtsVwFpDyznfjEg34P6e8haQet", - "FTec3ZNhFzZdiHiqjLvf7iv7VkgUuEfW7/2utrBHvtv2bST53+32LckoWGUtLgtLbyR7ksoZaXa5vTSV", - "fFI1BjRrc+2jv0F/8LHsIkaujDdcSNWvyT399anp4D6Q9nVXN0HZt3P/DrHfAmI/p5UfLtEEWOoj2XGH", - "wUwk1wYRFru0d8tDYEkAdgMPcIReHx4PWaBFkQH3EyTmIJ0sILg5hQ/+fo6eH5510RFUekS/puP1PjrM", - "wGLBmTtkYy4Ev3LOXN2Ji/iA8z4RfE5DfTywEDGiqZIQIalUJPwRcTUj4opKMmSgpwDZfpCIX7F8OEG5", - "S5RKEvbRaxYtXEltc1k0ZEYlNFI0wAyNzfYhoU9PMEQENr7LqHXdwQNVRzZb1HPF45jGRat3OzOCQ1CN", - "PnZectOZB6z47KXeyXql7JcZ/3WWakGdM6LEoncwUUTUmz2xCVssg8+w2oJDo7MapMHA1B1KB8KW92mU", - "FIOSsb3V8YBmfPpefuHuS5Xez3WdCVgxhe/GKYCOMsh2wOHicQVVyRnKpLRPFhf1hqyAQVO6spVlSy0d", - "6LIpBP0r8v0vlV0lSPr/1N0FM320N15JaZ00E2eFT1ZeObss5ZlBRrY3ZgFOcEDVootwFNkzyp4EWWhM", - "L9PDx4Lgy5Bfsf6QnWUlV2xmMTo8fdt1N8YopPKya/UTuBTuo9dzImQ6zgaHYKMZ9QNoTsIhUxwFOArS", - "SKsbZDIhASQFQyUV2XCpnA2lc4d7J+/EW/alEF6fPrpqc36egNXL2aLKcRtmqTcECSJM42YccquoQeQj", - "xDyMdaOcIcomkY3tCgSXEtmmeiSiUzqObKSS7KM3Wo/FMRmyJMKMEQH6KhovYOi9RBApU5NprhsAvF7D", - "UV2UYwwmgisbIxFxLqQJa9Ac/u4ESUWSJWx2Zlo+gTnfkW5rGrc9PZC3vDKGZp+MfQXpBTGcYgiu+SiN", - "XCTlvcbEmwE9tJb4WDb+G0GnU205Co6NkDVxgWZbO3KaTV9KnW6sPHmevdWu8mTWaiE9spA6uBQjbpTD", - "boedm4Ufejq/pI0wgvbRzdKZf9Mftey7nDbrH4R99Jmz/FYK+p8XshXbetJyDn9sfq3CyEtbtZTxuxrf", - "q3WK712m3LYG8now/K7HDNuFS3m8TQbv18cIg/uFm7jv2myPm7dKsFsl27QBe2A1sP5XwYF3g6j/wHAr", - "t0DU/6oAAADy/OGAWLwb9aES+kuX4K7s7TcPin9XefwGGR9w4Zry+I3Us1G0Sw2ld/addmaSbfFb0uBt", - "4OUN9HdH9u9WfwuToUCsVXfhmuFJnKiFi6yzd5V59JukH0i/4SI4C6C9u6vgW8SWfjn2cHzaGFn6bVap", - "f5DgVVtFkEp0fOQp//7IwA6Le650sGzoU6eHRTCjc9LsdC/vYEuiRJBewhO4XAkNwSw93FmmsOhPPyDb", - "vAV/tf+CMpSA2k9CFFJBAhUtTElQLRFMHz9IJLi2BOA5F4vmKBGzRX4RPD6ws1lxHto9ZZ1hecBjvOiF", - "WOHe3EmbJS60zwgzdYGdWuAhytCLn9EauVbCFLtAE235IDrJSGrq/kvgyfXigDcHDZ5N+oGMpuM2o1xS", - "tuS1LQuDglQqHru1Pz5Ca1AGbUqYXgut6k9Ak3XRRKUxduY8MlTdbCDoTf2uWqnIatg548IM7kF0mDYH", - "0vQDTcpiIQuJGVOGYXArC4SU95RBE9D9YcpcAI5dIzeK70eYtfzWnLGjORFKcloiKs4N1vT692PuMR9z", - "xawsd6aVTjsXnrPced0uUatl/tRdVKDIkvju12397uvJLaLyUaYVWdf5PDNIm9zmXxcLDu7vfLhvd/m7", - "R5yL+oI447vgKocGdIs+hnkJweUhmZOIJzGURod3O91OKqLOfmemVLK/sQFB6DMu1f7OsyfbnU/vP/1/", - "AQAA///3h2dLttABAA==", + "H4sIAAAAAAAC/+y9+3IbOZI3+ioIntloaYakqKttdXR8R5bcbm1bba1ke76dpg8FVoEkWlVANYCiRDv8", + "7zzAPOI8yQkkgLqiyJJsSdbYsRvTMqsKl0QikZnI/OXHTsDjhDPClOzsf+zIYEZiDH8eKIWD2TsepTE5", + "I3+mRCr9cyJ4QoSiBF6KecrUKMFqpv8VEhkImijKWWe/c4rVDF3NiCBoDq0gOeNpFKIxQfAdCTvdDrnG", + "cRKRzn5nI2ZqI8QKd7odtUj0T1IJyqadT92OIDjkLFqYbiY4jVRnf4IjSbqVbk900whLpD/pwTdZe2PO", + "I4JZ5xO0+GdKBQk7+78Xp/E+e5mP/yCB0p0fpIqfK8zC8eKURzRY1Cf7irL0GnpDOFU8xooGSJpvUAIf", + "oTGWJEScIRwoOieIsjFPWYjeHJ6igDNGAt2YHDI+lkTMSYgmgsdIzQiacangHSVwcIkUHkekP2SdbmU9", + "CNNPwtVU+vuMqBkRnsFSiWwraMIFUjMqEWX6aUD6xQVTIiV1ynY7NIzISNGY8FTVCfULv0IRZ1OYlmsX", + "xalUaIbnBH0ggqM/UxzRyYKyaTORxmTCBUG/LBISY4aSCAdEIqoQZYq72Rga5Ty2G/uYi04ZF2QUEqko", + "w7r9UcKF2RHl0b+GP3CECu/C0OB9pGZYOS5nXKFLQpLyRPEVviyT8fetre6zwWDwvtuhisRmW+FrGqdx", + "Z39vd3d7t9uJKTP/3sxGT5kiUyL08O0vWAi8KExH8lQEZBTQUCybSRBRwhQ6PD46u+UEOpuDPvzfxtNO", + "t7P5bKu/ufcU/r251ylOq0b48sg/Ld965wqrVNZlkNlNI8soowKT1Gf9WxqPiUB8goJUCMJUtECwpUjY", + "gulK0x74liLgbEKnqXBb0LflSuScYYkwM0KjV5EXeWOt9l2ghVjIr9hIkBhTpmlcG8SZe4T0DkV2E+kh", + "BZwpwaNICwWlSJwo6XZRV4txhnCSRDQA0VPaVDvxQHa6HZZGkX5YGWG+2iSiUwovtCINlYVFct8ixRFh", + "iohsh7chTUksNnWck9u7GrlcbNXljEfhKGWKRvVe3+qfLU2LPVGJZiQKEZ9MuohOEEa6Ff2z4fEy2bcG", + "W3u9wU5vsPdm88n+4Nn+YPcfnW5nwkWMVWe/E2JFenqV26wNCG1JWeBfHVZlkVgfSIIEZnWyA6u0gGMS", + "8Jgg3fTSke98/sgbj5vDfFH1i8i+mJ+snqXuV46MdtwdYakyIQTrRdVihD1jekNjIhWOEy2H9BgKxGyS", + "Qq7B6jo4yi8l8OZnEZiRazWyFPLOx8cf5DohgT4RuZMmmYKh27PsnYmse+FxQbDUA9Zqkj5Mf++kTKaJ", + "PrpJOEoirHS7WqcCNhjFVEr9afZDSKWRI92OY/IR42okUsbMi4yoKy4ui2/aVkY06XQ7MyxH82mSdrrL", + "jq0yU0MXJMKJhPbsiosREYKLjlGNF6MJF26R9Jmbk3BJUzUKyeyI9VCo0+2UCJCJczcXN+5sVb2Dg16A", + "l4SxKowZAJOpD7zYVn242dCWC3Zzihgl2i0zsh/LsgQIKZ4yLhUNZCsxD8qDXt6Yhx7ReZQ1h2hImKIT", + "SoTVqwkS+nyICXKNIN0IogylsrIPMtV/RObaVhvNd0YqSOpEqRg2xcUr6Cb5iVg4lbPlz3bKCiYtz91r", + "OM0xhT15RObUHC1l3c0uzSgUdE6ER3xnCoARheY9tKb3uhYhjDOyXqIUm9OQ4jbiIIQxjaiHe04Pj5F5", + "jI6P0NqMXJc72XoyftppbpLh2MMLv6QxZj29IfSwXPvwbrHtVzteE4XHcTqaCp4m9ZaPX5+cvEXwEDHQ", + "cIstPt3yaapJQEc4DAWR0j9/97A4tsFgMNjHW/uDQX/gG+WcsJCLRpKax36Sbg5CsqTJViS17ddI+tu7", + "46PjA3TIRcIF2GwrN06RPMV5FdmmvCo+/n+e0iisc/1Y/0wa6ESEpFJpefXcvIYEMaYcuppxSVCAgxlB", + "Y2O0gHkO7bVh+axjd3r5RnBEZcJBzqN3J7k2B6YhuSZBqkr9/ohCqm3hwCpVhcm1GFHA9Rrp89enWQAF", + "kH1Hm9rKqU63VgICQfCK7vQbrTqr7//UsNcolk2tu1e0mI9pFFFJAs5CWeyDMrW30zyZwi42x2atqxf6", + "ZxQTKfGUoDVwS4G5YSS81rYmmEYkXG+nYTdN5g8+LpxrpT0HbNDD42Bza9sr0GI8JaOQTq1fscqD+net", + "H+t2FIK3/RMBDaPdPKBLQSb1/n6G8wQ6EWRCBNEc/5ndJYLPCcPWpPoL9Nv5fzZyh+uG9bZuADFP89c/", + "dTt/piQlo4RLakZYExP2iWYjIDWCL/xjhkfL1rrAUVJhsXx/wBtfYCfmyuZK2ljXj9a38HTlJ2/0O1WB", + "DuIoU3AKUqBRbr/QmpZHZeFM2QcVFzCfoogyYwZp+W3WApS9RUJ+ivh0vfPF6JCRv7759bhvIbzMDw2t", + "6WfdzCqI+LRIzRnBQo1JiZgN56ptKB9dI/lPS9uncoBiSUbLJcgpZYyE4HO3G9u8qXVrr+0Du+iSqtFc", + "n8C+PQfD+pUqZN9obCriweWERmQ0w3JmnZRhSI3D9bQ0E48KWbrMwOAkcA2CagNG9fkvB1u7e8h24KGh", + "9f7qF+ozKXytm7fqhcJijKPIyxvN7HbzM7rOIX4OyB2+TWdPxoGOMY2k69jVtMZ7KmfmL5DdelRw9mkx", + "oNkr0n/XGLfbue7pVntzLIDquvnCsP7H9VT47XneaeHX06z/wo9ndiiFn352oyr8dpgP0FHFmEleFzhM", + "2q9VcspUjzJYAq2AW8+TOTpyN6v1ZCC9jJmGByf7ZylWhzWVCq2d/Xy4vb39bL29ckXl5UjSD2Q0HS/V", + "m+2graKsP0P6M31UTukUjxeKyD46xIxxhcYEBTPMpiREeKL0V3asJQN8d7DqEqZRSSKiSUVSH3bws6fX", + "11g926NX8tmHeCymf2x77z/Bu6jlTbMHDtvV1K8igZlnEW93UMf4Wlv+hkt8l58n5roKZS/ZNZDVIfRR", + "btBP6LWmuUKbJUp7ieu3/rIrLMZZL2X0zxRWO4nwom4DKoLjHnCEj7xGbpgNJFdvH6n3j/nGzfQKU6V3", + "TnZnaqfcRTwK9bkzoUKqTvvbsBspBkR8KRWptMv8GlOdI6oEbDzUs3E27pWITkiwCCJSF+5OfCcite7W", + "kOgzB/7MHIhtxbgbSkkSux9Psy5KPx/l/ZV+f2E6/9Q1ko7Yh41hE8tF2eFSufU6pkprAKk093bGG4zs", + "df8XkFr5zTCOIiJ6Mk2SiMIFihNjXeM0nRJGhOYK44pTSJs8NKzEdeQ7r7fp23t3vrlvtCFqfGtW1Dgw", + "Gxd0xRymEdea1QK5ieS+vz46LtOuizA8gCuRMpWzS6uCfw6tkf6030XDThLQ3mAwGPTwVm8w6A2GnbKH", + "LdrpmZuHBCtFhB7g//c77n046P1j0Hv2Pv9z1O+9/9tffJRs6zR06oWd55qTMl3kBlv0JFYHusrLeGsB", + "Vxz++8alPtbWwk1X+vC47j4wcw15cElEn/KNiI4FFosNNqXsej/CikhVnvnyd70r4m6t6reMWExJ4b4D", + "S8TlBhbB7PeNORYUM/XejBENOxFl6fUGjsO9nWFnvYtirAKttKIjGBHq9Vw7VgC5K4zSnUof/cYVwvZo", + "12J3H16q9zsVOI6x0Dw+xxENgb2NJOtJGhKEWYgog2f6jZRIJIhKBUM7g4F7MioMSpvaXLqpDDvoiqoZ", + "0t0imBbiAmER7+2Yq8aAJODG5CGu3PcWSFHfK3qj/G0j+2vN/dl//7f1//OXL8qswGhLuJRNNR/fkE8r", + "DnOQM2sRvyIi0IZyRPRsZVfbylTJLixDCDYm4ixa/IgCozUbHxAXiLDQUhreK7NzvOjhhPaoGarRG14R", + "NlWzzv7etpe4a/aP3vu/up8aCCvSiHik0BlPQQ+Dx8UINjeGTP9atiKOumkE3riYsmPz2WZdTfu8FXYT", + "WbbSxgPfuNT6lMiuV1cMpB7KqE232KcGzIkQeisauXZ4coTWInpJrEDTpiIapoPBdgAvwJ/E/hLwOMYs", + "NL+tl3UW64zxqCu/d0gw4+BPiiJ+k8gxUE9BO8HRUnfLMtJ4qX2YtVt3zvzCperFmGFtPOYDQGPBL4ke", + "qIknoUSiS7LQpsICTXWjvTmVEMxE2BzNsbmx6g/ZG7jfgVfcIwlxIXROUMyDSxPlOOOgKRux2EVXMxoZ", + "VVAQHOXSMsaUDZkW0D0Z8ERracy+BlNDF4TNL1CME9jmWBDY41r2E0FxRD+YaFWIUCEh1TJ6yAhsDJRg", + "veeDgIsQgsk4IjiYFajwg0QXxq90Ac1fUKbZ+sJszEpc5sfO67dvnr9++9vR6PXpi98Ojke/vvhf/bP5", + "qLP/+8eOiUrOLNjnBAsi0F8+wnw/GS8kOEc6B6macUE/mJs+iMWSChR/nNA+TwjDtB/wuNPt/LX4z/ef", + "3ju/mQmBmOtt4BnYJ6+iaJQdj0g6cjfJEtnbSRcXo0mmRdTL07cbWn1KsJRqJng6nZU3htXdbrQlwMig", + "fDROpPeS7xIdb7xGWrNEEdUbNNMkNweDk+cbctjR/9h1/1jvoyOza2H4WgZxYRVcOdPskwU4H56+RTiK", + "eGCvuiZNsYyuK5+AJ0yJRaIt75XCKX+1LqN6vfzpDUTRxpiyDamXoRfcjO7AN7f2+L5gcyo4iwlTCNSl", + "cURkea/89vroxejFb+86+/ogCNPA3mifvj5709nvbA8Gg46PQTUHrZCBL0/fmog52DYER2o2CmYkuFz1", + "4S/w7iG8CjtOJVE6BRu3vnoHGWlQTGIuzCWJ/QatzcpKitnyCNZ12Nl++dzw5eZLYEm3njY0KWvFNFyJ", + "Jnv53Mdos0VCxJxK303qL9kzc+POQsc89eB49CeJU/1MmshCEcyoIoFKBekxDCH/cEpjEaIxxyL8Eb7o", + "xTQQfB6bL//nxcnbHyQCsx1HCPIKrAYL3xiVjBMJRqON70F5wIrsVkjQ1UInNlG+mCFCpzOF5lTv3F4c", + "U+6+K+9u61Vw2zb3iFpfTBDxNOwVKNftTKggAcRS63/pmdn/uAl2up35h3KMl6cZ/7VtK5NrhS2Fo4Qy", + "ssSY+kp05isuLiOOQ3CXfEmV2UYnesLizYPycmc2nmPwWrbJGLPwioZqNgr5FdND9hw09gnKXs5Om2s9", + "Exz9+5//eneSewY2X44Te/Rsbu1+5tFTOWx00967v2wiaeKfxtvEP4l3J//+57/cTB52EkYzu5WWa9f/", + "hWmhGjBv84jM7UBDmGWmzuTxQty6gOBz5HhvZUyl7+TicyIivCgcJ3ZMnc0ByPTKqASFDClkvyv4Upcf", + "Lro1p/W8rLqltgb+4+O7C+Yrd8EIAqJ3lGS5e8s2yJl5O7eOPUzn4bnn+iyw2kwbTssYbXPrxP65VWc5", + "P8fJS5qMwEwc4WkWi7Is3e/8kibW9oQvzDaNIiPowxSs1THnqj9kJpxf703gHHJNAuAFqbBCB6fHEl3R", + "KAL1Aw6NuiKkLdlCHgi8LpX+X5GyLhqnSpunXBFkHQU2rlCPBV4eE5Qy7IKHK8ainWA9FhvIckkEI9HI", + "GIOyJWXMR8h+1EgcmOoES5t+JFSalOl19OvJOVo7WjAc0wD9alo94WEaEXRuVLX1MvW6Q5YIiOnWnWh2", + "pLZfPkE8VT0+6SlBiBtiDI1lXn8b2Tp/efrWxkbL9f6QnRFNWMJCm8TpNAqb4Bdy9oOWyCQsN1vsv0L0", + "pvh3yXAiZ7zt5jq3r+e7q733rNuZB0laXtKtbmOCn1ZxUxzps7RkP3mjoU2CssdONvnPRXvdnmt5QqQq", + "R4y2dTGaliFb2ZsK6fEUGk24taew4Luq+QydY+Vju8GuaP+YuYEs9ZTmvpXP6OvcNFLLdDA/d93MbkGl", + "44wmFf/qlyHPgSz4ololFpuT3Rx/Eq1d4IT2LR/3Ax5fdNHFX0s/6L3vDGKtP14hQw2QJ0z/VGy/6oVb", + "6R+7USpvcXGwvP16HMjGtBA030RKYCZNQs8MJ6SPfgEhjhSJtWoCeXQSZXkwiPGrHxE3Sqv7dMj00CRa", + "07LSkiPzkko6ZZRN17UZpw8mHIbGlTpJtWWPBJlTmVOzzDrOXVnTDM3obFQ9ZL9TFkRpSNCFc2lelJWi", + "usOz7siwHtCaBWtIApYr2PZqI06V7l5POFM0eapMlo2dejkDquJWXRXoaceSxRzdYv3PM3FRBTmYe0xY", + "PTl77Qx+8IJDvsnvbRUVv0/+kixgyZ3/Hdc88EXXu99BLojk0ZzYY7fovB8DjAM3ilPutzceeOt019u/", + "CmDgc0evWgpNr9bkL5uCHvgGqXpusjnHWOvOJdA6KaQnZ/oDL5QkQHwwLfcRqGMXXWMLE7A8ENPMEqGQ", + "ChKoWvOUTYcMYtsv7C9929qF3uRaR/kioBiQYw5Ke3FpUWFlndoHzeip8ZgqRcJuWTe4JCSRqyel1Wt7", + "U+O5ThLkSlAnyFwGZkv1jLAJFwGJrZHweY6BF4XGloTKtG2iHmpu6FsYs8u9xzYCCdberAfcK5RS8qvI", + "NGHFajOhzeUuL3AUXaA1+9I6EuQPSFu2a8U4y5n9zeGpY4EsEOfdSVdzpJYCFzOlkpH+HznSu/ii2pj9", + "1u3wHDXk6QDsq52dbbuq1sdqBlxptuw/9YZrNy/NaSpnjdfIlSvUZbyi2zksvN7spXXwLMZT606gJJUz", + "tKbwVG8dG5evz4n1z/XgUiZJkAqyeq8fgHpkY8NhTBGmrPfLmzenSJAp1W0WI+uLOb/g1mlIIrKGbR4G", + "5KareNmNt7W9s7v35Omzfngp+iQQ/VT2CJaqt9nHMf7AGb6SWu3aiBc4SfbnmyvlvFmCbHzNIt9ZYc2s", + "wGMtkl0aRBuL7jD/JL9BuqQsbNvAr/rd1eGAzuC8az98IkgvTaYCQ1rql/TC3zpeBKjZvKoroLqWh3qm", + "UvG4mKO+tjzCs0ysOY96IVb4JgGeZrj1WM54YZoyJrnX/dUUOHtejZYtBZZ7AXs+N3jHjcW3LE1J78aR", + "QMKR4p5cbqc7HB8BRoZ9t1V2L+ToKz6aTyhfDolgAzNLEDtGK7HuDd1ELwmo9SqBqhvMTAKkIQLYDu9O", + "ijEL/SHrgRa2j46yDrJmsyYxmBg4NNe+a1wUBmHAD9B4sY4wenfSR2+y0f4gkb1FtSgAMyzRmBCGUrhh", + "AqWoZ1Sy4gBSCbqTqn5uXWgGMGAdQjO4fdbPYMXAWZeBpEEM75hW5mOAfGCh7FmiD7zcGdrKebksfebM", + "nEfLUmhKkCm7vcFmb3P3zeZgf6D//x/tc2y+PCaCr62DsmyxUdFF6XP49vhoy/rM12+dNfPFURP8ouwo", + "D+dGa6kkoufEJFx5eIK4C7HSDUHat469vqNQ6jw/dNm7hhJv9Jt3AQfhS1ixGaU3B2yoCsyVWcGFydUd", + "OosE3A/5Liko8jbsPqCe/NguhGI9FwRfAsxN/dzWWp0cmfPMH8eVShNcSK6tk0twribSKN1l5/fmzpOd", + "p9t7O08HAw/gQJ3heUBHkFjRagCvD49RhBdZ7t0a3GuHaBzxcZnRd7f3nj4ZPNvcajsOc5Pbjg6Z/e2+", + "QmuWIn9zUKTuSWlQW1tP9ra3twd7e1s7rUZlrw1aDcpdMZRUkifbT3Y2n27ttKKCz6574QAgqgq8D43o", + "wAD46X/1ZEICOqEBgsQppD9AazEcYSS7tCzvyTEOHWaT/+xQmEZyabiX6cy+afytcRopmkTEPIMFaXUl", + "ATM/gpa8IJiMZRhZN2vJwmasDARyc8leAXUmJON0OrU5aRnpTgwIWEF5oiQK91Fmsi2Vc7Ca+cDeN/GB", + "nUNLbnilTadeROYkKjKBOboMGpUgKOMTs2ilWbkYAMqS1MsSjaT8ORWgi5pGER7zVJnbZgtqlndybDwH", + "cPuasrCdu+NnLi5Xxs3rkzjDbmvpMJhYjx2c4jjLl7YZ9AWlL7sVNnfn9rlEZ+YL4yjMf07SMnBtF3qy", + "DkWGBJGKgyS1fmPbTFvt0q+3gM/chZGY/nLZeU8hbr2JiQr6sha2mBLALFQrNRbNKSZO5xxeb52Goz9c", + "6UhpQXdGru6D6JBI1tNs25MMJ3dD8WWRtJmvIX8JTmFBQ9JHsLsg+M3B11R22rniSULCzP/THzKbx5L9", + "JM1Fmv7Q0EHNCBWICzql5Y7LftZ7jGW9CWc65ro1dxY/bIpNk9k1oE8GGESEiQkNNUBdxYRsuyadbuc8", + "A3e0gqlMmrMMILNGkTzivDbEl6dvbxqRmgg+oT6EYYiQsU+toeZiNV/tDM57m/9josk1+4HGRpmJqol5", + "WMFitO+3O4henr49bRpTho6IiqOrzSmLg1oGZ+2iXSYpM/iMBZ8KtrjJxrwxt3K2sz56TqTqkcmEC7Vv", + "A1ngQqqAzY4XSJAw1fxghkIjquAOQ9JxBHs061Ur/PoXPaGJwDEZp5MJEeW09Gc+Xbrw9ij2OPd+1s+R", + "eSHLiz95Xtantd7e1mo/LXEDmO0THFA2XW+93B6HYGUa3cLyvffzx5mF92tCRtCkzCAADShCH/2WAWBq", + "UkuUB8t5PIUV8IGmPKXT2ULSAEemRYNeRVnRwQe7obWGfpp/aF2hHj3dj5vqdh5am0+TFPb9+Vnv+PW7", + "jTgk825pTBDgNuMR0eNeL4inuUt3z5OqSlJp3uRpMYwh2+7YAq0ykdGaSAUB4aGO4gpHIxlxX8zQG/0Q", + "wUO09u5nk+2qR9BFSWkp9e8FKpT4e8+7YwBppqHbc+iw6rItbXCv7VouuGHcO4XplTr1bRWTYlTXseqg", + "xfyyvND8cjVQrmmkud9DlwVVcapbRRaZZCkEyVIuDAKZT43X3LpmJEmwwIpEC6PZZGK2DHxC6jfa5JoE", + "N0jDeqFf/2TArlJBRmomiJzxqBwHsd2tA6ZLiMWdE4sRaeZUcPwrjmIsLuEkdoo8SpmhQDmUe3sVAMlM", + "qeQGk/rlzZtTY90rIuY4qiYDyNoN/xGJ8AKNiboihLmpYIkweskzMM5qTqJswOkTapQQQXmZhp1tT7/n", + "Jj4YTQUOCDJfuXILdkkkxBe1JaXtxQOCHQREyob13Vy2vvbTSRq1W2PfsDZXVicJbrLAbw5PHd5chufv", + "yLxVp/IpET2z5Ryw//Kl3ZLLkQ9dV4wzUu9M8DEBKESbgVFMkHTxTQD1qD8vJSUWhIMsptHZfmAXGFJ1", + "zT5/30q7rG5330V+jFnoq4tgAu9N5vc0jfWC6CGLFFQ/GprgJ5OMbuyAYv6AIDikjEhZSZsNUhF1up3e", + "xM5qf2Mj4gGOZlyq/Z3tzacby8NIl8YP23CpUUiX2ZcuqMqE3bjsUlOfACZdZokNnCQtPHCGjivOBxBP", + "9XhFqIWgz7aChucC2AeDWgbwNQ6UQ28Fl1zpyhUXt22iWbI0H2gwqx+0++xZcX8OvHfgefkux/4bNd6H", + "uBiIadQ8YtwdFToaJv/g1ai48EF0cqFsJtyYuKDH7Dx0IYX2UqfU2dPB0+IsW1VJAmFT2eZ233mmat4u", + "5/qSArnt/rUNQHhWWedwW3o5yrimywqe0hKxxlLLCcoTwm5Ez92d7a2b0bPtRI5dWFhFLvkQRw5PjoxO", + "FHCmMGVEoJgobEvCFYQM+LK0lAEMOExiSJeZ/LhctDTETxQhRG4LYPmlbt8bsGzPTBhxiGLM6EQLZPtm", + "sWc5w1u7e/sGcTskk53dvX6/f1NghRc5kkKrpdgwQXoFjIW+nH3eOtwBfkKbuXzsnB68+UULslQKc2ht", + "yDFl+4V/Z//MH8Af5p9jyry4C61A2umkBs5ejkfTBr/5fb9Qk8zpPa0K+fid0RCdDGAvXhSzUpTm3cGV", + "ZTxuBnCbnNklOaRaXXkN5SMbaHNrvPW8Eokq4KwXs81aYK7TD8vv1527C96xfUJtsRyOvn6zfquCAnIp", + "5nINkjMhLENZjiLzV8DZXG9XH+Ry6Yh0z1ogdsIxYqE5sy6LP2a9F348LA6k8LtD/Cz8ZLGX398wJGap", + "Qvr3uh66Wgo5dXTFZvb7HrNToS3OvYU588aCP/BZeJuwsXLvr6f//ef/ladP/tj889W7d/87f/nfR7/R", + "/30Xnb7+LOyP5Qh3DwpT98WQ6UztliI8XVtWOsEq8PjotPnXQGH7xHgcVDADZG40JvtD1kOvqCICR/to", + "2KkkPw47aI2ApQRfaXVXN2VzuNf1x6fmRlN//NGpwZ+qbYQ2WVvYBckwOGQ6DnmMKVsfsiGzbSE3EQl2", + "gf4rRAFOFLhzKEPa/l2gsYBitPaKKe+8iz7iJPm0PmS2mo4SegYJFirDTHU9AFPYUZkwWPs6yeAYTEbi", + "kGWndQbxZu4Z+5nyD+Em1VwyP1GW22/Wcno68IHhQT6LXsgIkNhRdmNHJTB6lmiDng7W6/bcChsj46El", + "7Gcv4k3C40Hqcxc3JUkekZAGIFdcnuDMZpJmKZqG06wTLxH8egFrc0Zs9UucqpmWRSbEqosCzi8p6cKS", + "duE6DCI/4Etznz/jSW+86M14koEPYGGiXbC5ES8b2f+3Zyfae0cEndievHWzNIt4lE7YMnZmJuUw8y7U", + "JvZGL5wkTKs+c2JfpxIlFms/tKHcKhXMBLlp3js4PTYoFxn3EZ9O/qOtPCxdveEZ4Lkp3YwPkq0zyFJm", + "8DgIyaT673JIw9buHliw7t/bW60zVg3plnFZGnls2tiJvhYS24hJGIBRD0bOEb4iCEmfgPYeF/wUisN/", + "z5FrKN9xmSAxt1Mmh06a4AoVyUL23Lo3zcluAwulMcJ2N7U5j2q7sJQ11aIFE50An0UtQDxemGzNN6/O", + "kSIidvnza4FeHdglttArlTLV8pNidHB48mK936I2PCzV0qyq8qA9WAs2WqEpCCP30eCYdNHxEWTL2mMl", + "t8UgveFnLlBkTsX8MNpHby0afKkpZKKrDWNEizzkwagtw866azGpHm/76CwzAXE2lFI9+nIURX6YQLM2", + "AM7kXtRa79YqPQtn/tnzGDItsMpksVYTm8+v9j5HB8mkD6qKh+zGB1IxCqXR/1VY+y+N2vvldfTtm+no", + "rnpnMsPSx92z4q0mvLSkwjetXNWI5pihUr8rq1YXjiFF9DlX+txXb3i3t7n5ZnPn5u67mwKuliGiCvBx", + "GeZqe7DUuwAdrbuyrqkaNQaXI/3YhpI7v8i7EzTDkv2g4GHFO7K5/aRVlUPda9uw7GJANp+YIWVSyuFN", + "ZeHEBnnrkkaRUWAknTIcoWdo7fz45a/Hr16tox56/fqkuhTLvvCtTwvsVScqXp6+hZtxLEcusrE5mQ/n", + "CbHkmkol63BcrQKEPwfr1XzarlqNm6RpIy9asxww9pcSoqkXX239DpBeXXR3jZwPAIL6kFl0Xx8A61LI", + "1M/FPbWG+B3BnjbKfB9kaAUyYLdJ/N8ewPROhlNCXPGJraIK4FKcb40Y2u1QT3rngdSSmYTo+DSvlZN7", + "613zlTk92+pv7j3tbw4G/c1Bu+JowZK+Tw4O23c+2DIaxz4e7wfhPli0t73UsYxtdHQcXeGFRENnRQ07", + "xgosmH+FbWstrVYBKHVg1tvhsFb1nAakVdD3XGSPtFWWm3SeFvl7VVSxOI1MinixMnNZiZxhiWRiIDRN", + "zYNMxR0yGGDXQnnByUFjgnAQiDT3zbkSeUYhThPL90MmiEw400qyNnbQr2QhUUzhljDrHmIDJcrSRMIh", + "WxMupSjLHUpwKkmof4AA/a4N/Q710KiC2g76gyGTs1RpIbbeR4ecyTQmwrot0ZjClco6kqmx+WC8QI2F", + "FpiShkQMmX7NA9L5MdPf9/cGg8EA7iSMqbOt/z3wcdOd3g72LQitSYoFODxm4WgBZ0+kDKUsJAKdcUmU", + "wogYdqjGkN3wZvEz0Wfd5+3UK/t5rlf5N+YqsNp2KLSfC/25rNT0ebnIdGvrdfcfn1WPmrTVX206jv1q", + "dJMrfoICnkahNonG+rQzHisSWj+dJCqv3w0H5Ft2yfQeLU3dxuYqjv5MiVigdycnpbgAQSa2PHGLiYOU", + "aFgHntxoGbZWOBFWjuaWgLD3AQJb1VQKGuIXh3wtXsG5LGHDoS2u4nJD0ptZQZlZGs0nS+ZUcW+HZD5K", + "U58hoh85bJC3b4+PSsyB8d7m08HTZ72n48293k442Ozhze293tYuHky2gyfbm1vbS7IwWqRy3T47y2uN", + "+iq+u5DpkQvd9kXUNgXOV859Gwx8RVnIr0pHizc6s9i7jfxc1X09rrv1ELzZIFB8GFpqkBIncICSQLdt", + "oqEr9cAbvGx7bwabK7xsK+UFDK5B/r4RKTN3eibBPfNfx4UBFxerPM7biVMYkMu6WEWtYuftiTbY3322", + "v/u5RHOZA6vGWGWne1zcpnAkBwBcSU1w6XEFH40DzOxYfcO4lG0mQ6fbyZIt4G84aCuBvNnjVhlETRu2", + "6xcjy+R3Q+bucckWgDAJAwAX7mstwOn6ALOf5Ydr9eIw4mmICn4ug4cFl0DHBbtANwN3Mtb9ZfAtTSaA", + "th8AUBng/CnTghguv3QjNs13H72Ed+ERjo3JZAdhikoU731wuDDBGnp/ua6NAbN8yOfWdoFvtCGD9L9g", + "2poM1h26vAmj+eyj3zh8k1lSjFf9quZ1MGHqr1d9sGsWK9rBNkBnVo3bRz9nqlum/Fllb00S++fICqwc", + "LWW9lLNuV7yjuSVfuUL+dbdjKNrpdhyhIE+7nrH9Nuf62v4rsqIviongyNREzxJUU0UjixENM6FS0UDa", + "zAW9uE36ha13Q8KRMUyaAiJN1qM1XrKPnPry7gStAQzg35B1Gut/rWfBk6WzbuvZzrO9J1vP9lqB/eQD", + "XK12HkJObn1wK3XQIElH1u/QNPXD07fGrxAYiz0LvHh3UsRWSATXokfP3DVY7PxZ/1kR4yjk6Tgq3GpZ", + "QDQDqQoL5oXxymRRQxDenzSa08mE/fkhuNz6Q9B483pPbo03G7BbTUd+l9Zx8Wa75v8l456pXeOHoQGG", + "ErIRqemMSJgBOicKAf/0EA7AdMhSaS3LOTwnS3EvY+1sb28/fbK71Yqv7OgKG2cEDi7PoWxHUNhi8CZa", + "Ozs/RxsFhjNtOkADgNdm1qz07zNk67cOygppf3Ow7eOShoM75xrb9jxuJPk7a5rZSVmiQ0ZwZrbVdrmX", + "2tvbgyc7u093221j63odievlEsblyxjyWBT44sqvgTb55uAUQTbqBAdlv4kLT7rRqNSNRgUVDAzy+A0G", + "9vTJ3u7O9tZmO8gxX8SDBdMrbdiy7PJsOg9TeFbDQ4q66O02nRY+dcow2BkJIkzjg8DF91dOH4MwPhLm", + "tXwR2hwM1ttfO7hafNvKcZS5g0x2iFENuEApy+pa9Fdfd36RW8tmqW2Oh9VS3ZcTwjT1LDaOqWt1C1Im", + "gswpT+UXaIgrk7A5iTgXN/q2yWA5IzKNlLlapBK9O/kBZIrmNSQVSco2lOXGJQhCt5zcjfZziUX8TN5E", + "rFar0Wbpl02427Bru8vQHErSoBG3K9SSK2WrIw8PcRSkUMkFZ+upZwUAVJCPniTRwgSWRxHnDAUzzOAC", + "QhTgdtCMR2HfG4apn4wm3ogFfoUibhCHLwlJbJETMwj9mVZh6JygtUI+PDKsVClGuRsbIWPLWJS5cTf2", + "V9XD0pcpleVha3pixQtguOaTkssx4lMJRqGCkPl+FYM9wcJEwmNmivbMY2NLeqJ9PUOsCHPfiWpOUj6x", + "Bq5VOSDL2VASB4JLiUhEp1Ag5t1JJXl2ScJVlkK7OpqwPNgWrGsuCz1HmYFial3by3c+epJJPueEBB6G", + "hLUlcXrOORljlkLZkwIjk+uECsMe7WLxZlyqUQaKdMPBSjWCWgapIDlUW5bynfmD3Dvec9GJttuQywa9", + "3urrGlf5m2oaYLNM9VLUT61uxoM+Nq7DQi1Fosqhrao4RjdBSsvB76mEVmkBMwutQcJFQSwV8M/W2wSg", + "+E1W3U/NWrUlG1/tDM7bYoothxA7xWp2zCbcAzxxg1tJ54m2oZIJETGFah4oJIyS0NmS2fWkdXVBtnIk", + "CQpTYiln9FOBLcGx2d4AHsGcj4yyaUXWVzts4x42Y1he6gD6tS+2CSWS/mzONyIFWplYQIlwntfZKsCS", + "ypH/OqvesCDTNMICWTTANkOWizii7LJN63IRj3lEA6Q/qN45T3gU8auRfiR/grmst5qd/mDUVBfn3AzO", + "JqWZBan0m0/hJz3L9UpKLHhiNsz3GwBX0iYyyxum/DONiIWWe8vodYHRy1jgO1uDplTthkZLSdp1WMKb", + "Sm7Lst4dn0pPYttSLcdV9CGhBWg3ak+SSlNcpMWp5BBA3RXg7W50ylkLn4eLcWjkdQUVA40JJJ24qdWl", + "Rgux2GYq3roGqZyhP/i47BBtG1LrqZa1xnJ8BkEm3uByWNGlDmnzRo0mhdW9CQACiFU9UfjohrgCq+qK", + "5bFLTfLkrFZia0Ysyaiboym31aJ8hIutyHLnba/tk+irxdCayrcsCkX9JBpzIQDxOIvXB4pqsdhHfy/Y", + "YhYpHJzbP0jEr5irT7YYsmKL2lJKpSsNWMELwlJecREuGZx7BUqWwK0lUvySsKJwyJrxGlumoZH5qh5z", + "ZuqXwlNr0aFSSVFULRm6vL9UEuFXu7IpZa+0CgYpMF9B3TeYHgBwAaXi7F9GaGagFi0gLPLm/8c1mf90", + "mjVe/q3yWgGmwqHGHhhHqNepGZikk0r4TPluamXwB2Q1LQMnqXvn0ZoL+HX1Nspna6HuRauTp11iVTW0", + "3Y1mQ5Kg3PvO090ney0Lj3zW9ZcBY/rSl13zeMklV8NKnbS5SXm6+/TZs+2d3WdbN7qzcFkQDevTlAlR", + "XB+0Rq6V3qzRv//5r3cnlXuUXYgYHtxoUCYPwj+khlyI8oDenfz7n/9yo7r1gHyCpg743HAT3hj3EhVX", + "0l29ly/F2l07LbGYD0pmN87EDFojkwkBx+LI0K2XD6aC6tBOr8QJDqhaeAQ5vjKx2dkrFeDiNhcs5cH6", + "lEjTtgW51JJLpuM8h3DNdY7+am5bK7zwtHX9IpmOm252X1d7Nfe6+T1AMWqgxaV9Xmm/7jLO5nOFZSn8", + "WP8dgMbh8qHquSHmjeUgqtXAfYgLsWW6CsF1PvDtioZmPyouf2U5CzeBJUdJleLvl+zD5i14Iz+q50T2", + "uFGD1YmgFflgD8DbfTUaFyuLLS3dVipDlp+6N++3Ra5rHfY+O8Fu3l8hve8mH1YhXoEf7RgsyfO2uyWW", + "aOCmQuaGxyXFI9LLQt9sWgeSqblj03veooZ78g2DSz6ZlKFLd5uhrgHFBVKTXC9YKRInqovItfMCVHGS", + "DWTLsLMrhx1tAgw7m/GwU7kI8ib7xfh6ZDsoQ3UMlmFPZzU1q4OUbgbjiAeXpmgW1GLuowGKCWYSpQw2", + "f+WeanOw/L6l20kKa5MhPRMTNFQTWzCmMZnhOYUCA/aWYloKbSTXVEkIwYR29lHIDXhPqWKonaF+zaTi", + "7eeThkMHs4VtWDeo3+PMxYjm74LLbAJ1StkHInjX5p5rif369UnXhARAMJ8ZWCli0E3UjEALyKyLClp+", + "/rs/IncckRGMu4q+HtfpWMyghrtKQSRR0sIx5+xQYQIU8JSpKix73M6QKydB1Y+klEH4nA2oAJgt27st", + "Nx+SAHakrO/FMqPfgrkrkfiW0r5Q/G0fC8OmgNsZ/+3rmb1irQ7AuAcKxX5NO8VIaXPvNpKK2+pQ2a4e", + "keuAkLCK3+h/pW30uf3SG33+ClvIl6wOr30bIojrs+vfXToSjLWJ2sUoecZZD8Am3JJaYAiD9GahR8qM", + "VkKSLgAwjHxomb4X2uQHk+vltP6NXCuAuw7TyGCY+VnXiip7GK2i+K3z8Jo2NBery9bfQRUyE8F9qzpk", + "Nvj7IUqR2bfupPxYbXXOiXLvnls2ai5CX6rbUbokciH07pVy1IphpS6yBzzajNcrLLgz83tFLMZqy4xC", + "hmMySgSZ0OslzGNeMIZxGYQj30hZToCBi1yL8TXaeYKCGRayMnZGpzMVLcohLTseaJzPqtEniCLMOQrb", + "rHy+mu7DevyYXc5i6z7l+LwAZFOrUGFV0tEyGOTD/P7KRugkeAFenMZrtyfbO4PB9tbgVjjIblg3INdh", + "/omtaFdupylJrfCdvTovxX0WW8hSgutlUq8EhczijExSCYLjfUhlSXBAUEQmgHmWladefVdX7Xr54K1C", + "ZeFTMv53C2XXzd1qlyugZF1ZCGk3jY67risj5hSfr7hibBAzQQ0hzZPFtt0b7L3Z3N7f3dvf3LwL7OKM", + "SE35E08+bF49ibbwZCd6unjy5+bsyXQr3vbaYZfUFHppw6u/6ncb41byQ7KMvFMSaWjNziEholr/tlo3", + "WpKIMtKTWc7R6sS/JbLA3Giv3P838/ObGSzVHc7LkyyqEFjlxClx1sOgRdnJLL27qM7m+Gj5LG6V01Md", + "iJ/fqkMB9mo3GCg4sNn5TByBlLU8ht4WXmx9EC3NM1t1FPnurGGne1e5geI+9i7JydKGW3aA1w85j+90", + "ygVVs3j5aZG9lqFCQyTyB6nCMjpRHx1PGRS/Lv6cBZ4VjSj9cafbiT7slPeM/b09TpUFlM0Y0C51USto", + "EZgFtdWXUwFeyQ0PYWLDta2ux/zTZm/zGYRHRx92fhr0nlVDA4BaRfJturdLvw7a0LBY0c1VAtp8dqMY", + "ZkfPZRz0K/XVI8vPZQs1a3k8LzXsjg6Xwlpa4PxxbY0ruDONCujnanr2cBsVlaaQRHjhgxovOGplxXos", + "Mhkakyllso3fdnuQOW5342Gnjw4s3jPYsnlh+VLzUFK8wCc0jklItY5pTP/mnICtlr64qi1xs1IT7iuP", + "ttb3q2vPVoMOrEphWnVM9j8jw/WzrN92Fu8yPAzwqzkTFRCv4MUuohOEWaXeJGVzHNHQpqZDqiFEgO07", + "WLGcZa0MkLke6PwkXTTlCuVJ6S39bSlr9gtm4yfX4G9dgkJhGGLri0CMZHBXdJn4Oj5CieBhGuQZmREM", + "OsfQEGkFUGyJkr86yPUu/RuQ6jzhAq32bzQ5NNr5J5vWu+Kb1AzbvNSbg9VLfSdOkW4nTcLVMsy81E6C", + "3QiIe0VSn8dFUyZ7RRMsTOZ9C4l+VqRg3eY1vuRAq0Rp4i5YNE/VOclz3QJXDL5I2SMSEX1M1RtBPArz", + "vAMqcym6WqRu7j2dNV1xwo1UfSC/EpJoWwUQhaC/GLOFd2CuimR2lqwNXIVYaS68eqb6jKVWeXBPVmpi", + "jUtVdOE2IeIbKV/xeRsEksuC+ztDpC6qZnVMESfwS0raWTOiu/3Shb01+o/vwi33kEraa3v1UAEZdRC4", + "WWyw69/kVAJIMYsq5/WO73rexxZvrGXcBJxazassep0Pev8wXmY06u9v/PS3/7f3/q9eb3PFbpZE9EIy", + "gUCjS7LomVoy2kbvl2FDAcheK9NTyyoEx+BDAkhuuxmL490dZEJj8RuOa1OACK1CIZjNlRP621+a45sK", + "ZHwLcnIly352nYe7qIepuDuO1mIiptoupC5NCrKDhwyyvS7JQqJCeSmr0jhG/UFmnxSC29GFUQP7hM0v", + "0JhCvT45ZNqqxUFAEm1N2Iol1BSd5iB9BMFRsR1b5sqlUtsLSRNPQNC7kxrm7Ou3b56/fvvb0ej16Yvf", + "Do5Hv774XwjxuOqZHsKe5r2d3T1barpIyU3PEn8GKv5nwb762M2AR3r4C9IWoX63R2GmEkAHXLhB4WW0", + "RuJELVztSpf9uH4zMMuDrEFvsNsXLlEyePYlygi+XVo3cM6jntaoG6DkvQ5MQwtvsDY0ZYLgO01+7unY", + "ozZab+KUTrHHte2ttP8lyv25Aa1Me6mtf2PdI3/o/FEVZ99IA0OqCi58xS6VqtccWR9rRWqU1zQvx2uk", + "zCYg0kI4VznbMGZqw5bl9IEehBzQnpelnOa7zEHc9eCj1ZmUS1X5wswKI2lemxOnsVZ06iUEOtWkuZoR", + "QQoLAR/k+OU3JJlNBWkBpWHKmiVE5GGSLo9EK0JwwynRWuZscCTIUkbrHtjl+PQn+DrrAbz3WNauvGAe", + "eQGZzZfPAUv8zBXNoxPXBAyjYk/4kbPLXLSMJo6r6otR5Kr6vM373o1nZdUS6de0tyrMmfdRYk0fP/4d", + "U/UzF2CBNANX3DkAN1g3IREA5FWF126FTU1jEo54qpbvf1uT26JWZIUV88KcztrCwMRBKSezSRY4aIV8", + "DHVKa3KQIBVULbR5bpXhMWTeuWqYQEjoCH7OO4YKhJ8+gZ9y4slReEkYETSA+o56P8aYgZKO3p0UynyZ", + "im810E1QL18fHlsL1+G2gsVCFbCeC/U7OD3udDtzIoyV1xn0t/sD2MwJYTihnf3Odn+zP+iAIj+DKW5A", + "TW6bBGuTRjNb6Ti0mtBz95L+UuCYKPjid086N4S62ddB68XTgt2SYCqs4ZJEkGZuGIbqrwGP3R2o++ZU", + "7hqyt3bTSbWwCRckeW0X9z0olbB3YJpbg4FFp1b2+IV0EROjvvGHDVjM+22l1VkSeeDJa5aF0y0z0n/q", + "dnYGmzca07KhwN71dfyWYZs3SsAg3L0hIW7V6TEzmWA2E9dG4BR3HDBSca/9/l6vmUzjGIuFI1iRWgmX", + "TYoxkQi7dw1orpIo0KICiqj00WtGbCF6rBA2wbIiZVCc1X2oObS8C0zbbpEzpJnnPFx8MRKW+nBm8aey", + "ONPb5VONn78c72RsXF9I+8jBJBuuvQcGeo6zysIPtlN2Bs/uvtNDziYRDRTqZQxsQ2CphCiTCECfHYAM", + "F+jPlCuMsgjyR7Slrc46ztitmx9FGx9p+Mls74j4PK+nRMSYmXh8886KTV/bzsYLnm/npaeaY3yozwAn", + "lUNSMQcVKHLlLVo8tqrKYP042vEk/ds+zfTCB2T8nXvY4XayWXHJh9xyULEQpZI8pu1kb3XGuRLi1eVe", + "EvW18PzgPo8siwT/De6ix8LAL0mm4eWrVTsUNhKRMmMAezXAszxHzn73Q1n5e5M/KQRmgCtdNw01CZS5", + "ysPhoo8cTY3RrxaAkyMIzDOsHyunenhfyw7buo8dBjPOLie+H1Pfj6llu9xwi5sCbMzCLm/hg7iRB+Lb", + "8z/c2Pvw3ffQ3vfQyvPAyJX1LvzBx31kgyChsruc8TQK0ZggA7Hjwh0UFv3pB4RFMKNzMmT2tiBOI0UT", + "LCCYIUYhVthc2zY6Jpa6JbLmNnRzPRf6lhO4Cp0gyQgg3kZNIIJ50BtljIRIf2LR5HJMuFq9ZbP3vQ72", + "rMH8aERXMy7hakNSqaBKWnaaQ0atNNYxNNsfsjcWrVMTEOJ3nayRJALM0SX+H84QHjL7wY9OhLjYI4nj", + "XHJhAcBy1MALmmWpZ1PpkY5kwH3wLm8Iw0z1ZEICOqGBndYlWdgQQm+DrYrn6AG7cb47yXIE0Na6HyIM", + "io37EVaPsmfIclL5/oZB3G0QpWF+yeVQa7AY4yjyVleYRnyMo5GhzyXx3Am+hDcsUYp12d1tEuMhMTW2", + "k4WacWb+TscpU6n5eyz4lSRi2FnvDxnE/ltaO2BAwwNXUI0rTrjeZ4LHps8NM8SNj5dk8ak/ZAdhTJnj", + "CPgER5Ijcg3fQZEigGkw0quBH8xu8t+DH6ZS8bgIX+n4zgyTpypJlU1ikER1fdCNQ6Y4+ujgBD9tfMx7", + "/ASXxQSHmk8Kr5gpgW7dNGo5wnr2I3jVc91OgADDjj5Ihx3991RgpgwqY4aHiKbFJV3LIO71Jl2vUjjA", + "DCU8MeUBgKlmWLNcqQ2AB8BRhBRsJfetVtxhJRvmY9He4nEj1JvB5qpsI8rQyfPCZhrsPPXvJ0kCQXwR", + "Jf99/vo3BKeyXgPzWh4hZLIImFYYUJjC1amTaS9wMEPmogoqwg07NBx2suvccB3GmkqbMN/rwZ3iT3po", + "P5luujT8qd/XTZnryn30+0fTyr7eS0lsoCeHnU9dVHgwpWqWjrNn7/0EbULMOi8JArRmjrl1kCSYArhJ", + "4cQ3RyRmIeL2FIgWCKNcAhUDV8aUYbFYlrvmIb2lIJ+Y4LkCMT4OIVhu2NkfunC5Yac77BA2h99sTN2w", + "88lPAXtr2Vx+DM6z7HIzY6K9wWB9NZyxpa/nzrLFxcAXtgEbraKsdqJeQYv8+W3dD/xH25/Z1Q9muvMc", + "DccY/s75/ggvIAoae9ES9VxBVNRuzAISObV7taPn/i8P9GIFJIrum0Efij2z67EMbv1RsSMsVr6Nlrrv", + "H5jjBvd1qJTc9g/Dv4/Of+7xnlvfOZm7UGd/sQmAPbGmNDIvIyzROYypd66N7xfwa9/+19l+AON3EfHp", + "xb4x3VHEpyiizIagFwKVtXpgaQkfGeST7DsLhOIqfa0ZTeLf//wXDIqy6b//+S8LJ/7vf/4LtvuGQfSC", + "QsEXM4KFGhOsLvbRr4QkPRzROXGTgVKeZE7EAm0PrM8fHqFCvXKrpckhG7IzolLBCqH6puiWtA3aqwI9", + "H8pSIi1yjH6RTmxFEBPb6PHbuL1sSHmvO7rrQeCDGRQmoE9FxwMAX0ZNtWRriXb8LlMz55LTtBqmWQvW", + "Wy1fFLlWhnt7ZoA3FDBAYt++gwd20mjt/PzFeh+BtWW4Aqq+gO2QN2PNiP53mbRaJhmJUhYoQGUjmwyc", + "0nKn/5F9p53X37b4Lbn9bSGwG/j9jfMHQBXdCny/A2hxB+Cnm7sP8Dnljxxe2N0FC5ouHihW0PFenebm", + "SYFkD+EMQGsOiAEcqlyg08NjhMNQECnX/7NdBXqmhkvzowNxBkUBHuLW2o6FCwtRZU21MoM8FnFwZkeN", + "sJtXtb5i8XzbKJWKaDzpsqoR+ZF396dHpdObHCN5Dcic176fJCvj9KgMuP62wC29ACdASKe+ZPu0yEWr", + "HFImAjA7cpaqS1Y8Hx+5DXl/rinbdcqqZ8M9CMWjikB8QEFYztIsVk19TNz8NltFh426xHP1dbHm4P60", + "oPv2YvnY/DG5scIK2bQUNHgCjQfoS6IMikDnDhfa9uCZ+DkRble7Itcw62xa5lNk4BBgQnA1v9z2PTav", + "tDN9TXvfkuUL5LmJxmJJ/l1FaWHs5rRaZuAe2xKgd2ffQg83Mm+/3I23ZTAPkSHsZuw81kKREK1huWDB", + "+vdL7y/O0SYkKjdiBcpK86MkwgqiIwGIJbOz9Ni27kGvy+qlCqyIjRt6jMl4p2kUuauZOREKvT48NiKg", + "eFhtfIRIstVGiBMLS8+tt2eveoQFHEIHs7A3v7Znn3xhU8RwVinB7v75+REmmVF38DapYp+x/ibCE5mg", + "1D7l/7X1c0THAovFf239jKOEMvJf2wcRVkSq9TtjlsF9nSH3bRo8YubTlgEtEw1EE5sCdOAKVTp7q6U2", + "7d7/phRqM+kbqdQZXb9r1W206iK5lirWdinuVLU2fTzQ3VHGbD5qw6PvOBP34I60HFnAmSjdz+RIEzMu", + "FTx6fEmHNtKTZhxXPDZa+tXzDbn0+HCse3zUBUJCZVFANrc5PffkZXfjuHfl1vZ7/y72g3hMpylPZTFd", + "KMYqmBFpU+kiUhbAj03tzo/nRsX7K+bSwX0eHfeuV3/n+zvS+KsLaoS3uSpbpfO7t9rq/PZ9rfMbmEGb", + "bmjh17uuNMd6Q/SjAxpsy8YlPMZ6VKZvXD5bBL3VhkpuLiCwIPaH7P9o++N3RXD8/ieX15QOBlt78Dth", + "8/c/udQmduJYhTCoGQ4prge/HcH95BQQGqHYUp5FWR2Hqd0KrOfgpf/jDKT8ira9heS48LuF1MpCKpBr", + "uYVk1+JuTaQyRP2920iO33wEt0C/36aV9BVfPNy7BSfTyYQGlDAA+odsUVmLtDOW3PebkVtmCTJ701cI", + "0ylpIq3NyExqrdDQ89qi9x6idZwXU7lv69GVMX2c2Q48sXUBrb2WawvNBtvXxg+D+z297t9Qe8wsZiyi", + "OukSrXR7ynWYQjVxqiC8NAf4gfhdJIxZk7XYR4dZXrdMk4QLJU2xG7AQTDnMmbYQfIVxyrVufMVtoKAL", + "JbI7ZFDuVD82+BQbl2RhStlQzrKqNdlMbUUYXxZduZTQg26jL6+E+usktVJC73kb28p3D6eEPpjouBd1", + "77hUUHQt2xhgcY9JtpN5lqZJP1A2XX9UscRGWGVzK8CReVStDZwq7srib8y4QSbyg7OdRjgAbDb9moEN", + "snm/Bies2BQk8woeRUQYOKgkVa5u1pBlg6OsUBfYFqm40M2PUqZodNE10TSQ0y8RZguLiTJkpc6wUiRO", + "tGCzKD8wQkESM+JKwTA9aMpTCW91keSlLhGOrvBCDpkgk4gEdm5QXFGQwCCnRVEf/cIhkRrhKabM5vbq", + "N025rR/kkF3QMCIjmwd9gahEcsaFIoyEKOZzIsv9EiwiSgRM4hBrykkU4wUAEhlsNkMfnhAD+lPKtub6", + "35iFFIpR6Z6zKe8PGUZbgwGKCWYSUUjIlXhC9Fe2DQSDKA3oR4TRzuCZ/aqybgCa6ci/pveLEGTOAzyO", + "FohoLoYTUa3DAmb7C+o66uWbUCHNemXuRVv1p7SwVLr6lGEXpSyA8omp0P/iAqXMHq+6RQE55jBPewlH", + "qMjKjtmE+DEJsKYn4+V+AIqMB0EqfIejXupCYbz/RCWzML1zIJVPOmk6INhTIaw542oGe5rDVlr/sYGr", + "cqb6Ng4Z7ybhAmFU4OvcoQA1pNkUrQF010Vecou5qo0X6z+6vaO3rxUEbvsb8KzHcj4BE/HJpLQBVx9N", + "ZgMvS1yos/C3uk8PXa3FoogLKZ4yLhUNnDCsVgP+bjy2Nh6XU9bLzRMuLou6VZl/f+bisq31de5K3D8q", + "I6w4w6/wHkAPD8BXH/46AJzRxlDRTHPvBlqVv7JdCkoXVdLFGXMUcTbVuyh3it+7175i0QVRCmq5M+Wc", + "E0QbISP7oynXqCdji+GBhz+wrT60LNK938Nd0G9cIRonEYkJlHPsGWbTi51p1abaMpVoltXRu5ms1Luq", + "mJRrbEFprv+7Th0CvnILtgbae325vEI14tPVQFxZ5w51yoPENWSmGjRxpaMvUCaDtUJrYK/R1YwGM0Dl", + "ArtVt29Au3CSXGSApOv76CVs5CIuK3S+ZsCuNa9JHhEDtjWP44v9esHCdycn8JEB5DKlCS/2kStSmJ0f", + "Ur9VRNnSs4iwVOg3ix22lhnjsKIXCmt7M5vfusXfygFjh8yHxcXIlW2QTtBFAZbrogGXy8nbV3z6YMpY", + "txnm28xFcWRNR+BNwsJOU4wFjfyIXJuDgQ99tiU6mBnGHYOD1Qbzik8ziPESK+Mkacu+dpjAxfM4XsLD", + "aC2XIEiqkKfqb1KFRAj42HJ3E3OjNRzY8jL4UjMqM1LJbex1YD9vJJHB/PWSSgvVTrdDWBp39n+3/5rH", + "cafbseMpYAXfQLlfgbJWbbAe8aJXpgCl9l0tvwlIWlnYF1DSKieHNaebNfIz88I3f7PofHYPyIagH1Sc", + "uF+TCloYb9nhwziSDCdyxtXjwmWyrqaK1tbsqnGz7OnhhamrgdEmhOPcfnruvvwKrN9VkR1uzMhN995D", + "POojeMyZsLI2mwkXVTCfVbEfXz0jfbklqU21DYd8582b+/laMWbiq8bvliY0NZFwqniMFQ2gHkcw41wW", + "2H5MZnhOub0qdXdWGWeCc8PYmTaE/kKz6oV1BF9YRX7fOq0QLj6yffThcxt47//CPcq/+Llgl2cSv+uU", + "b8CshoLBgpIJSnAqidar0pggU4rfFmAhOJihACcqFQRqSxEUU0bjNC64GrThJOY4QlSii834oovGqUIR", + "FlOwi8xDE04vSMDjmLCQgIdsyGYEz6k26gSKsCIsWPQkgZqUc5JX+tdGvo3CMTWtBNEcSDnropgoHGKF", + "QdW40Dt+ZLJ4LrIylcawZuQ654ZwyETKfjQ427rZCzfQC0SkwuOIyllWzizAIWGBF8T6/OsWY1/eG3xO", + "VHWiDxSXcytZ+pCBOkWvpxvO1xHD88iCkbmwy9hGzC9RemWzEVlOf3Bs9J+5pc1c3Rwf6IonI/GyXfx1", + "3O1kTPfV3O88/AUOFyhMTXeFXQls/q3eymQCpRjuBKmVZhlvezWT1W3KyHwjmbfx0f15fAtv2lciCbuN", + "hn1ThZB80l+DyLVUvZXMfSA3ovUlFbxiDyiCXUzVg6lPXBSk3GNxd1qBbbZmJreL0kkJDNYXZ9/FdlVs", + "25CD24pt55utXaoXBDllPYjS9Etw68ZtFNXWdfAfmgtSmV1BZD64iMzvDu5NLB5ngtCIxgQvIo7DbyFM", + "d8kNTsCFMPgPgCjxmPBHC17DYoA++Oa6mYToutzKdycn601SQqilMkKoRywhyjX6g9hXRH9OhKChKw5+", + "eHJkA2apRCJlffQ6plCx+5KQJM8pASCPvp6fQ8KolzkuQV50O4QpsUg4ZWrlKPJX72Ywn25VHPme5aSF", + "iv5+Id36Qho8+49PnIGUgawJM4HllqnCqjEU0IXGUWZqn2u9DI95qlvXMkiTSa/nFE7BCY2IXEhFYhMX", + "OEkj2G5QdsBWpbTfmVXuQlSs3jkmYS0hIqZSUs7kkNlsjYQI3bf+XLdfCHHyXggonMnXUyMkv47wOT0Y", + "EzGGVRPVALMIasJ39jsbOEk2QqxwQ4iWHd5nDOlniIdDchGPeUQDFFF2KdFaRC+NeYLmEkX6j/WlAXUj", + "+O5L19y8/c7SlD5mE+4tS2Z4NmPmbyqvyoo1dzH56MTaS1LcLE7+wEL7xZpcKdcEwVFP0ZhkyDUoVTSi", + "H4yo041QqWhgkn5yyIJ3JzlqwZCdECX0OxiSy6KIBMo5bDYSwYONYToYbAcJBfizbQKDA4HX/DiGHg9P", + "35pEUBJzsegOmf4HNPzm4NTc7k6w9SYUBsqIuuLiEh1vvF4RYnwOZPoPjtEzE1yKHeBd8O9XgjdHBGnc", + "Q7Jhi/JkmanEk28+iNRqcN/9Co/TrwCQTNls1qYCB6AUy1mqQn7F/D6EOY/SWP/D/HG8CthL4WD2Dl79", + "arRdM5yV3bgJPopNaecUElM28UEuPQzBHmvMqiacmwIoMaVoQO8pcKC+Re7+8u77Ih2/wutOS1FXkvSr", + "2Vv3ffLZMTiMiyI9Hss2N5zmZqL4cu/TFabN3qfnEQ8upQVDKboNtd0GAOP6xxwQ2l4RgpoAuZnIgggh", + "cp1QAchvFQekwdyRCCNFREwZjjZgzqYRgLZ2Xiw85xRSpIOIQpIaDQG1KAJ0uqsZYUjPBhxVroHCja60", + "paWK7xQvIxVHYxLwmDi473Wf6fZ3TNXPXJSxu78WufimQH89Hz1VPc8VcOXNPX4WfPkJvoZQ6TC1F8pu", + "RGsvef6jcQV1EazNsLM9kMNOFw07W/Gwo1fgEIMLFSu0i2LKUkVkHx0Z/xYkwe4NkCQBZ6F0qOPOg7c9", + "kE0psYYtG/Ir9+C7+1R7LFcBKc9sJz7xoN9D+ntI2kFrxQ1n92TYhU0XIp4q4+63+8q+FRIF7pH1e7+r", + "LeyR77Z9G0n+d7t9SzIKVlmLy8LSG8mepHJGml1ur0wln1SNAc3aXPvob9AffCy7iJEr4w0XUvVrck9/", + "fWo6uA+kfd3VTVD27dy/Q+y3gNjPaeWHSzQBlvpIdtxhMBPJtUGExS7t3fIQWBKA3cADHKHXh8dDFmhR", + "ZMD9BIk5SCcLCG5O4YO/n6MXh2dddASVHtEv6Xi9j16zaOEqWZs7miEzmpgRXgFmaGy4loS+49mMHbjn", + "LoPFdQcPVJTY7AzPzYpbKxck3u3MCA5BI/nYecVNZx6M4LNXegPp9bNfZsveWap8dM6IEovewUQRUW/2", + "xOZJsQy1wh7SDgTOKm4GelJ3KB32Wd6n0Q0MOMX2VseDVfHpe9WDu68Qej+3ZCZOxNSbG6eA9ckgyQCH", + "i8cVyyRnKBOOPhFYPK6zugFNWcJWli01MKDLpsjvr8jlvlR2lZDg/1N3F8z00V40JaV10kyc1RtZedPr", + "koNnBpDYXlQFOMEBVYsuwlFkzyh7EmQRKb1M/R0Lgi9DfsX6Q3aWVTqxCb3o8PRt113UopDKS9OCvYvt", + "o9dzImQ6zgaHYKOZW2OgOQmHTHEU4ChII61ukMmEBJCLCwVMZMNdbjaUzh3unbwTb7WVQlR7+uiKvPl5", + "AlYvZ4sqx22Ypd4QJIgwjZvhv62iBgGHEGow1o1yhiibRDakKhBcSmSb6pGITuk4sgFCso/ezAiSOCZD", + "lkSYMSJQKk1UvB56LxFEytQkeOsGACbXcFQX5dB+ieDKhiZEnAtpogk0h787QVKRZAmbnZmWT2DOd6Tb", + "msZtTw/kpK6ModkVYl9BekEMpxiCaz5KIxfAeK+h6GZAD60lPpaN/0bQ6ZQIvSuwEbImHM9sa0dOs+lL", + "GcuNBR/Ps7faFXzMWi1kJRYy9pZCs41ytOuwc7OoP0/nl7QRvc8+ulkW8a/6o5Z9l7NV/YOwjz5zlt9K", + "Hf3zQpJgWwdWzuGPzZ1UGHlpq5YSbVfDarXOrL3LTNfW+FkPBpv1mNGycCl9tsng/foYYXC/KA/3XRLt", + "cfNWCe2qZJs2pPyvxrP/KjjwboDsHxjl5BZA9l9V3j0gjT8c/ol3oz5UHn3p7tlVm/3msejvKn3eANID", + "HFtT+ryRejZ4damh9M6+085Msi1+Sxq8jXe8gf7uyP7d6m9hMhSIteoKWjM8iRO1cAFt9q4yDzqT9APp", + "N1wEZ3Grd3cVfIuQzi/HHo5PGwM6v83i8A8SM2qL91GJjo88VdcfGcZgcc+VDpYNfer0sAhmdE6ane7l", + "HWxJlAjSS3gClyuhIZilhzvLFBb96Qdkm7eYq/ZfUP0RwPJJiEIqSKCihanEqSWC6eMHiQTXlgA852LR", + "HCVitsjPgscHdjYrzkO7p6wzLI8zjBe9ECvcmztps8SF9hnRnS6eUgs8RBl6+RytkWslTI0JNNGWD6KT", + "jKSm3L4EnlwvDnhz0ODZpB/IaDpuM8ol1UJe22osKEil4rFb++MjtAbVx6aE6bXQqv4ENNlE8DkNSVga", + "Y2fOI0PVzQaC3tTvqpWKrHScMy7M4B5Eh2lzIE0/0KQsFrKQmDFlGAa3si5HeU+ZJH7dH6bMBeDYNXKj", + "+H6EWctvzRk7mhOhEqYlouLcQDyvfz/mHvMxV0yGcmda6bRz4TnLndft8qNapi3dReGHLHfuft3W776e", + "lB4qH2U2j3WdzzODtMlt/nWx4OD+zof7dpe/e8QpoC+JM74LrnJoQLfoY5hXENMdkjmJeBJDRXJ4t9Pt", + "pCLq7HdmSiX7GxsQ+z3jUu3vPHuy3fn0/tP/HwAA//9KWe6LIc8BAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 78788412e..054e3744e 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -1,7 +1,10 @@ package resources import ( + "context" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/logger" ) // GPUResourceStatus represents the GPU resource status for the API response. @@ -16,31 +19,26 @@ type GPUResourceStatus struct { // GetGPUStatus returns the current GPU resource status. // Returns nil if no GPU is available or the mode is "none". -func GetGPUStatus() *GPUResourceStatus { - mode := devices.DetectHostGPUMode() - if mode == devices.GPUModeNone { +func GetGPUStatus(ctx context.Context) *GPUResourceStatus { + framework, vfs, err := devices.DiscoverVGPU() + if err != nil { + // Only report passthrough once vGPU discovery confirms no vGPU + // framework. On a vGPU host a transient probe failure would otherwise + // expose the PFs/VFs as available passthrough slots while active vGPU + // assignments exist. + logger.FromContext(ctx).WarnContext(ctx, "failed to discover vGPU state", "error", err) return nil } - - switch mode { - case devices.GPUModeVGPU: - return getVGPUStatus() - case devices.GPUModePassthrough: - return getPassthroughStatus() - default: - return nil + if framework != devices.VGPUFrameworkNone { + return getVGPUStatus(ctx, framework, vfs) } + return getPassthroughStatus() } -// getVGPUStatus returns GPU status for vGPU mode (SR-IOV + mdev). -func getVGPUStatus() *GPUResourceStatus { - vfs, err := devices.DiscoverVFs() - if err != nil || len(vfs) == 0 { - return nil - } - - // Count used VFs (those with mdevs) +// getVGPUStatus returns GPU status for vGPU mode (SR-IOV). +func getVGPUStatus(ctx context.Context, framework devices.VGPUFramework, vfs []devices.VirtualFunction) *GPUResourceStatus { usedSlots := 0 + // Count used VFs (those with a vGPU assigned) for _, vf := range vfs { if vf.Allocated { usedSlots++ @@ -48,8 +46,9 @@ func getVGPUStatus() *GPUResourceStatus { } // Get available profiles (reuse VFs to avoid redundant discovery) - profiles, err := devices.ListGPUProfilesWithVFs(vfs) + profiles, err := devices.ListGPUProfilesWithVFs(framework, vfs) if err != nil { + logger.FromContext(ctx).WarnContext(ctx, "failed to list vGPU profiles; reporting none", "framework", framework, "error", err) profiles = nil } diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 32f584193..d69b1e2a4 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -199,7 +199,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { gpuProfileSlots, err := meter.Int64ObservableGauge( "hypeman_resources_gpu_profile_slots", - metric.WithDescription("Available GPU slots by vGPU profile"), + metric.WithDescription("Virtual functions able to create each vGPU profile (best-effort snapshot)"), ) if err != nil { return err diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index ab6c3e4dd..bef0740dc 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -198,7 +198,7 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { mgr, _, _ := monitoringTestManager(t) originalProvider := currentGPUStatusProvider() - setGPUStatusProvider(func() *GPUResourceStatus { + setGPUStatusProvider(func(context.Context) *GPUResourceStatus { return &GPUResourceStatus{ Mode: "vgpu", TotalSlots: 8, diff --git a/lib/resources/resource.go b/lib/resources/resource.go index caaf5ba50..86f644bda 100644 --- a/lib/resources/resource.go +++ b/lib/resources/resource.go @@ -37,13 +37,13 @@ var ( gpuStatusProvider = GetGPUStatus ) -func currentGPUStatusProvider() func() *GPUResourceStatus { +func currentGPUStatusProvider() func(context.Context) *GPUResourceStatus { gpuStatusProviderMu.RLock() defer gpuStatusProviderMu.RUnlock() return gpuStatusProvider } -func setGPUStatusProvider(fn func() *GPUResourceStatus) { +func setGPUStatusProvider(fn func(context.Context) *GPUResourceStatus) { if fn == nil { fn = GetGPUStatus } @@ -427,7 +427,7 @@ func (m *Manager) GetFullStatus(ctx context.Context) (*FullResourceStatus, error } // Get GPU status - gpuStatus := currentGPUStatusProvider()() + gpuStatus := currentGPUStatusProvider()(ctx) return &FullResourceStatus{ CPU: *cpuStatus, @@ -691,7 +691,7 @@ func (m *Manager) validateAllocationLocked(ctx context.Context, excludeID string // Check GPU if needed if req.GPUSlots > 0 { - gpuStatus := currentGPUStatusProvider()() + gpuStatus := currentGPUStatusProvider()(ctx) if gpuStatus == nil { return fmt.Errorf("insufficient GPU: no GPU available on this host") } diff --git a/openapi.yaml b/openapi.yaml index 2b3a0f41f..ed71ba5ee 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1583,7 +1583,7 @@ components: example: 1024 available: type: integer - description: Number of instances that can be created with this profile + description: "Number of virtual functions currently able to create this profile. Best-effort: creating an instance may reduce availability on sibling functions sharing GPU framebuffer." example: 59 PassthroughDevice: