From e7f49f185d670587d89407f78de9b1ef212ae751 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 28 Jul 2026 21:29:10 +0000 Subject: [PATCH 01/18] Generalize vGPU device lifecycle --- cmd/api/main.go | 8 ++-- lib/devices/mdev_darwin.go | 12 ++++++ lib/devices/mdev_linux.go | 29 ++++--------- lib/devices/types.go | 16 ++++++- lib/devices/vgpu_linux.go | 57 ++++++++++++++++++++++++ lib/hypervisor/config.go | 3 +- lib/hypervisor/qemu/config.go | 4 ++ lib/hypervisor/qemu/config_test.go | 20 +++++++++ lib/instances/create.go | 69 ++++++++++++++---------------- lib/instances/create_mdev_test.go | 8 ++-- lib/instances/delete.go | 10 ++--- lib/instances/start.go | 19 +++----- lib/instances/stop.go | 14 +++--- lib/instances/types.go | 7 ++- lib/instances/vgpu.go | 29 +++++++++++++ lib/instances/vgpu_test.go | 40 +++++++++++++++++ lib/resources/gpu.go | 5 +-- 17 files changed, 252 insertions(+), 98 deletions(-) create mode 100644 lib/devices/vgpu_linux.go create mode 100644 lib/instances/vgpu.go create mode 100644 lib/instances/vgpu_test.go diff --git a/cmd/api/main.go b/cmd/api/main.go index 98d7be720..345fd0120 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -362,11 +362,9 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - // Reconcile mdev devices (clears orphaned vGPUs from previous runs) - logger.Info("Reconciling mdev devices...") - if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil { - // Log but don't fail - mdev cleanup is best-effort - logger.Warn("failed to reconcile mdev devices", "error", err) + logger.Info("Reconciling vGPU devices...") + if err := devices.ReconcileVGPUs(app.Ctx); err != nil { + logger.Warn("failed to reconcile vGPU devices", "error", err) } // Wire up resource validator for aggregate limit checking diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 22dd3435a..3efd19489 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -30,6 +30,10 @@ func ListMdevDevices() ([]MdevDevice, error) { return []MdevDevice{}, nil } +func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { + return nil, ErrVGPUNotSupportedOnMacOS +} + // CreateMdev returns an error on macOS as mdev is not supported. func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevice, error) { return nil, ErrVGPUNotSupportedOnMacOS @@ -45,6 +49,14 @@ func IsMdevInUse(mdevUUID string) bool { return false } +func DestroyVGPU(ctx context.Context, framework VGPUFramework, devicePath, mdevUUID string) error { + return nil +} + +func ReconcileVGPUs(ctx context.Context) error { + return nil +} + // ReconcileMdevs is a no-op on macOS. func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { return nil diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 21335ca23..5d1ff6ac7 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -89,10 +89,10 @@ func getCachedProfiles(firstVF string) []profileMetadata { return cachedProfiles } -// DiscoverVFs returns all SR-IOV Virtual Functions available for vGPU. +// discoverMdevVFs returns all SR-IOV Virtual Functions available for mdev vGPU. // These are discovered by scanning /sys/class/mdev_bus/ which contains // VFs that can host mdev devices. -func DiscoverVFs() ([]VirtualFunction, error) { +func discoverMdevVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(mdevBusPath) if err != nil { if os.IsNotExist(err) { @@ -126,7 +126,7 @@ func DiscoverVFs() ([]VirtualFunction, error) { vfs = append(vfs, VirtualFunction{ PCIAddress: vfAddr, ParentGPU: parentGPU, - HasMdev: hasMdev, + Allocated: hasMdev, }) } @@ -135,18 +135,7 @@ func DiscoverVFs() ([]VirtualFunction, error) { // ListGPUProfiles returns available vGPU profiles with availability counts. // Profiles are discovered from the first VF's mdev_supported_types directory. -func ListGPUProfiles() ([]GPUProfile, error) { - vfs, err := DiscoverVFs() - if err != nil { - return nil, err - } - return ListGPUProfilesWithVFs(vfs) -} - -// ListGPUProfilesWithVFs returns available vGPU profiles using pre-discovered VFs. -// This avoids redundant VF discovery when the caller already has the list. -// Uses parallel sysfs reads for fast availability counting. -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +func listMdevGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { if len(vfs) == 0 { return nil, nil } @@ -253,7 +242,7 @@ func countAvailableVFsForProfilesParallel(vfs []VirtualFunction, profiles []prof // Group free VFs by parent GPU (done once, shared by all goroutines) freeVFsByParent := make(map[string][]VirtualFunction) for _, vf := range vfs { - if vf.HasMdev { + if vf.Allocated { continue } freeVFsByParent[vf.ParentGPU] = append(freeVFsByParent[vf.ParentGPU], vf) @@ -305,7 +294,7 @@ func countAvailableForSingleProfile(freeVFsByParent map[string][]VirtualFunction // findProfileType finds the internal type name (e.g., "nvidia-556") for a profile name (e.g., "L40S-1Q") func findProfileType(profileName string) (string, error) { - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil || len(vfs) == 0 { return "", fmt.Errorf("no VFs available") } @@ -453,7 +442,7 @@ func selectLeastLoadedVF(ctx context.Context, vfs []VirtualFunction, profileType allGPUs := make(map[string]bool) for _, vf := range vfs { allGPUs[vf.ParentGPU] = true - if !vf.HasMdev { + if !vf.Allocated { freeVFsByGPU[vf.ParentGPU] = append(freeVFsByGPU[vf.ParentGPU], vf) } } @@ -531,7 +520,7 @@ func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevic } // Discover all VFs - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return nil, fmt.Errorf("discover VFs: %w", err) } @@ -697,7 +686,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log := logger.FromContext(ctx) _ = instanceInfos - vfs, err := DiscoverVFs() + vfs, err := discoverMdevVFs() if err != nil { return fmt.Errorf("discover managed VFs: %w", err) } diff --git a/lib/devices/types.go b/lib/devices/types.go index 57d870592..fd717d83f 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -60,7 +60,12 @@ func ValidateDeviceName(name string) bool { // GPUMode represents the host's GPU configuration mode type GPUMode string +type VGPUFramework string + const ( + VGPUFrameworkNone VGPUFramework = "" + VGPUFrameworkMdev VGPUFramework = "mdev" + // GPUModePassthrough indicates whole GPU VFIO passthrough GPUModePassthrough GPUMode = "passthrough" // GPUModeVGPU indicates SR-IOV + mdev based vGPU @@ -73,7 +78,16 @@ const ( type VirtualFunction struct { PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" - HasMdev bool `json:"has_mdev"` // true if an mdev is created on this VF + Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF +} + +type VGPUDevice struct { + Framework VGPUFramework + VFAddress string + ProfileType string + ProfileName string + SysfsPath string + MdevUUID string } // MdevDevice represents an active mediated device (vGPU instance) diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go new file mode 100644 index 000000000..b6f15c716 --- /dev/null +++ b/lib/devices/vgpu_linux.go @@ -0,0 +1,57 @@ +//go:build linux + +package devices + +import ( + "context" + "fmt" + "path/filepath" +) + +func DiscoverVFs() ([]VirtualFunction, error) { + return discoverMdevVFs() +} + +func ListGPUProfiles() ([]GPUProfile, error) { + vfs, err := DiscoverVFs() + if err != nil { + return nil, err + } + return ListGPUProfilesWithVFs(vfs) +} + +func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { + return listMdevGPUProfilesWithVFs(vfs) +} + +func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { + mdev, err := CreateMdev(ctx, profileName, instanceID) + if err != nil { + return nil, err + } + return &VGPUDevice{ + Framework: VGPUFrameworkMdev, + VFAddress: mdev.VFAddress, + ProfileType: mdev.ProfileType, + ProfileName: mdev.ProfileName, + SysfsPath: mdev.SysfsPath, + MdevUUID: mdev.UUID, + }, nil +} + +func DestroyVGPU(ctx context.Context, framework VGPUFramework, devicePath, mdevUUID string) error { + if framework != VGPUFrameworkNone && framework != VGPUFrameworkMdev { + return fmt.Errorf("unknown vGPU framework %q", framework) + } + if mdevUUID == "" { + if devicePath == "" { + return nil + } + mdevUUID = filepath.Base(devicePath) + } + return DestroyMdev(ctx, mdevUUID) +} + +func ReconcileVGPUs(ctx context.Context) error { + return ReconcileMdevs(ctx, nil) +} diff --git a/lib/hypervisor/config.go b/lib/hypervisor/config.go index 2562868da..456775868 100644 --- a/lib/hypervisor/config.go +++ b/lib/hypervisor/config.go @@ -24,7 +24,8 @@ type VMConfig struct { VsockSocket string // PCI device passthrough (GPU, etc.) - PCIDevices []string + PCIDevices []string + VGPUDevicePath string // Boot configuration KernelPath string diff --git a/lib/hypervisor/qemu/config.go b/lib/hypervisor/qemu/config.go index 7c65af299..1e59ad264 100644 --- a/lib/hypervisor/qemu/config.go +++ b/lib/hypervisor/qemu/config.go @@ -82,6 +82,10 @@ func BuildArgs(cfg hypervisor.VMConfig) []string { args = append(args, "-device", fmt.Sprintf("vhost-vsock-pci,guest-cid=%d", cfg.VsockCID)) } + if cfg.VGPUDevicePath != "" { + args = append(args, "-device", fmt.Sprintf("vfio-pci,sysfsdev=%s", cfg.VGPUDevicePath)) + } + // PCI device passthrough (GPU, mdev vGPU, etc.) for _, devicePath := range cfg.PCIDevices { var deviceArg string diff --git a/lib/hypervisor/qemu/config_test.go b/lib/hypervisor/qemu/config_test.go index f4a3e452d..e90a511c0 100644 --- a/lib/hypervisor/qemu/config_test.go +++ b/lib/hypervisor/qemu/config_test.go @@ -122,6 +122,26 @@ func TestBuildArgs_Vsock(t *testing.T) { assert.Contains(t, args, "vhost-vsock-pci,guest-cid=123") } +func TestBuildArgs_VGPU(t *testing.T) { + t.Parallel() + + for _, path := range []string{ + "/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123", + "/sys/bus/pci/devices/0000:82:00.4", + } { + path := path + t.Run(path, func(t *testing.T) { + t.Parallel() + args := BuildArgs(hypervisor.VMConfig{ + VCPUs: 1, + MemoryBytes: 512 * 1024 * 1024, + VGPUDevicePath: path, + }) + assert.Contains(t, args, "vfio-pci,sysfsdev="+path) + }) + } +} + func TestBuildArgs_PCIPassthrough(t *testing.T) { cfg := hypervisor.VMConfig{ VCPUs: 1, diff --git a/lib/instances/create.go b/lib/instances/create.go index fcd36aa15..caa517e2a 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -52,11 +52,11 @@ var systemDirectories = []string{ "/var", } -func wrapCreateMdevErr(profile string, err error) error { +func wrapCreateVGPUErr(profile string, err error) error { if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) { return fmt.Errorf("%w: %w", ErrInvalidRequest, err) } - return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err) + return fmt.Errorf("create vGPU for profile %s: %w", profile, err) } // generateVsockCID converts first 8 chars of instance ID to a unique CID @@ -260,7 +260,10 @@ func (m *manager) createInstance( // whatever devices have been attached when cleanup runs. var attachedDeviceIDs []string var resolvedDeviceIDs []string + var gpuDevice *devices.VGPUDevice var gpuProfile string + var gpuFramework devices.VGPUFramework + var gpuDevicePath string var gpuMdevUUID string // Setup cleanup stack early so device attachment errors trigger cleanup @@ -280,23 +283,20 @@ func (m *manager) createInstance( }) } - // Handle vGPU profile request - create mdev device if req.GPU != nil && req.GPU.Profile != "" { - log.InfoContext(ctx, "creating vGPU mdev", "instance_id", id, "profile", req.GPU.Profile) - mdev, err := devices.CreateMdev(ctx, req.GPU.Profile, id) + log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) + gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id) if err != nil { - log.ErrorContext(ctx, "failed to create mdev", "profile", req.GPU.Profile, "error", err) - return nil, wrapCreateMdevErr(req.GPU.Profile, err) + log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err) + return nil, wrapCreateVGPUErr(req.GPU.Profile, err) } - gpuProfile = req.GPU.Profile - gpuMdevUUID = mdev.UUID - log.InfoContext(ctx, "created vGPU mdev", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID) - - // Add mdev cleanup to stack + gpuProfile = gpuDevice.ProfileName + gpuFramework = gpuDevice.Framework + gpuDevicePath = gpuDevice.SysfsPath + gpuMdevUUID = gpuDevice.MdevUUID cu.Add(func() { - log.DebugContext(ctx, "destroying mdev on cleanup", "instance_id", id, "uuid", gpuMdevUUID) - if err := devices.DestroyMdev(ctx, gpuMdevUUID); err != nil { - log.WarnContext(ctx, "failed to destroy mdev on cleanup", "instance_id", id, "uuid", gpuMdevUUID, "error", err) + if err := devices.DestroyVGPU(ctx, gpuDevice.Framework, gpuDevice.SysfsPath, gpuDevice.MdevUUID); err != nil { + log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err) } }) } @@ -364,6 +364,8 @@ func (m *manager) createInstance( VsockSocket: vsockSocket, Devices: resolvedDeviceIDs, GPUProfile: gpuProfile, + GPUFramework: gpuFramework, + GPUDevicePath: gpuDevicePath, GPUMdevUUID: gpuMdevUUID, Entrypoint: req.Entrypoint, Cmd: req.Cmd, @@ -885,12 +887,6 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima } } - // Add vGPU mdev device if configured - if inst.GPUMdevUUID != "" { - mdevPath := filepath.Join("/sys/bus/mdev/devices", inst.GPUMdevUUID) - pciDevices = append(pciDevices, mdevPath) - } - // Build topology if available var topology *hypervisor.CPUTopology if hostTopo := calculateGuestTopology(inst.Vcpus, m.hostTopology); hostTopo != nil { @@ -910,21 +906,22 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima } return hypervisor.VMConfig{ - VCPUs: inst.Vcpus, - MemoryBytes: inst.Size, - HotplugBytes: inst.HotplugSize, - Topology: topology, - GuestMemory: m.guestMemoryConfig(), - Disks: disks, - Networks: networks, - SerialLogPath: m.paths.InstanceAppLog(inst.Id), - VsockCID: inst.VsockCID, - VsockSocket: inst.VsockSocket, - PCIDevices: pciDevices, - KernelPath: kernelPath, - InitrdPath: initrdPath, - KernelArgs: m.kernelArgs(inst.HypervisorType), - EnableRosetta: inst.EnableRosetta, + VCPUs: inst.Vcpus, + MemoryBytes: inst.Size, + HotplugBytes: inst.HotplugSize, + Topology: topology, + GuestMemory: m.guestMemoryConfig(), + Disks: disks, + Networks: networks, + SerialLogPath: m.paths.InstanceAppLog(inst.Id), + VsockCID: inst.VsockCID, + VsockSocket: inst.VsockSocket, + PCIDevices: pciDevices, + VGPUDevicePath: storedVGPUDevicePath(&inst.StoredMetadata), + KernelPath: kernelPath, + InitrdPath: initrdPath, + KernelArgs: m.kernelArgs(inst.HypervisorType), + EnableRosetta: inst.EnableRosetta, }, nil } diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go index db546c153..05b3e9d8c 100644 --- a/lib/instances/create_mdev_test.go +++ b/lib/instances/create_mdev_test.go @@ -45,7 +45,7 @@ func TestCreateInstanceRejectsUnsupportedVGPUBeforeResourceReservation(t *testin assert.Zero(t, validator.reserveCalls) } -func TestWrapCreateMdevErr(t *testing.T) { +func TestWrapCreateVGPUErr(t *testing.T) { t.Parallel() for _, tc := range []struct { @@ -61,15 +61,15 @@ func TestWrapCreateMdevErr(t *testing.T) { wantInvalidRequest: true, }, { - name: "other mdev error", + name: "other vGPU error", err: errors.New("boom"), - wantMessage: "create vGPU mdev for profile profile: boom", + wantMessage: "create vGPU for profile profile: boom", }, } { t.Run(tc.name, func(t *testing.T) { t.Parallel() - err := wrapCreateMdevErr("profile", tc.err) + err := wrapCreateVGPUErr("profile", tc.err) assert.ErrorIs(t, err, tc.err) if tc.wantInvalidRequest { diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 07c3da1b2..7bf2bbf41 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -171,12 +171,10 @@ func (m *manager) deleteInstanceWithOptions( } } - // 7c. Destroy vGPU mdev device if present - if inst.GPUMdevUUID != "" { - log.InfoContext(ctx, "destroying vGPU mdev", "instance_id", id, "uuid", inst.GPUMdevUUID) - if err := devices.DestroyMdev(ctx, inst.GPUMdevUUID); err != nil { - // Log error but continue with cleanup - log.WarnContext(ctx, "failed to destroy mdev, continuing with cleanup", "instance_id", id, "uuid", inst.GPUMdevUUID, "error", err) + // 7c. Release the vGPU assignment if present. + if path := storedVGPUDevicePath(stored); path != "" { + if err := devices.DestroyVGPU(ctx, stored.GPUFramework, path, stored.GPUMdevUUID); err != nil { + log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "error", err) } } diff --git a/lib/instances/start.go b/lib/instances/start.go index 3a0879a74..c8654bc56 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -144,22 +144,17 @@ func (m *manager) startInstance( } } - // 4b. Recreate vGPU mdev if this instance had a GPU profile - // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { - log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile) - mdev, err := devices.CreateMdev(ctx, stored.GPUProfile, id) + log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) + device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id) if err != nil { - log.ErrorContext(ctx, "failed to create mdev", "instance_id", id, "profile", stored.GPUProfile, "error", err) - return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) + log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) + return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } - stored.GPUMdevUUID = mdev.UUID - log.InfoContext(ctx, "created vGPU mdev", "instance_id", id, "profile", stored.GPUProfile, "uuid", mdev.UUID) - // Add mdev cleanup to stack + setStoredVGPUDevice(stored, device) cu.Add(func() { - log.DebugContext(ctx, "destroying mdev on cleanup", "instance_id", id, "uuid", mdev.UUID) - if err := devices.DestroyMdev(ctx, mdev.UUID); err != nil { - log.WarnContext(ctx, "failed to destroy mdev on cleanup", "instance_id", id, "uuid", mdev.UUID, "error", err) + if err := devices.DestroyVGPU(ctx, device.Framework, device.SysfsPath, device.MdevUUID); err != nil { + log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err) } }) } diff --git a/lib/instances/stop.go b/lib/instances/stop.go index fb60c68c9..cff8eb5a5 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -263,12 +263,10 @@ func (m *manager) stopInstance( } } - // 7. Destroy vGPU mdev device if present (frees vGPU slot for other VMs) - if inst.GPUMdevUUID != "" { - log.InfoContext(ctx, "destroying vGPU mdev on stop", "instance_id", id, "uuid", inst.GPUMdevUUID) - if err := devices.DestroyMdev(ctx, inst.GPUMdevUUID); err != nil { - // Log error but continue - mdev cleanup is best-effort - log.WarnContext(ctx, "failed to destroy mdev on stop", "instance_id", id, "uuid", inst.GPUMdevUUID, "error", err) + // 7. Release the vGPU assignment if present. + if path := storedVGPUDevicePath(stored); path != "" { + if err := devices.DestroyVGPU(ctx, stored.GPUFramework, path, stored.GPUMdevUUID); err != nil { + log.WarnContext(ctx, "failed to destroy vGPU on stop", "instance_id", id, "error", err) } } @@ -298,11 +296,11 @@ func (m *manager) stopInstance( } } - // 10. Update metadata (clear PID, mdev UUID, set StoppedAt) + // 10. Update metadata (clear PID, set StoppedAt) now := time.Now().UTC() stored.StoppedAt = &now stored.HypervisorPID = nil - stored.GPUMdevUUID = "" // Clear mdev UUID since we destroyed it + clearStoredVGPUDevice(stored) // Boot markers are per-boot-run and must not carry across stop/restore/start. stored.ProgramStartedAt = nil stored.GuestAgentReadyAt = nil diff --git a/lib/instances/types.go b/lib/instances/types.go index df3ab7ff8..27aa492e3 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -4,6 +4,7 @@ import ( "time" "github.com/kernel/hypeman/lib/autostandby" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/healthcheck" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances/phasetracking" @@ -148,8 +149,10 @@ type StoredMetadata struct { Devices []string // Device IDs attached to this instance // GPU configuration (vGPU mode) - GPUProfile string // vGPU profile name (e.g., "L40S-1Q") - GPUMdevUUID string // mdev device UUID + GPUProfile string // vGPU profile name (e.g., "L40S-1Q") + GPUFramework devices.VGPUFramework + GPUDevicePath string + GPUMdevUUID string // populated for mdev-backed vGPUs // Command overrides (like docker run ) Entrypoint []string // Override image entrypoint (nil = use image default) diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go new file mode 100644 index 000000000..d41606359 --- /dev/null +++ b/lib/instances/vgpu.go @@ -0,0 +1,29 @@ +package instances + +import ( + "path/filepath" + + "github.com/kernel/hypeman/lib/devices" +) + +func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { + stored.GPUFramework = device.Framework + stored.GPUDevicePath = device.SysfsPath + stored.GPUMdevUUID = device.MdevUUID +} + +func clearStoredVGPUDevice(stored *StoredMetadata) { + stored.GPUFramework = devices.VGPUFrameworkNone + stored.GPUDevicePath = "" + stored.GPUMdevUUID = "" +} + +func storedVGPUDevicePath(stored *StoredMetadata) string { + if stored.GPUDevicePath != "" { + return stored.GPUDevicePath + } + if stored.GPUMdevUUID != "" { + return filepath.Join("/sys/bus/mdev/devices", stored.GPUMdevUUID) + } + return "" +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go new file mode 100644 index 000000000..d75d68b88 --- /dev/null +++ b/lib/instances/vgpu_test.go @@ -0,0 +1,40 @@ +package instances + +import ( + "testing" + + "github.com/kernel/hypeman/lib/devices" + "github.com/stretchr/testify/assert" +) + +func TestStoredVGPUDevicePath(t *testing.T) { + t.Parallel() + + assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", storedVGPUDevicePath(&StoredMetadata{ + GPUDevicePath: "/sys/bus/mdev/devices/new-uuid", + GPUMdevUUID: "legacy-uuid", + })) + assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{ + GPUMdevUUID: "legacy-uuid", + })) + assert.Empty(t, storedVGPUDevicePath(&StoredMetadata{})) +} + +func TestSetAndClearStoredVGPUDevice(t *testing.T) { + t.Parallel() + + stored := &StoredMetadata{} + setStoredVGPUDevice(stored, &devices.VGPUDevice{ + Framework: devices.VGPUFrameworkMdev, + SysfsPath: "/sys/bus/mdev/devices/new-uuid", + MdevUUID: "new-uuid", + }) + assert.Equal(t, devices.VGPUFrameworkMdev, stored.GPUFramework) + assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", stored.GPUDevicePath) + assert.Equal(t, "new-uuid", stored.GPUMdevUUID) + + clearStoredVGPUDevice(stored) + assert.Empty(t, stored.GPUFramework) + assert.Empty(t, stored.GPUDevicePath) + assert.Empty(t, stored.GPUMdevUUID) +} diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index dc2437bbf..5ed0de8ac 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -32,17 +32,16 @@ func GetGPUStatus() *GPUResourceStatus { } } -// getVGPUStatus returns GPU status for vGPU mode (SR-IOV + mdev). +// getVGPUStatus returns GPU status for vGPU mode. func getVGPUStatus() *GPUResourceStatus { vfs, err := devices.DiscoverVFs() if err != nil || len(vfs) == 0 { return nil } - // Count used VFs (those with mdevs) usedSlots := 0 for _, vf := range vfs { - if vf.HasMdev { + if vf.Allocated { usedSlots++ } } From 025670fc6c7be548f5ce241faa0acdfa438852b6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:40:03 +0000 Subject: [PATCH 02/18] Harden vGPU lifecycle compatibility --- lib/devices/GPU.md | 15 +++++ lib/devices/mdev_linux.go | 5 +- lib/devices/types.go | 7 ++ lib/devices/types_test.go | 21 ++++++ lib/instances/create.go | 7 +- lib/instances/delete.go | 8 +-- lib/instances/lifecycle_noop_test.go | 19 ++++++ lib/instances/start.go | 8 +++ lib/instances/stop.go | 8 +-- lib/instances/vgpu.go | 12 ++++ lib/instances/vgpu_hypervisor.go | 30 +++++++++ lib/instances/vgpu_hypervisor_test.go | 92 +++++++++++++++++++++++++++ lib/instances/vgpu_test.go | 14 ++++ lib/resources/gpu.go | 2 +- 14 files changed, 231 insertions(+), 17 deletions(-) create mode 100644 lib/devices/types_test.go create mode 100644 lib/instances/vgpu_hypervisor.go create mode 100644 lib/instances/vgpu_hypervisor_test.go diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 54a19c472..4fd6bddee 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -235,6 +235,21 @@ To upgrade the NVIDIA driver version: - Run GPU passthrough E2E tests - Verify with real CUDA workloads (e.g., ollama inference) +## Rolling Back vGPU Changes + +Before downgrading Hypeman or the host to a version that does not support the active vGPU framework: + +1. Stop or delete all vGPU instances while the current Hypeman version can release their assignments. +2. Confirm `/resources` reports `used_slots: 0`. +3. Confirm no assignments remain in either framework: + ```bash + test -z "$(find /sys/bus/mdev/devices -mindepth 1 -maxdepth 1 2>/dev/null)" + find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type' -exec grep -H -v '^0$' {} + + ``` +4. Downgrade only after both checks are clean. + +If assignment cleanup fails, Hypeman retains the instance metadata so a compatible version can retry it. Do not remove that metadata manually while the assignment remains active. + ## Troubleshooting ### No GPU shown in /resources diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 5d1ff6ac7..6ff2b0f33 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -127,6 +127,7 @@ func discoverMdevVFs() ([]VirtualFunction, error) { PCIAddress: vfAddr, ParentGPU: parentGPU, Allocated: hasMdev, + HasMdev: hasMdev, }) } @@ -242,7 +243,7 @@ func countAvailableVFsForProfilesParallel(vfs []VirtualFunction, profiles []prof // Group free VFs by parent GPU (done once, shared by all goroutines) freeVFsByParent := make(map[string][]VirtualFunction) for _, vf := range vfs { - if vf.Allocated { + if vf.IsAllocated() { continue } freeVFsByParent[vf.ParentGPU] = append(freeVFsByParent[vf.ParentGPU], vf) @@ -442,7 +443,7 @@ func selectLeastLoadedVF(ctx context.Context, vfs []VirtualFunction, profileType allGPUs := make(map[string]bool) for _, vf := range vfs { allGPUs[vf.ParentGPU] = true - if !vf.Allocated { + if !vf.IsAllocated() { freeVFsByGPU[vf.ParentGPU] = append(freeVFsByGPU[vf.ParentGPU], vf) } } diff --git a/lib/devices/types.go b/lib/devices/types.go index fd717d83f..ce5e8f846 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -79,6 +79,13 @@ type VirtualFunction struct { PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF + // HasMdev is retained for source and JSON compatibility. + // Deprecated: use Allocated. + HasMdev bool `json:"has_mdev"` +} + +func (vf VirtualFunction) IsAllocated() bool { + return vf.Allocated || vf.HasMdev } type VGPUDevice struct { diff --git a/lib/devices/types_test.go b/lib/devices/types_test.go new file mode 100644 index 000000000..fee4048fd --- /dev/null +++ b/lib/devices/types_test.go @@ -0,0 +1,21 @@ +package devices + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVirtualFunctionAllocationCompatibility(t *testing.T) { + t.Parallel() + + legacy := VirtualFunction{HasMdev: true} + assert.True(t, legacy.IsAllocated()) + + vf := VirtualFunction{Allocated: true, HasMdev: true} + data, err := json.Marshal(vf) + require.NoError(t, err) + assert.JSONEq(t, `{"pci_address":"","parent_gpu":"","allocated":true,"has_mdev":true}`, string(data)) +} diff --git a/lib/instances/create.go b/lib/instances/create.go index caa517e2a..a61ca4b2e 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -99,9 +99,10 @@ func (m *manager) createInstance( if req.GPU != nil && req.GPU.Profile != "" && !devices.Capabilities().SupportsVGPU { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, devices.ErrVGPUNotSupportedOnMacOS) } - hvType := req.Hypervisor - if hvType == "" { - hvType = m.defaultHypervisor + hvType, err := resolveCreateHypervisor(req, m.defaultHypervisor) + if err != nil { + log.ErrorContext(ctx, "invalid create request", "error", err) + return nil, err } // 2. Validate image exists and is ready; auto-pull if not found diff --git a/lib/instances/delete.go b/lib/instances/delete.go index 7bf2bbf41..d89f981e2 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -7,7 +7,6 @@ import ( "syscall" "time" - "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" @@ -172,10 +171,9 @@ func (m *manager) deleteInstanceWithOptions( } // 7c. Release the vGPU assignment if present. - if path := storedVGPUDevicePath(stored); path != "" { - if err := devices.DestroyVGPU(ctx, stored.GPUFramework, path, stored.GPUMdevUUID); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "error", err) - } + if err := releaseStoredVGPU(ctx, stored); err != nil { + log.ErrorContext(ctx, "failed to destroy vGPU; retaining instance metadata", "instance_id", id, "error", err) + return fmt.Errorf("destroy vGPU: %w", err) } // 8. Delete all instance data diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 5ca7515fa..08205a544 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" @@ -147,6 +148,24 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T assertNoLifecycleEvent(t, events) } +func TestDeleteRetainsMetadataWhenVGPUReleaseFails(t *testing.T) { + m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) + meta, err := m.loadMetadata(id) + require.NoError(t, err) + meta.GPUProfile = "NVIDIA L40S-2Q" + meta.GPUFramework = devices.VGPUFramework("future-framework") + meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" + require.NoError(t, m.saveMetadata(meta)) + + err = m.DeleteInstance(context.Background(), id) + require.Error(t, err) + + stored, err := m.loadMetadata(id) + require.NoError(t, err) + assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) { t.Helper() diff --git a/lib/instances/start.go b/lib/instances/start.go index c8654bc56..a195db8c7 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,6 +48,14 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if err := validateVGPUHypervisor(stored.GPUProfile, stored.HypervisorType); err != nil { + log.ErrorContext(ctx, "invalid vGPU hypervisor", "instance_id", id, "error", err) + return nil, err + } + if err := releaseStoredVGPU(ctx, stored); err != nil { + log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) + return nil, fmt.Errorf("release stale vGPU before start: %w", err) + } // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil diff --git a/lib/instances/stop.go b/lib/instances/stop.go index cff8eb5a5..de7d6cff2 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -9,7 +9,6 @@ import ( "syscall" "time" - "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/guest" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/instances/phasetracking" @@ -264,10 +263,8 @@ func (m *manager) stopInstance( } // 7. Release the vGPU assignment if present. - if path := storedVGPUDevicePath(stored); path != "" { - if err := devices.DestroyVGPU(ctx, stored.GPUFramework, path, stored.GPUMdevUUID); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on stop", "instance_id", id, "error", err) - } + if err := releaseStoredVGPU(ctx, stored); err != nil { + log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "error", err) } // 8. Always remove stale runtime sockets after process exit. @@ -300,7 +297,6 @@ func (m *manager) stopInstance( now := time.Now().UTC() stored.StoppedAt = &now stored.HypervisorPID = nil - clearStoredVGPUDevice(stored) // Boot markers are per-boot-run and must not carry across stop/restore/start. stored.ProgramStartedAt = nil stored.GuestAgentReadyAt = nil diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index d41606359..c2294ac53 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -1,6 +1,7 @@ package instances import ( + "context" "path/filepath" "github.com/kernel/hypeman/lib/devices" @@ -18,6 +19,17 @@ func clearStoredVGPUDevice(stored *StoredMetadata) { stored.GPUMdevUUID = "" } +func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error { + path := storedVGPUDevicePath(stored) + if path != "" { + if err := devices.DestroyVGPU(ctx, stored.GPUFramework, path, stored.GPUMdevUUID); err != nil { + return err + } + } + clearStoredVGPUDevice(stored) + return nil +} + func storedVGPUDevicePath(stored *StoredMetadata) string { if stored.GPUDevicePath != "" { return stored.GPUDevicePath diff --git a/lib/instances/vgpu_hypervisor.go b/lib/instances/vgpu_hypervisor.go new file mode 100644 index 000000000..c1a031927 --- /dev/null +++ b/lib/instances/vgpu_hypervisor.go @@ -0,0 +1,30 @@ +package instances + +import ( + "fmt" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +func resolveCreateHypervisor(req CreateInstanceRequest, defaultHypervisor hypervisor.Type) (hypervisor.Type, error) { + hvType := req.Hypervisor + if hvType == "" { + hvType = defaultHypervisor + } + + profile := "" + if req.GPU != nil { + profile = req.GPU.Profile + } + if err := validateVGPUHypervisor(profile, hvType); err != nil { + return "", err + } + return hvType, nil +} + +func validateVGPUHypervisor(profile string, hvType hypervisor.Type) error { + if profile != "" && hvType != hypervisor.TypeQEMU { + return fmt.Errorf("%w: vGPU requires qemu, got %s", ErrInvalidRequest, hvType) + } + return nil +} diff --git a/lib/instances/vgpu_hypervisor_test.go b/lib/instances/vgpu_hypervisor_test.go new file mode 100644 index 000000000..f0e63b390 --- /dev/null +++ b/lib/instances/vgpu_hypervisor_test.go @@ -0,0 +1,92 @@ +package instances + +import ( + "errors" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +func TestResolveCreateHypervisorForVGPU(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + request CreateInstanceRequest + defaultHypervisor hypervisor.Type + want hypervisor.Type + wantErr bool + }{ + { + name: "explicit qemu", + request: CreateInstanceRequest{ + Hypervisor: hypervisor.TypeQEMU, + GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, + }, + defaultHypervisor: hypervisor.TypeCloudHypervisor, + want: hypervisor.TypeQEMU, + }, + { + name: "qemu default", + request: CreateInstanceRequest{ + GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, + }, + defaultHypervisor: hypervisor.TypeQEMU, + want: hypervisor.TypeQEMU, + }, + { + name: "explicit cloud hypervisor", + request: CreateInstanceRequest{ + Hypervisor: hypervisor.TypeCloudHypervisor, + GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, + }, + defaultHypervisor: hypervisor.TypeQEMU, + wantErr: true, + }, + { + name: "cloud hypervisor default", + request: CreateInstanceRequest{ + GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, + }, + defaultHypervisor: hypervisor.TypeCloudHypervisor, + wantErr: true, + }, + { + name: "firecracker", + request: CreateInstanceRequest{ + Hypervisor: hypervisor.TypeFirecracker, + GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, + }, + defaultHypervisor: hypervisor.TypeQEMU, + wantErr: true, + }, + { + name: "non-GPU cloud hypervisor", + request: CreateInstanceRequest{ + Hypervisor: hypervisor.TypeCloudHypervisor, + }, + defaultHypervisor: hypervisor.TypeQEMU, + want: hypervisor.TypeCloudHypervisor, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := resolveCreateHypervisor(tt.request, tt.defaultHypervisor) + if tt.wantErr { + if !errors.Is(err, ErrInvalidRequest) { + t.Fatalf("expected ErrInvalidRequest, got %v", err) + } + return + } + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if got != tt.want { + t.Fatalf("expected %q, got %q", tt.want, got) + } + }) + } +} diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index d75d68b88..6f2c46819 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -1,6 +1,7 @@ package instances import ( + "context" "testing" "github.com/kernel/hypeman/lib/devices" @@ -20,6 +21,19 @@ func TestStoredVGPUDevicePath(t *testing.T) { assert.Empty(t, storedVGPUDevicePath(&StoredMetadata{})) } +func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) { + t.Parallel() + + stored := &StoredMetadata{ + GPUFramework: devices.VGPUFramework("future-framework"), + GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4", + } + err := releaseStoredVGPU(context.Background(), stored) + assert.Error(t, err) + assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) + assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) +} + func TestSetAndClearStoredVGPUDevice(t *testing.T) { t.Parallel() diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index 5ed0de8ac..c523e3550 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -41,7 +41,7 @@ func getVGPUStatus() *GPUResourceStatus { usedSlots := 0 for _, vf := range vfs { - if vf.Allocated { + if vf.IsAllocated() { usedSlots++ } } From 3f14e18ccb1e48b2d6320c0020ad36f9ffeea185 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:50:42 +0000 Subject: [PATCH 03/18] Validate unsupported vGPU frameworks on Darwin --- lib/devices/mdev_darwin.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 3efd19489..71688210c 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -50,6 +50,9 @@ func IsMdevInUse(mdevUUID string) bool { } func DestroyVGPU(ctx context.Context, framework VGPUFramework, devicePath, mdevUUID string) error { + if framework != VGPUFrameworkNone && framework != VGPUFrameworkMdev { + return fmt.Errorf("unknown vGPU framework %q", framework) + } return nil } From 9ba94930fb97fa42a088fea2cebbcf1c0d284232 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:59:05 +0000 Subject: [PATCH 04/18] Reduce vGPU refactor diff noise --- cmd/api/main.go | 8 +++++--- lib/devices/mdev_darwin.go | 4 ---- lib/devices/mdev_linux.go | 27 +++++++++++++++++++-------- lib/devices/vgpu_linux.go | 20 -------------------- lib/instances/create.go | 3 +++ lib/instances/start.go | 3 +++ lib/resources/gpu.go | 5 +++-- 7 files changed, 33 insertions(+), 37 deletions(-) diff --git a/cmd/api/main.go b/cmd/api/main.go index 345fd0120..98d7be720 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -362,9 +362,11 @@ func run() error { return fmt.Errorf("reconcile device state: %w", err) } - logger.Info("Reconciling vGPU devices...") - if err := devices.ReconcileVGPUs(app.Ctx); err != nil { - logger.Warn("failed to reconcile vGPU devices", "error", err) + // Reconcile mdev devices (clears orphaned vGPUs from previous runs) + logger.Info("Reconciling mdev devices...") + if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil { + // Log but don't fail - mdev cleanup is best-effort + logger.Warn("failed to reconcile mdev devices", "error", err) } // Wire up resource validator for aggregate limit checking diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 71688210c..2bd44282e 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -56,10 +56,6 @@ func DestroyVGPU(ctx context.Context, framework VGPUFramework, devicePath, mdevU return nil } -func ReconcileVGPUs(ctx context.Context) error { - return nil -} - // ReconcileMdevs is a no-op on macOS. func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error { return nil diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index 6ff2b0f33..a9e486262 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -89,10 +89,10 @@ func getCachedProfiles(firstVF string) []profileMetadata { return cachedProfiles } -// discoverMdevVFs returns all SR-IOV Virtual Functions available for mdev vGPU. +// DiscoverVFs returns all SR-IOV Virtual Functions available for vGPU. // These are discovered by scanning /sys/class/mdev_bus/ which contains // VFs that can host mdev devices. -func discoverMdevVFs() ([]VirtualFunction, error) { +func DiscoverVFs() ([]VirtualFunction, error) { entries, err := os.ReadDir(mdevBusPath) if err != nil { if os.IsNotExist(err) { @@ -136,7 +136,18 @@ func discoverMdevVFs() ([]VirtualFunction, error) { // ListGPUProfiles returns available vGPU profiles with availability counts. // Profiles are discovered from the first VF's mdev_supported_types directory. -func listMdevGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { +func ListGPUProfiles() ([]GPUProfile, error) { + vfs, err := DiscoverVFs() + if err != nil { + return nil, err + } + return ListGPUProfilesWithVFs(vfs) +} + +// ListGPUProfilesWithVFs returns available vGPU profiles using pre-discovered VFs. +// This avoids redundant VF discovery when the caller already has the list. +// Uses parallel sysfs reads for fast availability counting. +func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { if len(vfs) == 0 { return nil, nil } @@ -243,7 +254,7 @@ func countAvailableVFsForProfilesParallel(vfs []VirtualFunction, profiles []prof // Group free VFs by parent GPU (done once, shared by all goroutines) freeVFsByParent := make(map[string][]VirtualFunction) for _, vf := range vfs { - if vf.IsAllocated() { + if vf.HasMdev { continue } freeVFsByParent[vf.ParentGPU] = append(freeVFsByParent[vf.ParentGPU], vf) @@ -295,7 +306,7 @@ func countAvailableForSingleProfile(freeVFsByParent map[string][]VirtualFunction // findProfileType finds the internal type name (e.g., "nvidia-556") for a profile name (e.g., "L40S-1Q") func findProfileType(profileName string) (string, error) { - vfs, err := discoverMdevVFs() + vfs, err := DiscoverVFs() if err != nil || len(vfs) == 0 { return "", fmt.Errorf("no VFs available") } @@ -443,7 +454,7 @@ func selectLeastLoadedVF(ctx context.Context, vfs []VirtualFunction, profileType allGPUs := make(map[string]bool) for _, vf := range vfs { allGPUs[vf.ParentGPU] = true - if !vf.IsAllocated() { + if !vf.HasMdev { freeVFsByGPU[vf.ParentGPU] = append(freeVFsByGPU[vf.ParentGPU], vf) } } @@ -521,7 +532,7 @@ func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevic } // Discover all VFs - vfs, err := discoverMdevVFs() + vfs, err := DiscoverVFs() if err != nil { return nil, fmt.Errorf("discover VFs: %w", err) } @@ -687,7 +698,7 @@ func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) erro log := logger.FromContext(ctx) _ = instanceInfos - vfs, err := discoverMdevVFs() + vfs, err := DiscoverVFs() if err != nil { return fmt.Errorf("discover managed VFs: %w", err) } diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go index b6f15c716..429e99888 100644 --- a/lib/devices/vgpu_linux.go +++ b/lib/devices/vgpu_linux.go @@ -8,22 +8,6 @@ import ( "path/filepath" ) -func DiscoverVFs() ([]VirtualFunction, error) { - return discoverMdevVFs() -} - -func ListGPUProfiles() ([]GPUProfile, error) { - vfs, err := DiscoverVFs() - if err != nil { - return nil, err - } - return ListGPUProfilesWithVFs(vfs) -} - -func ListGPUProfilesWithVFs(vfs []VirtualFunction) ([]GPUProfile, error) { - return listMdevGPUProfilesWithVFs(vfs) -} - func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) { mdev, err := CreateMdev(ctx, profileName, instanceID) if err != nil { @@ -51,7 +35,3 @@ func DestroyVGPU(ctx context.Context, framework VGPUFramework, devicePath, mdevU } return DestroyMdev(ctx, mdevUUID) } - -func ReconcileVGPUs(ctx context.Context) error { - return ReconcileMdevs(ctx, nil) -} diff --git a/lib/instances/create.go b/lib/instances/create.go index a61ca4b2e..0c3fc2535 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -284,6 +284,7 @@ func (m *manager) createInstance( }) } + // Handle vGPU profile request - create mdev device if req.GPU != nil && req.GPU.Profile != "" { log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id) @@ -295,6 +296,8 @@ func (m *manager) createInstance( gpuFramework = gpuDevice.Framework gpuDevicePath = gpuDevice.SysfsPath gpuMdevUUID = gpuDevice.MdevUUID + + // Add mdev cleanup to stack cu.Add(func() { if err := devices.DestroyVGPU(ctx, gpuDevice.Framework, gpuDevice.SysfsPath, gpuDevice.MdevUUID); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err) diff --git a/lib/instances/start.go b/lib/instances/start.go index a195db8c7..b90eef119 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -152,6 +152,8 @@ func (m *manager) startInstance( } } + // 4b. Recreate vGPU mdev if this instance had a GPU profile + // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id) @@ -160,6 +162,7 @@ func (m *manager) startInstance( return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } setStoredVGPUDevice(stored, device) + // Add mdev cleanup to stack cu.Add(func() { if err := devices.DestroyVGPU(ctx, device.Framework, device.SysfsPath, device.MdevUUID); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err) diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index c523e3550..dc2437bbf 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -32,16 +32,17 @@ func GetGPUStatus() *GPUResourceStatus { } } -// getVGPUStatus returns GPU status for vGPU mode. +// getVGPUStatus returns GPU status for vGPU mode (SR-IOV + mdev). func getVGPUStatus() *GPUResourceStatus { vfs, err := devices.DiscoverVFs() if err != nil || len(vfs) == 0 { return nil } + // Count used VFs (those with mdevs) usedSlots := 0 for _, vf := range vfs { - if vf.IsAllocated() { + if vf.HasMdev { usedSlots++ } } From f64d623bb26ee3dc2a97c2705054b247a9f951f8 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 29 Jul 2026 15:14:34 +0000 Subject: [PATCH 05/18] Leave vGPU hypervisor selection to callers --- lib/instances/create.go | 7 +- lib/instances/start.go | 4 -- lib/instances/vgpu_hypervisor.go | 30 --------- lib/instances/vgpu_hypervisor_test.go | 92 --------------------------- 4 files changed, 3 insertions(+), 130 deletions(-) delete mode 100644 lib/instances/vgpu_hypervisor.go delete mode 100644 lib/instances/vgpu_hypervisor_test.go diff --git a/lib/instances/create.go b/lib/instances/create.go index 0c3fc2535..1baa2bb27 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -99,10 +99,9 @@ func (m *manager) createInstance( if req.GPU != nil && req.GPU.Profile != "" && !devices.Capabilities().SupportsVGPU { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, devices.ErrVGPUNotSupportedOnMacOS) } - hvType, err := resolveCreateHypervisor(req, m.defaultHypervisor) - if err != nil { - log.ErrorContext(ctx, "invalid create request", "error", err) - return nil, err + hvType := req.Hypervisor + if hvType == "" { + hvType = m.defaultHypervisor } // 2. Validate image exists and is ready; auto-pull if not found diff --git a/lib/instances/start.go b/lib/instances/start.go index b90eef119..deb3d60dc 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,10 +48,6 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } - if err := validateVGPUHypervisor(stored.GPUProfile, stored.HypervisorType); err != nil { - log.ErrorContext(ctx, "invalid vGPU hypervisor", "instance_id", id, "error", err) - return nil, err - } if err := releaseStoredVGPU(ctx, stored); err != nil { log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) return nil, fmt.Errorf("release stale vGPU before start: %w", err) diff --git a/lib/instances/vgpu_hypervisor.go b/lib/instances/vgpu_hypervisor.go deleted file mode 100644 index c1a031927..000000000 --- a/lib/instances/vgpu_hypervisor.go +++ /dev/null @@ -1,30 +0,0 @@ -package instances - -import ( - "fmt" - - "github.com/kernel/hypeman/lib/hypervisor" -) - -func resolveCreateHypervisor(req CreateInstanceRequest, defaultHypervisor hypervisor.Type) (hypervisor.Type, error) { - hvType := req.Hypervisor - if hvType == "" { - hvType = defaultHypervisor - } - - profile := "" - if req.GPU != nil { - profile = req.GPU.Profile - } - if err := validateVGPUHypervisor(profile, hvType); err != nil { - return "", err - } - return hvType, nil -} - -func validateVGPUHypervisor(profile string, hvType hypervisor.Type) error { - if profile != "" && hvType != hypervisor.TypeQEMU { - return fmt.Errorf("%w: vGPU requires qemu, got %s", ErrInvalidRequest, hvType) - } - return nil -} diff --git a/lib/instances/vgpu_hypervisor_test.go b/lib/instances/vgpu_hypervisor_test.go deleted file mode 100644 index f0e63b390..000000000 --- a/lib/instances/vgpu_hypervisor_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package instances - -import ( - "errors" - "testing" - - "github.com/kernel/hypeman/lib/hypervisor" -) - -func TestResolveCreateHypervisorForVGPU(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - request CreateInstanceRequest - defaultHypervisor hypervisor.Type - want hypervisor.Type - wantErr bool - }{ - { - name: "explicit qemu", - request: CreateInstanceRequest{ - Hypervisor: hypervisor.TypeQEMU, - GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, - }, - defaultHypervisor: hypervisor.TypeCloudHypervisor, - want: hypervisor.TypeQEMU, - }, - { - name: "qemu default", - request: CreateInstanceRequest{ - GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, - }, - defaultHypervisor: hypervisor.TypeQEMU, - want: hypervisor.TypeQEMU, - }, - { - name: "explicit cloud hypervisor", - request: CreateInstanceRequest{ - Hypervisor: hypervisor.TypeCloudHypervisor, - GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, - }, - defaultHypervisor: hypervisor.TypeQEMU, - wantErr: true, - }, - { - name: "cloud hypervisor default", - request: CreateInstanceRequest{ - GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, - }, - defaultHypervisor: hypervisor.TypeCloudHypervisor, - wantErr: true, - }, - { - name: "firecracker", - request: CreateInstanceRequest{ - Hypervisor: hypervisor.TypeFirecracker, - GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, - }, - defaultHypervisor: hypervisor.TypeQEMU, - wantErr: true, - }, - { - name: "non-GPU cloud hypervisor", - request: CreateInstanceRequest{ - Hypervisor: hypervisor.TypeCloudHypervisor, - }, - defaultHypervisor: hypervisor.TypeQEMU, - want: hypervisor.TypeCloudHypervisor, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got, err := resolveCreateHypervisor(tt.request, tt.defaultHypervisor) - if tt.wantErr { - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - return - } - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if got != tt.want { - t.Fatalf("expected %q, got %q", tt.want, got) - } - }) - } -} From 602f8939c636a6ba850c6768563a0af0dee3d8ea Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 29 Jul 2026 16:05:58 +0000 Subject: [PATCH 06/18] Require qemu for vGPU instances --- lib/instances/vgpu_hypervisor.go | 30 +++++++++ lib/instances/vgpu_hypervisor_test.go | 92 +++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 lib/instances/vgpu_hypervisor.go create mode 100644 lib/instances/vgpu_hypervisor_test.go diff --git a/lib/instances/vgpu_hypervisor.go b/lib/instances/vgpu_hypervisor.go new file mode 100644 index 000000000..c1a031927 --- /dev/null +++ b/lib/instances/vgpu_hypervisor.go @@ -0,0 +1,30 @@ +package instances + +import ( + "fmt" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +func resolveCreateHypervisor(req CreateInstanceRequest, defaultHypervisor hypervisor.Type) (hypervisor.Type, error) { + hvType := req.Hypervisor + if hvType == "" { + hvType = defaultHypervisor + } + + profile := "" + if req.GPU != nil { + profile = req.GPU.Profile + } + if err := validateVGPUHypervisor(profile, hvType); err != nil { + return "", err + } + return hvType, nil +} + +func validateVGPUHypervisor(profile string, hvType hypervisor.Type) error { + if profile != "" && hvType != hypervisor.TypeQEMU { + return fmt.Errorf("%w: vGPU requires qemu, got %s", ErrInvalidRequest, hvType) + } + return nil +} diff --git a/lib/instances/vgpu_hypervisor_test.go b/lib/instances/vgpu_hypervisor_test.go new file mode 100644 index 000000000..f0e63b390 --- /dev/null +++ b/lib/instances/vgpu_hypervisor_test.go @@ -0,0 +1,92 @@ +package instances + +import ( + "errors" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" +) + +func TestResolveCreateHypervisorForVGPU(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + request CreateInstanceRequest + defaultHypervisor hypervisor.Type + want hypervisor.Type + wantErr bool + }{ + { + name: "explicit qemu", + request: CreateInstanceRequest{ + Hypervisor: hypervisor.TypeQEMU, + GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, + }, + defaultHypervisor: hypervisor.TypeCloudHypervisor, + want: hypervisor.TypeQEMU, + }, + { + name: "qemu default", + request: CreateInstanceRequest{ + GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, + }, + defaultHypervisor: hypervisor.TypeQEMU, + want: hypervisor.TypeQEMU, + }, + { + name: "explicit cloud hypervisor", + request: CreateInstanceRequest{ + Hypervisor: hypervisor.TypeCloudHypervisor, + GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, + }, + defaultHypervisor: hypervisor.TypeQEMU, + wantErr: true, + }, + { + name: "cloud hypervisor default", + request: CreateInstanceRequest{ + GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, + }, + defaultHypervisor: hypervisor.TypeCloudHypervisor, + wantErr: true, + }, + { + name: "firecracker", + request: CreateInstanceRequest{ + Hypervisor: hypervisor.TypeFirecracker, + GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, + }, + defaultHypervisor: hypervisor.TypeQEMU, + wantErr: true, + }, + { + name: "non-GPU cloud hypervisor", + request: CreateInstanceRequest{ + Hypervisor: hypervisor.TypeCloudHypervisor, + }, + defaultHypervisor: hypervisor.TypeQEMU, + want: hypervisor.TypeCloudHypervisor, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := resolveCreateHypervisor(tt.request, tt.defaultHypervisor) + if tt.wantErr { + if !errors.Is(err, ErrInvalidRequest) { + t.Fatalf("expected ErrInvalidRequest, got %v", err) + } + return + } + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if got != tt.want { + t.Fatalf("expected %q, got %q", tt.want, got) + } + }) + } +} From d796fd9b181abb1109740432d376f5801156db95 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 29 Jul 2026 17:50:03 +0000 Subject: [PATCH 07/18] Preserve vGPU hypervisor support --- lib/hypervisor/cloudhypervisor/config.go | 12 ++- lib/hypervisor/cloudhypervisor/config_test.go | 11 +++ lib/instances/vgpu_hypervisor.go | 30 ------ lib/instances/vgpu_hypervisor_test.go | 92 ------------------- 4 files changed, 20 insertions(+), 125 deletions(-) delete mode 100644 lib/instances/vgpu_hypervisor.go delete mode 100644 lib/instances/vgpu_hypervisor_test.go diff --git a/lib/hypervisor/cloudhypervisor/config.go b/lib/hypervisor/cloudhypervisor/config.go index e9f91fe4a..c5f506af9 100644 --- a/lib/hypervisor/cloudhypervisor/config.go +++ b/lib/hypervisor/cloudhypervisor/config.go @@ -125,10 +125,16 @@ func ToVMConfig(cfg hypervisor.VMConfig) vmm.VmConfig { } // Device passthrough configuration + devicePaths := make([]string, 0, len(cfg.PCIDevices)+1) + devicePaths = append(devicePaths, cfg.PCIDevices...) + if cfg.VGPUDevicePath != "" { + devicePaths = append(devicePaths, cfg.VGPUDevicePath) + } + var devices *[]vmm.DeviceConfig - if len(cfg.PCIDevices) > 0 { - deviceConfigs := make([]vmm.DeviceConfig, 0, len(cfg.PCIDevices)) - for _, path := range cfg.PCIDevices { + if len(devicePaths) > 0 { + deviceConfigs := make([]vmm.DeviceConfig, 0, len(devicePaths)) + for _, path := range devicePaths { deviceConfigs = append(deviceConfigs, vmm.DeviceConfig{ Path: path, }) diff --git a/lib/hypervisor/cloudhypervisor/config_test.go b/lib/hypervisor/cloudhypervisor/config_test.go index b5cdb96e9..235be3906 100644 --- a/lib/hypervisor/cloudhypervisor/config_test.go +++ b/lib/hypervisor/cloudhypervisor/config_test.go @@ -8,6 +8,17 @@ import ( "github.com/stretchr/testify/require" ) +func TestToVMConfig_VGPU(t *testing.T) { + t.Parallel() + + path := "/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123" + vmCfg := ToVMConfig(hypervisor.VMConfig{VGPUDevicePath: path}) + + require.NotNil(t, vmCfg.Devices) + require.Len(t, *vmCfg.Devices, 1) + assert.Equal(t, path, (*vmCfg.Devices)[0].Path) +} + func TestToVMConfig_GuestMemoryBalloon(t *testing.T) { cfg := hypervisor.VMConfig{ VCPUs: 1, diff --git a/lib/instances/vgpu_hypervisor.go b/lib/instances/vgpu_hypervisor.go deleted file mode 100644 index c1a031927..000000000 --- a/lib/instances/vgpu_hypervisor.go +++ /dev/null @@ -1,30 +0,0 @@ -package instances - -import ( - "fmt" - - "github.com/kernel/hypeman/lib/hypervisor" -) - -func resolveCreateHypervisor(req CreateInstanceRequest, defaultHypervisor hypervisor.Type) (hypervisor.Type, error) { - hvType := req.Hypervisor - if hvType == "" { - hvType = defaultHypervisor - } - - profile := "" - if req.GPU != nil { - profile = req.GPU.Profile - } - if err := validateVGPUHypervisor(profile, hvType); err != nil { - return "", err - } - return hvType, nil -} - -func validateVGPUHypervisor(profile string, hvType hypervisor.Type) error { - if profile != "" && hvType != hypervisor.TypeQEMU { - return fmt.Errorf("%w: vGPU requires qemu, got %s", ErrInvalidRequest, hvType) - } - return nil -} diff --git a/lib/instances/vgpu_hypervisor_test.go b/lib/instances/vgpu_hypervisor_test.go deleted file mode 100644 index f0e63b390..000000000 --- a/lib/instances/vgpu_hypervisor_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package instances - -import ( - "errors" - "testing" - - "github.com/kernel/hypeman/lib/hypervisor" -) - -func TestResolveCreateHypervisorForVGPU(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - request CreateInstanceRequest - defaultHypervisor hypervisor.Type - want hypervisor.Type - wantErr bool - }{ - { - name: "explicit qemu", - request: CreateInstanceRequest{ - Hypervisor: hypervisor.TypeQEMU, - GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, - }, - defaultHypervisor: hypervisor.TypeCloudHypervisor, - want: hypervisor.TypeQEMU, - }, - { - name: "qemu default", - request: CreateInstanceRequest{ - GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, - }, - defaultHypervisor: hypervisor.TypeQEMU, - want: hypervisor.TypeQEMU, - }, - { - name: "explicit cloud hypervisor", - request: CreateInstanceRequest{ - Hypervisor: hypervisor.TypeCloudHypervisor, - GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, - }, - defaultHypervisor: hypervisor.TypeQEMU, - wantErr: true, - }, - { - name: "cloud hypervisor default", - request: CreateInstanceRequest{ - GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, - }, - defaultHypervisor: hypervisor.TypeCloudHypervisor, - wantErr: true, - }, - { - name: "firecracker", - request: CreateInstanceRequest{ - Hypervisor: hypervisor.TypeFirecracker, - GPU: &GPUConfig{Profile: "NVIDIA L40S-2Q"}, - }, - defaultHypervisor: hypervisor.TypeQEMU, - wantErr: true, - }, - { - name: "non-GPU cloud hypervisor", - request: CreateInstanceRequest{ - Hypervisor: hypervisor.TypeCloudHypervisor, - }, - defaultHypervisor: hypervisor.TypeQEMU, - want: hypervisor.TypeCloudHypervisor, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got, err := resolveCreateHypervisor(tt.request, tt.defaultHypervisor) - if tt.wantErr { - if !errors.Is(err, ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - return - } - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if got != tt.want { - t.Fatalf("expected %q, got %q", tt.want, got) - } - }) - } -} From f6d2d41610af4c32edff50a8242f402ff9d9db18 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:06:08 +0000 Subject: [PATCH 08/18] Track VF allocation with a single field --- lib/devices/mdev_linux.go | 5 ++--- lib/devices/types.go | 7 ------- lib/devices/types_test.go | 21 --------------------- lib/resources/gpu.go | 2 +- 4 files changed, 3 insertions(+), 32 deletions(-) delete mode 100644 lib/devices/types_test.go diff --git a/lib/devices/mdev_linux.go b/lib/devices/mdev_linux.go index a9e486262..1a398a418 100644 --- a/lib/devices/mdev_linux.go +++ b/lib/devices/mdev_linux.go @@ -127,7 +127,6 @@ func DiscoverVFs() ([]VirtualFunction, error) { PCIAddress: vfAddr, ParentGPU: parentGPU, Allocated: hasMdev, - HasMdev: hasMdev, }) } @@ -254,7 +253,7 @@ func countAvailableVFsForProfilesParallel(vfs []VirtualFunction, profiles []prof // Group free VFs by parent GPU (done once, shared by all goroutines) freeVFsByParent := make(map[string][]VirtualFunction) for _, vf := range vfs { - if vf.HasMdev { + if vf.Allocated { continue } freeVFsByParent[vf.ParentGPU] = append(freeVFsByParent[vf.ParentGPU], vf) @@ -454,7 +453,7 @@ func selectLeastLoadedVF(ctx context.Context, vfs []VirtualFunction, profileType allGPUs := make(map[string]bool) for _, vf := range vfs { allGPUs[vf.ParentGPU] = true - if !vf.HasMdev { + if !vf.Allocated { freeVFsByGPU[vf.ParentGPU] = append(freeVFsByGPU[vf.ParentGPU], vf) } } diff --git a/lib/devices/types.go b/lib/devices/types.go index ce5e8f846..fd717d83f 100644 --- a/lib/devices/types.go +++ b/lib/devices/types.go @@ -79,13 +79,6 @@ type VirtualFunction struct { PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4" ParentGPU string `json:"parent_gpu"` // e.g., "0000:82:00.0" Allocated bool `json:"allocated"` // true if a vGPU is assigned to this VF - // HasMdev is retained for source and JSON compatibility. - // Deprecated: use Allocated. - HasMdev bool `json:"has_mdev"` -} - -func (vf VirtualFunction) IsAllocated() bool { - return vf.Allocated || vf.HasMdev } type VGPUDevice struct { diff --git a/lib/devices/types_test.go b/lib/devices/types_test.go deleted file mode 100644 index fee4048fd..000000000 --- a/lib/devices/types_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package devices - -import ( - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestVirtualFunctionAllocationCompatibility(t *testing.T) { - t.Parallel() - - legacy := VirtualFunction{HasMdev: true} - assert.True(t, legacy.IsAllocated()) - - vf := VirtualFunction{Allocated: true, HasMdev: true} - data, err := json.Marshal(vf) - require.NoError(t, err) - assert.JSONEq(t, `{"pci_address":"","parent_gpu":"","allocated":true,"has_mdev":true}`, string(data)) -} diff --git a/lib/resources/gpu.go b/lib/resources/gpu.go index dc2437bbf..78788412e 100644 --- a/lib/resources/gpu.go +++ b/lib/resources/gpu.go @@ -42,7 +42,7 @@ func getVGPUStatus() *GPUResourceStatus { // Count used VFs (those with mdevs) usedSlots := 0 for _, vf := range vfs { - if vf.HasMdev { + if vf.Allocated { usedSlots++ } } From 43b318282d6217dcc6ab0127d6fe767341be4980 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:12:31 +0000 Subject: [PATCH 09/18] Update vGPU lifecycle comments --- lib/instances/create.go | 4 ++-- lib/instances/start.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index 1baa2bb27..ff6707d53 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -283,7 +283,7 @@ func (m *manager) createInstance( }) } - // Handle vGPU profile request - create mdev device + // Handle vGPU profile request if req.GPU != nil && req.GPU.Profile != "" { log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile) gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id) @@ -296,7 +296,7 @@ func (m *manager) createInstance( gpuDevicePath = gpuDevice.SysfsPath gpuMdevUUID = gpuDevice.MdevUUID - // Add mdev cleanup to stack + // Add vGPU cleanup to stack cu.Add(func() { if err := devices.DestroyVGPU(ctx, gpuDevice.Framework, gpuDevice.SysfsPath, gpuDevice.MdevUUID); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err) diff --git a/lib/instances/start.go b/lib/instances/start.go index deb3d60dc..2ac623251 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -148,7 +148,7 @@ func (m *manager) startInstance( } } - // 4b. Recreate vGPU mdev if this instance had a GPU profile + // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) @@ -158,7 +158,7 @@ func (m *manager) startInstance( return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) } setStoredVGPUDevice(stored, device) - // Add mdev cleanup to stack + // Add vGPU cleanup to stack cu.Add(func() { if err := devices.DestroyVGPU(ctx, device.Framework, device.SysfsPath, device.MdevUUID); err != nil { log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err) From 3744a90d53892dba6201bc64ab38f2c169182c59 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Wed, 5 Aug 2026 14:16:24 +0000 Subject: [PATCH 10/18] Drop the dead mdev branch from QEMU PCI passthrough args Since vGPU devices attach through VGPUDevicePath, PCIDevices only carries whole-device passthrough paths; the mdev special case was unreachable. --- lib/hypervisor/qemu/config.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/hypervisor/qemu/config.go b/lib/hypervisor/qemu/config.go index 1e59ad264..66549816d 100644 --- a/lib/hypervisor/qemu/config.go +++ b/lib/hypervisor/qemu/config.go @@ -86,13 +86,10 @@ func BuildArgs(cfg hypervisor.VMConfig) []string { args = append(args, "-device", fmt.Sprintf("vfio-pci,sysfsdev=%s", cfg.VGPUDevicePath)) } - // PCI device passthrough (GPU, mdev vGPU, etc.) + // Whole-device PCI passthrough (vGPU attaches via VGPUDevicePath above) for _, devicePath := range cfg.PCIDevices { var deviceArg string - if strings.HasPrefix(devicePath, "/sys/bus/mdev/devices/") { - // mdev device (vGPU) - use sysfsdev parameter - deviceArg = fmt.Sprintf("vfio-pci,sysfsdev=%s", devicePath) - } else if strings.HasPrefix(devicePath, "/sys/bus/pci/devices/") { + if strings.HasPrefix(devicePath, "/sys/bus/pci/devices/") { // Full sysfs path for regular PCI device - extract the PCI address // Using filepath.Base is more robust than manual string splitting pciAddr := filepath.Base(strings.TrimSuffix(devicePath, "/")) From bf8cebab647076a6646ead5c128f327ef97d5be6 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:19:11 +0000 Subject: [PATCH 11/18] Fix Darwin vGPU build --- lib/devices/mdev_darwin.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/devices/mdev_darwin.go b/lib/devices/mdev_darwin.go index 2bd44282e..8ec67db49 100644 --- a/lib/devices/mdev_darwin.go +++ b/lib/devices/mdev_darwin.go @@ -2,7 +2,10 @@ package devices -import "context" +import ( + "context" + "fmt" +) // SetGPUProfileCacheTTL is a no-op on macOS. func SetGPUProfileCacheTTL(ttl string) { From 3c95e199c79157272099da7658eafe28fed498db Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 20:20:07 +0000 Subject: [PATCH 12/18] Preserve QEMU vGPU device ordering --- lib/hypervisor/qemu/config.go | 10 +++++----- lib/hypervisor/qemu/config_test.go | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/lib/hypervisor/qemu/config.go b/lib/hypervisor/qemu/config.go index 66549816d..480ef1097 100644 --- a/lib/hypervisor/qemu/config.go +++ b/lib/hypervisor/qemu/config.go @@ -82,11 +82,7 @@ func BuildArgs(cfg hypervisor.VMConfig) []string { args = append(args, "-device", fmt.Sprintf("vhost-vsock-pci,guest-cid=%d", cfg.VsockCID)) } - if cfg.VGPUDevicePath != "" { - args = append(args, "-device", fmt.Sprintf("vfio-pci,sysfsdev=%s", cfg.VGPUDevicePath)) - } - - // Whole-device PCI passthrough (vGPU attaches via VGPUDevicePath above) + // Whole-device PCI passthrough (vGPU attaches via VGPUDevicePath below) for _, devicePath := range cfg.PCIDevices { var deviceArg string if strings.HasPrefix(devicePath, "/sys/bus/pci/devices/") { @@ -101,6 +97,10 @@ func BuildArgs(cfg hypervisor.VMConfig) []string { args = append(args, "-device", deviceArg) } + if cfg.VGPUDevicePath != "" { + args = append(args, "-device", fmt.Sprintf("vfio-pci,sysfsdev=%s", cfg.VGPUDevicePath)) + } + // Serial console output to file. Use a chardev with append=on so QEMU // opens the file with O_APPEND. Without it, QEMU writes at its internal // fd offset; if the file is externally truncated (e.g. log rotation via diff --git a/lib/hypervisor/qemu/config_test.go b/lib/hypervisor/qemu/config_test.go index e90a511c0..e49b65f28 100644 --- a/lib/hypervisor/qemu/config_test.go +++ b/lib/hypervisor/qemu/config_test.go @@ -142,6 +142,29 @@ func TestBuildArgs_VGPU(t *testing.T) { } } +func TestBuildArgs_VGPUAfterPCIDevices(t *testing.T) { + args := BuildArgs(hypervisor.VMConfig{ + VCPUs: 1, + MemoryBytes: 512 * 1024 * 1024, + PCIDevices: []string{"0000:01:00.0"}, + VGPUDevicePath: "/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123", + }) + + pciDeviceIndex := -1 + vgpuDeviceIndex := -1 + for i, arg := range args { + switch arg { + case "vfio-pci,host=0000:01:00.0": + pciDeviceIndex = i + case "vfio-pci,sysfsdev=/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123": + vgpuDeviceIndex = i + } + } + + assert.Greater(t, pciDeviceIndex, -1) + assert.Greater(t, vgpuDeviceIndex, pciDeviceIndex) +} + func TestBuildArgs_PCIPassthrough(t *testing.T) { cfg := hypervisor.VMConfig{ VCPUs: 1, From 74df6f0fa6eff46a0b71943f28e9e375df4eff3d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Thu, 6 Aug 2026 21:47:23 +0000 Subject: [PATCH 13/18] Preserve mdev-era lifecycle semantics in the refactor Stop and delete keep best-effort vGPU release (log and continue, metadata always cleared) and start no longer releases a stale stored assignment. Those behavior changes belong to the lifecycle-hardening layer, not this behavior-preserving refactor. --- lib/instances/delete.go | 4 ++-- lib/instances/lifecycle_noop_test.go | 19 ------------------- lib/instances/start.go | 5 ----- lib/instances/stop.go | 6 ++++-- 4 files changed, 6 insertions(+), 28 deletions(-) diff --git a/lib/instances/delete.go b/lib/instances/delete.go index d89f981e2..c5e2151fc 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -172,8 +172,8 @@ func (m *manager) deleteInstanceWithOptions( // 7c. Release the vGPU assignment if present. if err := releaseStoredVGPU(ctx, stored); err != nil { - log.ErrorContext(ctx, "failed to destroy vGPU; retaining instance metadata", "instance_id", id, "error", err) - return fmt.Errorf("destroy vGPU: %w", err) + // Log error but continue with cleanup + log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "error", err) } // 8. Delete all instance data diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go index 08205a544..5ca7515fa 100644 --- a/lib/instances/lifecycle_noop_test.go +++ b/lib/instances/lifecycle_noop_test.go @@ -9,7 +9,6 @@ import ( "testing" "time" - "github.com/kernel/hypeman/lib/devices" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/assert" @@ -148,24 +147,6 @@ func TestLifecycleNoopStandbyWithOptionsStillRejectsStandbyInstance(t *testing.T assertNoLifecycleEvent(t, events) } -func TestDeleteRetainsMetadataWhenVGPUReleaseFails(t *testing.T) { - m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC()) - meta, err := m.loadMetadata(id) - require.NoError(t, err) - meta.GPUProfile = "NVIDIA L40S-2Q" - meta.GPUFramework = devices.VGPUFramework("future-framework") - meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4" - require.NoError(t, m.saveMetadata(meta)) - - err = m.DeleteInstance(context.Background(), id) - require.Error(t, err) - - stored, err := m.loadMetadata(id) - require.NoError(t, err) - assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework) - assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath) -} - func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) { t.Helper() diff --git a/lib/instances/start.go b/lib/instances/start.go index 2ac623251..eea23bd6a 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,11 +48,6 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } - if err := releaseStoredVGPU(ctx, stored); err != nil { - log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err) - return nil, fmt.Errorf("release stale vGPU before start: %w", err) - } - // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" diff --git a/lib/instances/stop.go b/lib/instances/stop.go index de7d6cff2..a6691126d 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -262,9 +262,11 @@ func (m *manager) stopInstance( } } - // 7. Release the vGPU assignment if present. + // 7. Release the vGPU assignment if present (frees the vGPU slot for other VMs). if err := releaseStoredVGPU(ctx, stored); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "error", err) + // Log error but continue - vGPU cleanup is best-effort + log.WarnContext(ctx, "failed to destroy vGPU on stop", "instance_id", id, "error", err) + clearStoredVGPUDevice(stored) } // 8. Always remove stale runtime sockets after process exit. From 1abcce64c6023e2d064977f8cf187c64276baf8d Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Fri, 7 Aug 2026 20:35:25 +0000 Subject: [PATCH 14/18] Keep the mdev create error text in the refactor --- lib/devices/GPU.md | 15 --------------- lib/instances/create.go | 2 +- lib/instances/create_mdev_test.go | 2 +- 3 files changed, 2 insertions(+), 17 deletions(-) diff --git a/lib/devices/GPU.md b/lib/devices/GPU.md index 4fd6bddee..54a19c472 100644 --- a/lib/devices/GPU.md +++ b/lib/devices/GPU.md @@ -235,21 +235,6 @@ To upgrade the NVIDIA driver version: - Run GPU passthrough E2E tests - Verify with real CUDA workloads (e.g., ollama inference) -## Rolling Back vGPU Changes - -Before downgrading Hypeman or the host to a version that does not support the active vGPU framework: - -1. Stop or delete all vGPU instances while the current Hypeman version can release their assignments. -2. Confirm `/resources` reports `used_slots: 0`. -3. Confirm no assignments remain in either framework: - ```bash - test -z "$(find /sys/bus/mdev/devices -mindepth 1 -maxdepth 1 2>/dev/null)" - find /sys/bus/pci/devices -path '*/nvidia/current_vgpu_type' -exec grep -H -v '^0$' {} + - ``` -4. Downgrade only after both checks are clean. - -If assignment cleanup fails, Hypeman retains the instance metadata so a compatible version can retry it. Do not remove that metadata manually while the assignment remains active. - ## Troubleshooting ### No GPU shown in /resources diff --git a/lib/instances/create.go b/lib/instances/create.go index ff6707d53..42afe3275 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -56,7 +56,7 @@ func wrapCreateVGPUErr(profile string, err error) error { if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) { return fmt.Errorf("%w: %w", ErrInvalidRequest, err) } - return fmt.Errorf("create vGPU for profile %s: %w", profile, err) + return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err) } // generateVsockCID converts first 8 chars of instance ID to a unique CID diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go index 05b3e9d8c..e6e4f55c5 100644 --- a/lib/instances/create_mdev_test.go +++ b/lib/instances/create_mdev_test.go @@ -63,7 +63,7 @@ func TestWrapCreateVGPUErr(t *testing.T) { { name: "other vGPU error", err: errors.New("boom"), - wantMessage: "create vGPU for profile profile: boom", + wantMessage: "create vGPU mdev for profile profile: boom", }, } { t.Run(tc.name, func(t *testing.T) { From ca1c510bcf672ab5fdb8cdc6fdd5e925e67af183 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Sun, 9 Aug 2026 07:34:16 +0000 Subject: [PATCH 15/18] Restore mdev start error text --- lib/instances/start.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/instances/start.go b/lib/instances/start.go index eea23bd6a..36cc311f3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -146,11 +146,11 @@ func (m *manager) startInstance( // 4b. Recreate the vGPU if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { - log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile) + log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile) device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id) if err != nil { log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err) - return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err) + return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) } setStoredVGPUDevice(stored, device) // Add vGPU cleanup to stack From 5f9e4fd9e70a33db8d9efd470365c4a0ea3386c1 Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 18:16:40 +0000 Subject: [PATCH 16/18] Restrict vGPU instances to QEMU --- lib/hypervisor/cloudhypervisor/config.go | 12 +++--------- lib/hypervisor/cloudhypervisor/config_test.go | 11 ----------- lib/instances/create.go | 5 +++++ lib/instances/start.go | 5 +++++ lib/instances/vgpu.go | 9 +++++++++ lib/instances/vgpu_test.go | 8 ++++++++ 6 files changed, 30 insertions(+), 20 deletions(-) diff --git a/lib/hypervisor/cloudhypervisor/config.go b/lib/hypervisor/cloudhypervisor/config.go index c5f506af9..e9f91fe4a 100644 --- a/lib/hypervisor/cloudhypervisor/config.go +++ b/lib/hypervisor/cloudhypervisor/config.go @@ -125,16 +125,10 @@ func ToVMConfig(cfg hypervisor.VMConfig) vmm.VmConfig { } // Device passthrough configuration - devicePaths := make([]string, 0, len(cfg.PCIDevices)+1) - devicePaths = append(devicePaths, cfg.PCIDevices...) - if cfg.VGPUDevicePath != "" { - devicePaths = append(devicePaths, cfg.VGPUDevicePath) - } - var devices *[]vmm.DeviceConfig - if len(devicePaths) > 0 { - deviceConfigs := make([]vmm.DeviceConfig, 0, len(devicePaths)) - for _, path := range devicePaths { + if len(cfg.PCIDevices) > 0 { + deviceConfigs := make([]vmm.DeviceConfig, 0, len(cfg.PCIDevices)) + for _, path := range cfg.PCIDevices { deviceConfigs = append(deviceConfigs, vmm.DeviceConfig{ Path: path, }) diff --git a/lib/hypervisor/cloudhypervisor/config_test.go b/lib/hypervisor/cloudhypervisor/config_test.go index 235be3906..b5cdb96e9 100644 --- a/lib/hypervisor/cloudhypervisor/config_test.go +++ b/lib/hypervisor/cloudhypervisor/config_test.go @@ -8,17 +8,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestToVMConfig_VGPU(t *testing.T) { - t.Parallel() - - path := "/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123" - vmCfg := ToVMConfig(hypervisor.VMConfig{VGPUDevicePath: path}) - - require.NotNil(t, vmCfg.Devices) - require.Len(t, *vmCfg.Devices, 1) - assert.Equal(t, path, (*vmCfg.Devices)[0].Path) -} - func TestToVMConfig_GuestMemoryBalloon(t *testing.T) { cfg := hypervisor.VMConfig{ VCPUs: 1, diff --git a/lib/instances/create.go b/lib/instances/create.go index 42afe3275..5eccb9556 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -103,6 +103,11 @@ func (m *manager) createInstance( if hvType == "" { hvType = m.defaultHypervisor } + if req.GPU != nil && req.GPU.Profile != "" { + if err := validateVGPUHypervisor(hvType); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) + } + } // 2. Validate image exists and is ready; auto-pull if not found log.DebugContext(ctx, "validating image", "image", req.Image) diff --git a/lib/instances/start.go b/lib/instances/start.go index 36cc311f3..05cd56fd5 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,6 +48,11 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + if stored.GPUProfile != "" { + if err := validateVGPUHypervisor(stored.HypervisorType); err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidState, err) + } + } // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index c2294ac53..45d8cfdb1 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,11 +2,20 @@ package instances import ( "context" + "fmt" "path/filepath" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" ) +func validateVGPUHypervisor(hvType hypervisor.Type) error { + if hvType != hypervisor.TypeQEMU { + return fmt.Errorf("vGPU is only supported with qemu, got %s", hvType) + } + return nil +} + func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 6f2c46819..904278dbe 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -5,9 +5,17 @@ import ( "testing" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/stretchr/testify/assert" ) +func TestValidateVGPUHypervisor(t *testing.T) { + t.Parallel() + + assert.NoError(t, validateVGPUHypervisor(hypervisor.TypeQEMU)) + assert.EqualError(t, validateVGPUHypervisor(hypervisor.TypeCloudHypervisor), "vGPU is only supported with qemu, got cloud-hypervisor") +} + func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From 33140f233d1db8b5b5b3241b32223a639e68ae9a Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:00:43 +0000 Subject: [PATCH 17/18] Preserve mdev hypervisor behavior --- lib/hypervisor/cloudhypervisor/config.go | 11 +++++++++-- lib/hypervisor/cloudhypervisor/config_test.go | 10 ++++++++++ lib/instances/create.go | 6 ------ lib/instances/start.go | 5 ----- lib/instances/vgpu.go | 9 --------- lib/instances/vgpu_test.go | 8 -------- 6 files changed, 19 insertions(+), 30 deletions(-) diff --git a/lib/hypervisor/cloudhypervisor/config.go b/lib/hypervisor/cloudhypervisor/config.go index e9f91fe4a..ca3d98a55 100644 --- a/lib/hypervisor/cloudhypervisor/config.go +++ b/lib/hypervisor/cloudhypervisor/config.go @@ -126,13 +126,20 @@ func ToVMConfig(cfg hypervisor.VMConfig) vmm.VmConfig { // Device passthrough configuration var devices *[]vmm.DeviceConfig - if len(cfg.PCIDevices) > 0 { - deviceConfigs := make([]vmm.DeviceConfig, 0, len(cfg.PCIDevices)) + deviceCount := len(cfg.PCIDevices) + if cfg.VGPUDevicePath != "" { + deviceCount++ + } + if deviceCount > 0 { + deviceConfigs := make([]vmm.DeviceConfig, 0, deviceCount) for _, path := range cfg.PCIDevices { deviceConfigs = append(deviceConfigs, vmm.DeviceConfig{ Path: path, }) } + if cfg.VGPUDevicePath != "" { + deviceConfigs = append(deviceConfigs, vmm.DeviceConfig{Path: cfg.VGPUDevicePath}) + } devices = &deviceConfigs } diff --git a/lib/hypervisor/cloudhypervisor/config_test.go b/lib/hypervisor/cloudhypervisor/config_test.go index b5cdb96e9..be39d13af 100644 --- a/lib/hypervisor/cloudhypervisor/config_test.go +++ b/lib/hypervisor/cloudhypervisor/config_test.go @@ -8,6 +8,16 @@ import ( "github.com/stretchr/testify/require" ) +func TestToVMConfigIncludesVGPUDevice(t *testing.T) { + path := "/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123" + + vmCfg := ToVMConfig(hypervisor.VMConfig{VGPUDevicePath: path}) + + require.NotNil(t, vmCfg.Devices) + require.Len(t, *vmCfg.Devices, 1) + assert.Equal(t, path, (*vmCfg.Devices)[0].Path) +} + func TestToVMConfig_GuestMemoryBalloon(t *testing.T) { cfg := hypervisor.VMConfig{ VCPUs: 1, diff --git a/lib/instances/create.go b/lib/instances/create.go index 5eccb9556..d7ea02c9d 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -103,12 +103,6 @@ func (m *manager) createInstance( if hvType == "" { hvType = m.defaultHypervisor } - if req.GPU != nil && req.GPU.Profile != "" { - if err := validateVGPUHypervisor(hvType); err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) - } - } - // 2. Validate image exists and is ready; auto-pull if not found log.DebugContext(ctx, "validating image", "image", req.Image) imageCtx, imageSpanEnd := m.startLifecycleStep(ctx, "resolve_image", diff --git a/lib/instances/start.go b/lib/instances/start.go index 05cd56fd5..36cc311f3 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,11 +48,6 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } - if stored.GPUProfile != "" { - if err := validateVGPUHypervisor(stored.HypervisorType); err != nil { - return nil, fmt.Errorf("%w: %w", ErrInvalidState, err) - } - } // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go index 45d8cfdb1..c2294ac53 100644 --- a/lib/instances/vgpu.go +++ b/lib/instances/vgpu.go @@ -2,20 +2,11 @@ package instances import ( "context" - "fmt" "path/filepath" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" ) -func validateVGPUHypervisor(hvType hypervisor.Type) error { - if hvType != hypervisor.TypeQEMU { - return fmt.Errorf("vGPU is only supported with qemu, got %s", hvType) - } - return nil -} - func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) { stored.GPUFramework = device.Framework stored.GPUDevicePath = device.SysfsPath diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go index 904278dbe..6f2c46819 100644 --- a/lib/instances/vgpu_test.go +++ b/lib/instances/vgpu_test.go @@ -5,17 +5,9 @@ import ( "testing" "github.com/kernel/hypeman/lib/devices" - "github.com/kernel/hypeman/lib/hypervisor" "github.com/stretchr/testify/assert" ) -func TestValidateVGPUHypervisor(t *testing.T) { - t.Parallel() - - assert.NoError(t, validateVGPUHypervisor(hypervisor.TypeQEMU)) - assert.EqualError(t, validateVGPUHypervisor(hypervisor.TypeCloudHypervisor), "vGPU is only supported with qemu, got cloud-hypervisor") -} - func TestStoredVGPUDevicePath(t *testing.T) { t.Parallel() From c819459538ca2a8262d47a670737cbdabc5f859c Mon Sep 17 00:00:00 2001 From: yummybomb <19238148+yummybomb@users.noreply.github.com> Date: Mon, 10 Aug 2026 20:13:01 +0000 Subject: [PATCH 18/18] Restore vGPU lifecycle log statements and spacing from the pre-refactor code --- lib/instances/create.go | 5 ++++- lib/instances/delete.go | 9 ++++++--- lib/instances/start.go | 5 ++++- lib/instances/stop.go | 11 +++++++---- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/lib/instances/create.go b/lib/instances/create.go index d7ea02c9d..7127b8d2c 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -103,6 +103,7 @@ func (m *manager) createInstance( if hvType == "" { hvType = m.defaultHypervisor } + // 2. Validate image exists and is ready; auto-pull if not found log.DebugContext(ctx, "validating image", "image", req.Image) imageCtx, imageSpanEnd := m.startLifecycleStep(ctx, "resolve_image", @@ -294,11 +295,13 @@ func (m *manager) createInstance( gpuFramework = gpuDevice.Framework gpuDevicePath = gpuDevice.SysfsPath gpuMdevUUID = gpuDevice.MdevUUID + log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID) // Add vGPU cleanup to stack cu.Add(func() { + log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID) if err := devices.DestroyVGPU(ctx, gpuDevice.Framework, gpuDevice.SysfsPath, gpuDevice.MdevUUID); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err) + log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err) } }) } diff --git a/lib/instances/delete.go b/lib/instances/delete.go index c5e2151fc..586e9ad81 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -171,9 +171,12 @@ func (m *manager) deleteInstanceWithOptions( } // 7c. Release the vGPU assignment if present. - if err := releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue with cleanup - log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "error", err) + if storedVGPUDevicePath(stored) != "" { + log.InfoContext(ctx, "destroying vGPU", "instance_id", id, "uuid", stored.GPUMdevUUID) + if err := releaseStoredVGPU(ctx, stored); err != nil { + // Log error but continue with cleanup + log.WarnContext(ctx, "failed to destroy vGPU, continuing with cleanup", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) + } } // 8. Delete all instance data diff --git a/lib/instances/start.go b/lib/instances/start.go index 36cc311f3..eec4b8e56 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -48,6 +48,7 @@ func (m *manager) startInstance( log.ErrorContext(ctx, "invalid state for start", "instance_id", id, "state", inst.State) return nil, fmt.Errorf("%w: cannot start from state %s, must be Stopped", ErrInvalidState, inst.State) } + // 2a. Clear stale exit info from previous run and apply command overrides stored.ExitCode = nil stored.ExitMessage = "" @@ -153,10 +154,12 @@ func (m *manager) startInstance( return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err) } setStoredVGPUDevice(stored, device) + log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", stored.GPUProfile, "uuid", device.MdevUUID) // Add vGPU cleanup to stack cu.Add(func() { + log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID) if err := devices.DestroyVGPU(ctx, device.Framework, device.SysfsPath, device.MdevUUID); err != nil { - log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err) + log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", device.MdevUUID, "error", err) } }) } diff --git a/lib/instances/stop.go b/lib/instances/stop.go index a6691126d..ec03a3fc1 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -263,10 +263,13 @@ func (m *manager) stopInstance( } // 7. Release the vGPU assignment if present (frees the vGPU slot for other VMs). - if err := releaseStoredVGPU(ctx, stored); err != nil { - // Log error but continue - vGPU cleanup is best-effort - log.WarnContext(ctx, "failed to destroy vGPU on stop", "instance_id", id, "error", err) - clearStoredVGPUDevice(stored) + if storedVGPUDevicePath(stored) != "" { + log.InfoContext(ctx, "destroying vGPU on stop", "instance_id", id, "uuid", stored.GPUMdevUUID) + if err := releaseStoredVGPU(ctx, stored); err != nil { + // Log error but continue - vGPU cleanup is best-effort + log.WarnContext(ctx, "failed to destroy vGPU on stop", "instance_id", id, "uuid", stored.GPUMdevUUID, "error", err) + clearStoredVGPUDevice(stored) + } } // 8. Always remove stale runtime sockets after process exit.