Abstract vGPU devices behind a framework dispatch

lib/devices/mdev_darwin.go

@@ -2,7 +2,10 @@
 
 package devices
 
-import "context"
+import (
+	"context"
+	"fmt"
+)
 
 // SetGPUProfileCacheTTL is a no-op on macOS.
 func SetGPUProfileCacheTTL(ttl string) {
@@ -30,6 +33,10 @@ func ListMdevDevices() ([]MdevDevice, error) {
 	return []MdevDevice{}, nil
 }
 
+func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) {
+	return nil, ErrVGPUNotSupportedOnMacOS
+}
+
 // CreateMdev returns an error on macOS as mdev is not supported.
 func CreateMdev(ctx context.Context, profileName, instanceID string) (*MdevDevice, error) {
 	return nil, ErrVGPUNotSupportedOnMacOS
@@ -45,6 +52,13 @@ func IsMdevInUse(mdevUUID string) bool {
 	return false
 }
 
+func DestroyVGPU(ctx context.Context, framework VGPUFramework, devicePath, mdevUUID string) error {
+	if framework != VGPUFrameworkNone && framework != VGPUFrameworkMdev {
+		return fmt.Errorf("unknown vGPU framework %q", framework)
+	}
+	return nil
+}
+
 // ReconcileMdevs is a no-op on macOS.
 func ReconcileMdevs(ctx context.Context, instanceInfos []MdevReconcileInfo) error {
 	return nil

lib/devices/mdev_linux.go

@@ -126,7 +126,7 @@ func DiscoverVFs() ([]VirtualFunction, error) {
 		vfs = append(vfs, VirtualFunction{
 			PCIAddress: vfAddr,
 			ParentGPU:  parentGPU,
-			HasMdev:    hasMdev,
+			Allocated:  hasMdev,
 		})
 	}
 
@@ -253,7 +253,7 @@ func countAvailableVFsForProfilesParallel(vfs []VirtualFunction, profiles []prof
 	// Group free VFs by parent GPU (done once, shared by all goroutines)
 	freeVFsByParent := make(map[string][]VirtualFunction)
 	for _, vf := range vfs {
-		if vf.HasMdev {
+		if vf.Allocated {
 			continue
 		}
 		freeVFsByParent[vf.ParentGPU] = append(freeVFsByParent[vf.ParentGPU], vf)
@@ -453,7 +453,7 @@ func selectLeastLoadedVF(ctx context.Context, vfs []VirtualFunction, profileType
 	allGPUs := make(map[string]bool)
 	for _, vf := range vfs {
 		allGPUs[vf.ParentGPU] = true
-		if !vf.HasMdev {
+		if !vf.Allocated {
 			freeVFsByGPU[vf.ParentGPU] = append(freeVFsByGPU[vf.ParentGPU], vf)
 		}
 	}

lib/devices/types.go

@@ -60,7 +60,12 @@ func ValidateDeviceName(name string) bool {
 // GPUMode represents the host's GPU configuration mode
 type GPUMode string
 
+type VGPUFramework string
+
 const (
+	VGPUFrameworkNone VGPUFramework = ""
+	VGPUFrameworkMdev VGPUFramework = "mdev"
+
 	// GPUModePassthrough indicates whole GPU VFIO passthrough
 	GPUModePassthrough GPUMode = "passthrough"
 	// GPUModeVGPU indicates SR-IOV + mdev based vGPU
@@ -73,7 +78,16 @@ const (
 type VirtualFunction struct {
 	PCIAddress string `json:"pci_address"` // e.g., "0000:82:00.4"
 	ParentGPU  string `json:"parent_gpu"`  // e.g., "0000:82:00.0"
-	HasMdev    bool   `json:"has_mdev"`    // true if an mdev is created on this VF
+	Allocated  bool   `json:"allocated"`   // true if a vGPU is assigned to this VF
+}
+
+type VGPUDevice struct {
+	Framework   VGPUFramework
+	VFAddress   string
+	ProfileType string
+	ProfileName string
+	SysfsPath   string
+	MdevUUID    string
 }
 
 // MdevDevice represents an active mediated device (vGPU instance)

lib/devices/vgpu_linux.go

@@ -0,0 +1,37 @@
+//go:build linux
+
+package devices
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+)
+
+func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) {
+	mdev, err := CreateMdev(ctx, profileName, instanceID)
+	if err != nil {
+		return nil, err
+	}
+	return &VGPUDevice{
+		Framework:   VGPUFrameworkMdev,
+		VFAddress:   mdev.VFAddress,
+		ProfileType: mdev.ProfileType,
+		ProfileName: mdev.ProfileName,
+		SysfsPath:   mdev.SysfsPath,
+		MdevUUID:    mdev.UUID,
+	}, nil
+}
+
+func DestroyVGPU(ctx context.Context, framework VGPUFramework, devicePath, mdevUUID string) error {
+	if framework != VGPUFrameworkNone && framework != VGPUFrameworkMdev {
+		return fmt.Errorf("unknown vGPU framework %q", framework)
+	}
+	if mdevUUID == "" {
+		if devicePath == "" {
+			return nil
+		}
+		mdevUUID = filepath.Base(devicePath)
+	}
+	return DestroyMdev(ctx, mdevUUID)
+}

lib/hypervisor/cloudhypervisor/config.go

@@ -126,13 +126,20 @@ func ToVMConfig(cfg hypervisor.VMConfig) vmm.VmConfig {
 
 	// Device passthrough configuration
 	var devices *[]vmm.DeviceConfig
-	if len(cfg.PCIDevices) > 0 {
-		deviceConfigs := make([]vmm.DeviceConfig, 0, len(cfg.PCIDevices))
+	deviceCount := len(cfg.PCIDevices)
+	if cfg.VGPUDevicePath != "" {
+		deviceCount++
+	}
+	if deviceCount > 0 {
+		deviceConfigs := make([]vmm.DeviceConfig, 0, deviceCount)
 		for _, path := range cfg.PCIDevices {
 			deviceConfigs = append(deviceConfigs, vmm.DeviceConfig{
 				Path: path,
 			})
 		}
+		if cfg.VGPUDevicePath != "" {
+			deviceConfigs = append(deviceConfigs, vmm.DeviceConfig{Path: cfg.VGPUDevicePath})
+		}
 		devices = &deviceConfigs
 	}
 

lib/hypervisor/cloudhypervisor/config_test.go

@@ -8,6 +8,16 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestToVMConfigIncludesVGPUDevice(t *testing.T) {
+	path := "/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123"
+
+	vmCfg := ToVMConfig(hypervisor.VMConfig{VGPUDevicePath: path})
+
+	require.NotNil(t, vmCfg.Devices)
+	require.Len(t, *vmCfg.Devices, 1)
+	assert.Equal(t, path, (*vmCfg.Devices)[0].Path)
+}
+
 func TestToVMConfig_GuestMemoryBalloon(t *testing.T) {
 	cfg := hypervisor.VMConfig{
 		VCPUs:       1,

lib/hypervisor/config.go

@@ -24,7 +24,8 @@ type VMConfig struct {
 	VsockSocket string
 
 	// PCI device passthrough (GPU, etc.)
-	PCIDevices []string
+	PCIDevices     []string
+	VGPUDevicePath string
 
 	// Boot configuration
 	KernelPath string

lib/hypervisor/qemu/config.go

@@ -94,13 +94,10 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string {
 		args = append(args, "-device", fmt.Sprintf("%s,guest-cid=%d", virtioDevice(microvm, "vhost-vsock"), cfg.VsockCID))
 	}
 
-	// PCI device passthrough (GPU, mdev vGPU, etc.)
+	// Whole-device PCI passthrough (vGPU attaches via VGPUDevicePath below)
 	for _, devicePath := range cfg.PCIDevices {
 		var deviceArg string
-		if strings.HasPrefix(devicePath, "/sys/bus/mdev/devices/") {
-			// mdev device (vGPU) - use sysfsdev parameter
-			deviceArg = fmt.Sprintf("vfio-pci,sysfsdev=%s", devicePath)
-		} else if strings.HasPrefix(devicePath, "/sys/bus/pci/devices/") {
+		if strings.HasPrefix(devicePath, "/sys/bus/pci/devices/") {
 			// Full sysfs path for regular PCI device - extract the PCI address
 			// Using filepath.Base is more robust than manual string splitting
 			pciAddr := filepath.Base(strings.TrimSuffix(devicePath, "/"))
@@ -112,6 +109,10 @@ func buildArgs(cfg hypervisor.VMConfig, machine MachineType) []string {
 		args = append(args, "-device", deviceArg)
 	}
 
+	if cfg.VGPUDevicePath != "" {
+		args = append(args, "-device", fmt.Sprintf("vfio-pci,sysfsdev=%s", cfg.VGPUDevicePath))
+	}
+
 	// Serial console output to file. Use a chardev with append=on so QEMU
 	// opens the file with O_APPEND. Without it, QEMU writes at its internal
 	// fd offset; if the file is externally truncated (e.g. log rotation via

lib/hypervisor/qemu/config_test.go

@@ -123,6 +123,49 @@ func TestBuildArgs_Vsock(t *testing.T) {
 	assert.Contains(t, args, "vhost-vsock-pci,guest-cid=123")
 }
 
+func TestBuildArgs_VGPU(t *testing.T) {
+	t.Parallel()
+
+	for _, path := range []string{
+		"/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123",
+		"/sys/bus/pci/devices/0000:82:00.4",
+	} {
+		path := path
+		t.Run(path, func(t *testing.T) {
+			t.Parallel()
+			args := BuildArgs(hypervisor.VMConfig{
+				VCPUs:          1,
+				MemoryBytes:    512 * 1024 * 1024,
+				VGPUDevicePath: path,
+			})
+			assert.Contains(t, args, "vfio-pci,sysfsdev="+path)
+		})
+	}
+}
+
+func TestBuildArgs_VGPUAfterPCIDevices(t *testing.T) {
+	args := BuildArgs(hypervisor.VMConfig{
+		VCPUs:          1,
+		MemoryBytes:    512 * 1024 * 1024,
+		PCIDevices:     []string{"0000:01:00.0"},
+		VGPUDevicePath: "/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123",
+	})
+
+	pciDeviceIndex := -1
+	vgpuDeviceIndex := -1
+	for i, arg := range args {
+		switch arg {
+		case "vfio-pci,host=0000:01:00.0":
+			pciDeviceIndex = i
+		case "vfio-pci,sysfsdev=/sys/bus/mdev/devices/aa618089-8b16-4d01-a136-25a0f3c73123":
+			vgpuDeviceIndex = i
+		}
+	}
+
+	assert.Greater(t, pciDeviceIndex, -1)
+	assert.Greater(t, vgpuDeviceIndex, pciDeviceIndex)
+}
+
 func TestBuildArgs_PCIPassthrough(t *testing.T) {
 	cfg := hypervisor.VMConfig{
 		VCPUs:       1,

lib/instances/create.go

@@ -52,7 +52,7 @@ var systemDirectories = []string{
 	"/var",
 }
 
-func wrapCreateMdevErr(profile string, err error) error {
+func wrapCreateVGPUErr(profile string, err error) error {
 	if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) {
 		return fmt.Errorf("%w: %w", ErrInvalidRequest, err)
 	}
@@ -257,7 +257,10 @@ func (m *manager) createInstance(
 	// whatever devices have been attached when cleanup runs.
 	var attachedDeviceIDs []string
 	var resolvedDeviceIDs []string
+	var gpuDevice *devices.VGPUDevice
 	var gpuProfile string
+	var gpuFramework devices.VGPUFramework
+	var gpuDevicePath string
 	var gpuMdevUUID string
 
 	// Setup cleanup stack early so device attachment errors trigger cleanup
@@ -277,23 +280,25 @@ func (m *manager) createInstance(
 		})
 	}
 
-	// Handle vGPU profile request - create mdev device
+	// Handle vGPU profile request
 	if req.GPU != nil && req.GPU.Profile != "" {
-		log.InfoContext(ctx, "creating vGPU mdev", "instance_id", id, "profile", req.GPU.Profile)
-		mdev, err := devices.CreateMdev(ctx, req.GPU.Profile, id)
+		log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile)
+		gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id)
 		if err != nil {
-			log.ErrorContext(ctx, "failed to create mdev", "profile", req.GPU.Profile, "error", err)
-			return nil, wrapCreateMdevErr(req.GPU.Profile, err)
+			log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err)
+			return nil, wrapCreateVGPUErr(req.GPU.Profile, err)
 		}
-		gpuProfile = req.GPU.Profile
-		gpuMdevUUID = mdev.UUID
-		log.InfoContext(ctx, "created vGPU mdev", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID)
+		gpuProfile = gpuDevice.ProfileName
+		gpuFramework = gpuDevice.Framework
+		gpuDevicePath = gpuDevice.SysfsPath
+		gpuMdevUUID = gpuDevice.MdevUUID
+		log.InfoContext(ctx, "created vGPU", "instance_id", id, "profile", gpuProfile, "uuid", gpuMdevUUID)
 
-		// Add mdev cleanup to stack
+		// Add vGPU cleanup to stack
 		cu.Add(func() {
-			log.DebugContext(ctx, "destroying mdev on cleanup", "instance_id", id, "uuid", gpuMdevUUID)
-			if err := devices.DestroyMdev(ctx, gpuMdevUUID); err != nil {
-				log.WarnContext(ctx, "failed to destroy mdev on cleanup", "instance_id", id, "uuid", gpuMdevUUID, "error", err)
+			log.DebugContext(ctx, "destroying vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID)
+			if err := devices.DestroyVGPU(ctx, gpuDevice.Framework, gpuDevice.SysfsPath, gpuDevice.MdevUUID); err != nil {
+				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "uuid", gpuDevice.MdevUUID, "error", err)
 			}
 		})
 	}
@@ -361,6 +366,8 @@ func (m *manager) createInstance(
 		VsockSocket:              vsockSocket,
 		Devices:                  resolvedDeviceIDs,
 		GPUProfile:               gpuProfile,
+		GPUFramework:             gpuFramework,
+		GPUDevicePath:            gpuDevicePath,
 		GPUMdevUUID:              gpuMdevUUID,
 		Entrypoint:               req.Entrypoint,
 		Cmd:                      req.Cmd,
@@ -898,12 +905,6 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima
 		}
 	}
 
-	// Add vGPU mdev device if configured
-	if inst.GPUMdevUUID != "" {
-		mdevPath := filepath.Join("/sys/bus/mdev/devices", inst.GPUMdevUUID)
-		pciDevices = append(pciDevices, mdevPath)
-	}
-
 	// Build topology if available
 	var topology *hypervisor.CPUTopology
 	if hostTopo := calculateGuestTopology(inst.Vcpus, m.hostTopology); hostTopo != nil {
@@ -923,21 +924,22 @@ func (m *manager) buildHypervisorConfig(ctx context.Context, inst *Instance, ima
 	}
 
 	return hypervisor.VMConfig{
-		VCPUs:         inst.Vcpus,
-		MemoryBytes:   inst.Size,
-		HotplugBytes:  inst.HotplugSize,
-		Topology:      topology,
-		GuestMemory:   m.guestMemoryConfig(),
-		Disks:         disks,
-		Networks:      networks,
-		SerialLogPath: m.paths.InstanceAppLog(inst.Id),
-		VsockCID:      inst.VsockCID,
-		VsockSocket:   inst.VsockSocket,
-		PCIDevices:    pciDevices,
-		KernelPath:    kernelPath,
-		InitrdPath:    initrdPath,
-		KernelArgs:    m.kernelArgs(inst.HypervisorType),
-		EnableRosetta: inst.EnableRosetta,
+		VCPUs:          inst.Vcpus,
+		MemoryBytes:    inst.Size,
+		HotplugBytes:   inst.HotplugSize,
+		Topology:       topology,
+		GuestMemory:    m.guestMemoryConfig(),
+		Disks:          disks,
+		Networks:       networks,
+		SerialLogPath:  m.paths.InstanceAppLog(inst.Id),
+		VsockCID:       inst.VsockCID,
+		VsockSocket:    inst.VsockSocket,
+		PCIDevices:     pciDevices,
+		VGPUDevicePath: storedVGPUDevicePath(&inst.StoredMetadata),
+		KernelPath:     kernelPath,
+		InitrdPath:     initrdPath,
+		KernelArgs:     m.kernelArgs(inst.HypervisorType),
+		EnableRosetta:  inst.EnableRosetta,
 	}, nil
 }