Report vendor VFIO profile availability per free VF

cmd/api/api/instances.go

@@ -343,7 +343,20 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 
 	inst, err := s.InstanceManager.CreateInstance(ctx, domainReq)
 	if err != nil {
+		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
+		// Checked first: it wraps the original create error, so a later
+		// errors.Is case would match the cause and hide the pending vGPU cleanup.
+		case errors.As(err, &vgpuPending):
+			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
+			message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID)
+			if !vgpuPending.Retained {
+				message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID)
+			}
+			return oapi.CreateInstance500JSONResponse{
+				Code:    "vgpu_cleanup_pending",
+				Message: message,
+			}, nil
 		case errors.Is(err, instances.ErrImageNotReady):
 			return oapi.CreateInstance400JSONResponse{
 				Code:    "image_not_ready",

cmd/api/api/instances_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/kernel/hypeman/lib/instances"
 	"github.com/kernel/hypeman/lib/instances/phasetracking"
 	mw "github.com/kernel/hypeman/lib/middleware"
+	"github.com/kernel/hypeman/lib/network"
 	"github.com/kernel/hypeman/lib/oapi"
 	"github.com/kernel/hypeman/lib/paths"
 	restartpolicy "github.com/kernel/hypeman/lib/restart-policy"
@@ -46,6 +47,59 @@ func TestGetInstance_NotFound(t *testing.T) {
 	require.Error(t, err)
 }
 
+type createErrorInstanceManager struct {
+	instances.Manager
+	err error
+}
+
+func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) {
+	return nil, m.err
+}
+
+// A retained-assignment error must win over the mapping of the create error
+// it wraps, or the response omits the instance the caller has to delete.
+func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
+		InstanceID: "inst-1",
+		Retained:   true,
+		Err:        network.ErrNameExists,
+	}}
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+		Body: &oapi.CreateInstanceRequest{Image: "test-image"},
+	})
+	require.NoError(t, err)
+
+	pending, ok := resp.(oapi.CreateInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "inst-1")
+	assert.Contains(t, pending.Message, "delete it to retry")
+}
+
+func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
+		InstanceID: "inst-1",
+		Err:        network.ErrNameExists,
+	}}
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+		Body: &oapi.CreateInstanceRequest{Image: "test-image"},
+	})
+	require.NoError(t, err)
+
+	pending, ok := resp.(oapi.CreateInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
+	assert.Contains(t, pending.Message, "startup reconcile")
+	assert.NotContains(t, pending.Message, "delete")
+}
+
 func TestCreateInstance_AutoPullImage(t *testing.T) {
 	t.Parallel()
 	if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) {

cmd/api/main.go

@@ -172,6 +172,28 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta
 	}, logger), nil
 }
 
+func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) {
+	allInstances, err := instanceManager.ListInstancesForReconcile(ctx)
+	if err != nil {
+		return nil, err
+	}
+	protected := make(map[string]struct{})
+	for _, inst := range allInstances {
+		if inst.GPUDevicePath == "" {
+			continue
+		}
+		// A nil PID does not mean the assignment is orphaned: the PID is
+		// persisted only after the hypervisor starts, so a crash during boot
+		// leaves the device path without one. Only skip protection when the
+		// recorded hypervisor is known to be gone.
+		if inst.HypervisorPID != nil && !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
+			continue
+		}
+		protected[inst.GPUDevicePath] = struct{}{}
+	}
+	return protected, nil
+}
+
 func run() error {
 	// Load config early for OTel initialization
 	// Config path can be specified via CONFIG_PATH env var or defaults to platform-specific locations
@@ -362,11 +384,16 @@ func run() error {
 		return fmt.Errorf("reconcile device state: %w", err)
 	}
 
-	// Reconcile mdev devices (clears orphaned vGPUs from previous runs)
-	logger.Info("Reconciling mdev devices...")
-	if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil {
-		// Log but don't fail - mdev cleanup is best-effort
-		logger.Warn("failed to reconcile mdev devices", "error", err)
+	// Reconcile vGPU devices (clears orphaned vGPUs from previous runs)
+	logger.Info("Reconciling vGPU devices...")
+	protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager)
+	if err != nil {
+		logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err)
+		protected = nil
+	}
+	if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil {
+		// Log but don't fail - vGPU cleanup is best-effort
+		logger.Warn("failed to reconcile vGPU devices", "error", err)
 	}
 
 	// Wire up resource validator for aggregate limit checking

cmd/api/main_test.go

@@ -2,15 +2,18 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os/exec"
 	"testing"
 	"time"
 
 	"github.com/getkin/kin-openapi/openapi3filter"
 	"github.com/go-chi/chi/v5"
 	"github.com/golang-jwt/jwt/v5"
+	"github.com/kernel/hypeman/lib/instances"
 	mw "github.com/kernel/hypeman/lib/middleware"
 	"github.com/kernel/hypeman/lib/oapi"
 	nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware"
@@ -338,3 +341,30 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) {
 		})
 	}
 }
+
+type vgpuReconcileManagerStub struct {
+	instances.Manager
+	list []instances.Instance
+}
+
+func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) {
+	return s.list, nil
+}
+
+// The hypervisor PID is persisted only after boot, so an assignment without
+// one may belong to a VM that is still starting and must stay protected.
+func TestLiveInstanceVGPUDevicePathsProtectsAssignmentsWithoutPID(t *testing.T) {
+	dead := exec.Command("true")
+	require.NoError(t, dead.Run())
+	deadPID := dead.Process.Pid
+
+	manager := vgpuReconcileManagerStub{list: []instances.Instance{
+		{StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4"}},
+		{StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", HypervisorPID: &deadPID}},
+	}}
+
+	protected, err := liveInstanceVGPUDevicePaths(context.Background(), manager)
+	require.NoError(t, err)
+	assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5")
+}

integration/vgpu_test.go

@@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) {
 	if framework == devices.VGPUFrameworkNone {
 		return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", ""
 	}
-	if framework == devices.VGPUFrameworkVendorVFIO {
-		// CreateVGPU rejects vendor VFIO until the instance lifecycle
-		// integration lands.
-		return "vGPU test requires the vendor VFIO instance lifecycle integration", ""
-	}
 
 	// Check for available profiles
 	profiles, err := devices.ListGPUProfiles()

lib/builds/manager_test.go

@@ -51,6 +51,10 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc
 	return result, nil
 }
 
+func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) {
+	return m.ListInstances(ctx, nil)
+}
+
 func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) {
 	return nil, nil
 }

lib/devices/vendor_vfio_linux.go

@@ -81,13 +81,13 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) {
 	return vfs, nil
 }
 
-// listProfiles aggregates creatable profiles per parent GPU. Free VFs on the
-// same GPU share its framebuffer, so counting each advertising VF overreports
-// availability. The driver only guarantees that a GPU still advertising a
-// type can fit one more instance of it, so report that per-GPU lower bound.
+// listProfiles counts each free VF advertising a type as one creatable
+// instance, matching the driver-reported units that mdev sums through
+// available_instances. This is a best-effort snapshot because creating on one
+// VF may revoke the type from siblings that share its GPU framebuffer.
 func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) {
 	profilesByType := make(map[string]profileMetadata)
-	creatableGPUs := make(map[string]map[string]struct{})
+	creatableVFs := make(map[string]int)
 	for _, vf := range vfs {
 		creatable, err := s.readCreatableProfiles(vf.PCIAddress)
 		if err != nil {
@@ -96,14 +96,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro
 		for _, profile := range creatable {
 			profilesByType[profile.TypeName] = profile
 			if !vf.Allocated {
-				gpu := vf.ParentGPU
-				if gpu == "" {
-					gpu = vf.PCIAddress
-				}
-				if creatableGPUs[profile.TypeName] == nil {
-					creatableGPUs[profile.TypeName] = make(map[string]struct{})
-				}
-				creatableGPUs[profile.TypeName][gpu] = struct{}{}
+				creatableVFs[profile.TypeName]++
 			}
 		}
 	}
@@ -119,7 +112,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro
 		profiles = append(profiles, GPUProfile{
 			Name:          profile.Name,
 			FramebufferMB: profile.FramebufferMB,
-			Available:     len(creatableGPUs[profile.TypeName]),
+			Available:     creatableVFs[profile.TypeName],
 		})
 	}
 	return profiles, nil

lib/devices/vendor_vfio_linux_test.go

@@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T
 	}
 }
 
-func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) {
+func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) {
 	t.Parallel()
 
 	sysfs := newTestVendorVFIOSysfs(t)
@@ -196,8 +196,8 @@ func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) {
 	require.NoError(t, err)
 	profiles, err := sysfs.listProfiles(vfs)
 	require.NoError(t, err)
-	assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"),
-		"free VFs share their parent GPU's capacity, so availability is per GPU")
+	assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"),
+		"each free VF advertising the type counts as one creatable instance")
 }
 
 func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) {

lib/devices/vgpu_linux.go

@@ -73,10 +73,7 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic
 			MdevUUID:    mdev.UUID,
 		}, nil
 	case VGPUFrameworkVendorVFIO:
-		// The instance lifecycle does not yet persist vendor VFIO assignments
-		// durably or guard their release against live claims, so keep the
-		// backend out of the create path until that integration lands.
-		return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle")
+		return hostVendorVFIO.create(ctx, profileName, instanceID)
 	default:
 		return nil, fmt.Errorf("vGPU framework not available")
 	}