Report real vGPU capacity and preserve create failure causes

cmd/api/api/instances.go

@@ -349,10 +349,10 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		// errors.Is case would match the cause and hide the pending vGPU cleanup.
 		case errors.As(err, &vgpuPending):
 			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
-			message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID)
+			message := fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.Err, vgpuPending.InstanceID)
 			innerCode := "vgpu_retained_instance"
 			if !vgpuPending.Retained {
-				message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID)
+				message = fmt.Sprintf("failed to create instance: %v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.Err, vgpuPending.InstanceID)
 				innerCode = "vgpu_unretained_instance"
 			}
 			return oapi.CreateInstance500JSONResponse{

cmd/api/api/instances_test.go

@@ -76,6 +76,8 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T)
 	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
 	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
 	assert.Contains(t, pending.Message, "inst-1")
+	assert.Contains(t, pending.Message, network.ErrNameExists.Error(),
+		"the underlying create failure must survive the cleanup guidance")
 	assert.Contains(t, pending.Message, "delete it to retry")
 	require.NotNil(t, pending.InnerError)
 	require.NotNil(t, pending.InnerError.Code)
@@ -101,6 +103,8 @@ func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(
 	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
 	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
 	assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
+	assert.Contains(t, pending.Message, network.ErrNameExists.Error(),
+		"the underlying create failure must survive the cleanup guidance")
 	assert.Contains(t, pending.Message, "startup reconcile")
 	assert.NotContains(t, pending.Message, "delete")
 	require.NotNil(t, pending.InnerError)

lib/devices/vendor_vfio_linux.go

@@ -81,22 +81,37 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) {
 	return vfs, nil
 }
 
-// listProfiles counts each free VF advertising a type as one creatable
-// instance, matching the driver-reported units that mdev sums through
-// available_instances. This is a best-effort snapshot because creating on one
-// VF may revoke the type from siblings that share its GPU framebuffer.
+// listProfiles reports a conservative estimate of how many instances of each
+// profile are concurrently creatable. Free VFs on the same GPU share its
+// framebuffer, so counting every advertising VF overreports capacity: one 48Q
+// assignment can revoke the type from all sibling VFs. Each GPU instead
+// contributes min(free VFs advertising the type, remaining framebuffer /
+// profile framebuffer), where the largest profile still creatable on the GPU
+// is a lower bound on its remaining framebuffer.
 func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) {
 	profilesByType := make(map[string]profileMetadata)
-	creatableVFs := make(map[string]int)
+	freeVFsByGPU := make(map[string]map[string]int)
+	remainingFBByGPU := make(map[string]int)
 	for _, vf := range vfs {
 		creatable, err := s.readCreatableProfiles(vf.PCIAddress)
 		if err != nil {
 			return nil, err
 		}
+		gpu := vf.ParentGPU
+		if gpu == "" {
+			gpu = vf.PCIAddress
+		}
 		for _, profile := range creatable {
 			profilesByType[profile.TypeName] = profile
-			if !vf.Allocated {
-				creatableVFs[profile.TypeName]++
+			if vf.Allocated {
+				continue
+			}
+			if freeVFsByGPU[gpu] == nil {
+				freeVFsByGPU[gpu] = make(map[string]int)
+			}
+			freeVFsByGPU[gpu][profile.TypeName]++
+			if profile.FramebufferMB > remainingFBByGPU[gpu] {
+				remainingFBByGPU[gpu] = profile.FramebufferMB
 			}
 		}
 	}
@@ -109,15 +124,35 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro
 
 	profiles := make([]GPUProfile, 0, len(metadata))
 	for _, profile := range metadata {
+		available := 0
+		for gpu, freeVFs := range freeVFsByGPU {
+			available += gpuProfileCapacity(freeVFs[profile.TypeName], remainingFBByGPU[gpu], profile.FramebufferMB)
+		}
 		profiles = append(profiles, GPUProfile{
 			Name:          profile.Name,
 			FramebufferMB: profile.FramebufferMB,
-			Available:     creatableVFs[profile.TypeName],
+			Available:     available,
 		})
 	}
 	return profiles, nil
 }
 
+// gpuProfileCapacity estimates how many instances of a profile one GPU can
+// still create concurrently. When a profile's framebuffer is unknown (0), the
+// free VF count is the only signal available.
+func gpuProfileCapacity(freeVFs, remainingFB, profileFB int) int {
+	if freeVFs == 0 {
+		return 0
+	}
+	if profileFB <= 0 || remainingFB <= 0 {
+		return freeVFs
+	}
+	if byFB := remainingFB / profileFB; byFB < freeVFs {
+		return byFB
+	}
+	return freeVFs
+}
+
 func (s vendorVFIOSysfs) create(ctx context.Context, profileName, instanceID string) (*VGPUDevice, error) {
 	vendorVFIOMu.Lock()
 	defer vendorVFIOMu.Unlock()

lib/devices/vendor_vfio_linux_test.go

@@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T
 	}
 }
 
-func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) {
+func TestVendorVFIOListProfilesReportsPerGPUCapacity(t *testing.T) {
 	t.Parallel()
 
 	sysfs := newTestVendorVFIOSysfs(t)
@@ -196,8 +196,30 @@ func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) {
 	require.NoError(t, err)
 	profiles, err := sysfs.listProfiles(vfs)
 	require.NoError(t, err)
-	assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"),
-		"each free VF advertising the type counts as one creatable instance")
+	assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"),
+		"one 48Q consumes a whole GPU, so two GPUs mean two creatable instances despite three free VFs")
+	assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-2Q"),
+		"small profiles stay capped by the free VF count")
+}
+
+func TestVendorVFIOListProfilesCapsCapacityByRemainingFramebuffer(t *testing.T) {
+	t.Parallel()
+
+	// One 48G GPU with a 24Q already assigned: siblings only advertise up to
+	// 24Q, so at most one more 24Q fits despite two free VFs.
+	remaining := "ID    : vGPU Name\n1147  : NVIDIA L40S-1Q\n1153  : NVIDIA L40S-24Q\n"
+	sysfs := newTestVendorVFIOSysfs(t)
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.4", "42", "1153", remaining)
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.5", "43", "0", remaining)
+	sysfs.addVF(t, "0000:82:00.0", "0000:82:00.6", "44", "0", remaining)
+
+	vfs, err := sysfs.discoverVFs()
+	require.NoError(t, err)
+	profiles, err := sysfs.listProfiles(vfs)
+	require.NoError(t, err)
+	assert.Equal(t, 1, profileAvailability(profiles, "NVIDIA L40S-24Q"))
+	assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-1Q"),
+		"framebuffer allows more 1Q instances than free VFs, so the VF count caps it")
 }
 
 func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) {