From 95f064c23dac30ee9a63ce0c64818a559aba41c4 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 19:05:18 +0000
Subject: [PATCH 01/17] Guard vGPU releases with live-instance claims

A vGPU assignment goes stale when its release succeeds but the
metadata save does not (or start fails between the release and its
first save). The backend's owner map only covers assignments created
since the last restart and the VFIO handle scan only covers VMs that
have opened the device, so after a restart a stale release could still
clear a VF during another live instance's pre-open boot window.
Consult live instance metadata on every release: when another instance
with a live hypervisor process claims the same device path, drop the
stale metadata without touching the device.

Tag assignments with the owning instance ID, persist the assignment
before booting a started instance, and retain assignment metadata when
rollback release fails in create and start so later release paths can
still find the device.
---
 lib/instances/create.go              | 41 ++++++++++++++++++++-
 lib/instances/delete.go              |  2 +-
 lib/instances/lifecycle_noop_test.go | 55 ++++++++++++++++++++++++++++
 lib/instances/start.go               |  9 ++++-
 lib/instances/stop.go                |  2 +-
 lib/instances/vgpu.go                | 45 ++++++++++++++++++-----
 lib/instances/vgpu_test.go           | 42 ++++++++++++++++-----
 7 files changed, 173 insertions(+), 23 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index cda7c49e..029ad13b 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -265,11 +265,13 @@ func (m *manager) createInstance(
 	var gpuFramework devices.VGPUFramework
 	var gpuDevicePath string
 	var gpuMdevUUID string
+	var stored *StoredMetadata
+	var retainedVGPU *StoredMetadata
 
 	// Setup cleanup stack early so device attachment errors trigger cleanup
 	cu := cleanup.Make(func() {
 		log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id)
-		m.deleteInstanceData(id)
+		m.cleanupFailedCreate(ctx, id, retainedVGPU)
 	})
 	defer cu.Clean()
 
@@ -306,6 +308,25 @@ func (m *manager) createInstance(
 			}
 			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
 				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err)
+				retainedVGPU = stored
+				if retainedVGPU == nil {
+					retainedVGPU = &StoredMetadata{
+						Id:                id,
+						Name:              req.Name,
+						Image:             req.Image,
+						ResolvedImage:     resolvedImageRef,
+						Platform:          imageInfo.Platform,
+						CreatedAt:         time.Now(),
+						HypervisorType:    hvType,
+						HypervisorVersion: hvVersion,
+						SocketPath:        m.paths.InstanceSocket(id, starter.SocketName()),
+						DataDir:           m.paths.InstanceDir(id),
+						GPUProfile:        gpuDevice.ProfileName,
+						GPUFramework:      gpuDevice.Framework,
+						GPUDevicePath:     gpuDevice.SysfsPath,
+						GPUMdevUUID:       gpuDevice.MdevUUID,
+					}
+				}
 			}
 		})
 	}
@@ -341,7 +362,7 @@ func (m *manager) createInstance(
 	}
 
 	// 11. Create instance metadata
-	stored := &StoredMetadata{
+	stored = &StoredMetadata{
 		Id:                       id,
 		Name:                     req.Name,
 		Image:                    req.Image,
@@ -575,6 +596,22 @@ func (m *manager) createInstance(
 	return &finalInst, nil
 }
 
+func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) {
+	if retainedVGPU == nil {
+		m.deleteInstanceData(id)
+		return
+	}
+
+	log := logger.FromContext(ctx)
+	if err := m.ensureDirectories(id); err != nil {
+		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
+		return
+	}
+	if err := m.saveMetadata(&metadata{StoredMetadata: *retainedVGPU}); err != nil {
+		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
+	}
+}
+
 // validateCreateRequest validates the create instance request.
 // The request is mutated in-place to persist normalized egress/credential policy fields.
 func validateCreateRequest(req *CreateInstanceRequest) error {
diff --git a/lib/instances/delete.go b/lib/instances/delete.go
index 949d7a6c..5d1b3168 100644
--- a/lib/instances/delete.go
+++ b/lib/instances/delete.go
@@ -139,7 +139,7 @@ func (m *manager) deleteInstanceWithOptions(
 	// or volume teardown. A failed release retains the instance metadata; the
 	// VMM has already been stopped, but its attachments are intact and the
 	// restart policy is blocked, so a retried delete is safe.
-	if err := releaseStoredVGPU(ctx, stored); err != nil {
+	if err := m.releaseStoredVGPU(ctx, stored); err != nil {
 		log.ErrorContext(ctx, "failed to destroy vGPU; retaining instance metadata", "instance_id", id, "error", err)
 		return fmt.Errorf("destroy vGPU: %w", err)
 	}
diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go
index 3e85e0cc..3eb5a46c 100644
--- a/lib/instances/lifecycle_noop_test.go
+++ b/lib/instances/lifecycle_noop_test.go
@@ -3,6 +3,7 @@ package instances
 import (
 	"context"
 	"errors"
+	"net"
 	"os"
 	"path/filepath"
 	"sync"
@@ -186,6 +187,46 @@ func TestDeleteBlocksRestartPolicyWhenVGPUReleaseFails(t *testing.T) {
 		"a failed delete must not leave the instance restartable")
 }
 
+func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) {
+	now := time.Now().UTC()
+	m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, now)
+	meta, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	meta.GPUProfile = "NVIDIA L40S-2Q"
+	meta.GPUFramework = devices.VGPUFramework("future-framework")
+	meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4"
+	require.NoError(t, m.saveMetadata(meta))
+
+	claimantID := "inst-live-claimant"
+	require.NoError(t, m.ensureDirectories(claimantID))
+	pid := os.Getpid()
+	socketPath := m.paths.InstanceSocket(claimantID, "noop.sock")
+	listener, err := net.Listen("unix", socketPath)
+	require.NoError(t, err)
+	defer listener.Close()
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:             claimantID,
+		Name:           claimantID,
+		Image:          "test-image",
+		CreatedAt:      now,
+		HypervisorType: lifecycleNoopHypervisorType,
+		HypervisorPID:  &pid,
+		SocketPath:     socketPath,
+		DataDir:        m.paths.InstanceDir(claimantID),
+		GPUProfile:     "NVIDIA L40S-2Q",
+		GPUFramework:   devices.VGPUFramework("future-framework"),
+		GPUDevicePath:  "/sys/bus/pci/devices/0000:82:00.4",
+	}}))
+
+	require.NoError(t, m.DeleteInstance(context.Background(), id))
+
+	_, err = m.loadMetadata(id)
+	require.Error(t, err, "deleted instance metadata should be gone")
+	claimant, err := m.loadMetadata(claimantID)
+	require.NoError(t, err)
+	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", claimant.GPUDevicePath, "live claimant keeps its assignment")
+}
+
 func TestDeleteReleasesVGPUBeforeTeardown(t *testing.T) {
 	m, id := newLifecycleNoopManagerWithInstance(t, StateStopped, time.Now().UTC())
 	deviceManager := &recordingDeviceManager{}
@@ -288,6 +329,20 @@ func (m *recordingDeviceManager) UnbindFromVFIO(ctx context.Context, id string)
 	return nil
 }
 
+func TestLifecycleNoopStandbyRejectsVendorVFIOVGPU(t *testing.T) {
+	m, id := newLifecycleNoopManagerWithInstance(t, StateRunning, time.Now().UTC())
+	meta, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	meta.GPUProfile = "NVIDIA L40S-2Q"
+	meta.GPUFramework = devices.VGPUFrameworkVendorVFIO
+	meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4"
+	require.NoError(t, m.saveMetadata(meta))
+
+	_, err = m.StandbyInstance(context.Background(), id, StandbyInstanceRequest{})
+	require.ErrorIs(t, err, ErrInvalidState)
+	assert.ErrorContains(t, err, "standby is not supported for instances with vGPU attached")
+}
+
 func newLifecycleNoopManagerWithInstance(t *testing.T, state State, now time.Time) (*manager, string) {
 	t.Helper()
 
diff --git a/lib/instances/start.go b/lib/instances/start.go
index adb911e1..2c9fe023 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -53,7 +53,7 @@ func (m *manager) startInstance(
 	// cannot leave on-disk metadata pointing at a device that is already
 	// gone (matching releaseRetainedVGPULocked).
 	if storedVGPUDevicePath(stored) != "" {
-		if err := releaseStoredVGPU(ctx, stored); err != nil {
+		if err := m.releaseStoredVGPU(ctx, stored); err != nil {
 			log.ErrorContext(ctx, "failed to release stale vGPU before start", "instance_id", id, "error", err)
 			return nil, fmt.Errorf("release stale vGPU before start: %w", err)
 		}
@@ -178,8 +178,15 @@ func (m *manager) startInstance(
 			}
 			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
 				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err)
+				if saveErr := m.saveMetadata(meta); saveErr != nil {
+					log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr)
+				}
 			}
 		})
+		if err := m.saveMetadata(meta); err != nil {
+			log.ErrorContext(ctx, "failed to save metadata after vGPU creation", "instance_id", id, "error", err)
+			return nil, fmt.Errorf("save metadata after vGPU creation: %w", err)
+		}
 	}
 
 	// 5. Regenerate config disk with new network configuration
diff --git a/lib/instances/stop.go b/lib/instances/stop.go
index 02e1faba..35e69822 100644
--- a/lib/instances/stop.go
+++ b/lib/instances/stop.go
@@ -246,7 +246,7 @@ func (m *manager) stopInstance(
 	}
 
 	// 7. Release the vGPU assignment if present.
-	if err := releaseStoredVGPU(ctx, stored); err != nil {
+	if err := m.releaseStoredVGPU(ctx, stored); err != nil {
 		log.WarnContext(ctx, "failed to destroy vGPU on stop; retaining assignment metadata", "instance_id", id, "error", err)
 	}
 
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index a8ca6ace..02d4ebc4 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -2,6 +2,7 @@ package instances
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
 
 	"github.com/kernel/hypeman/lib/devices"
@@ -20,23 +21,49 @@ func clearStoredVGPUDevice(stored *StoredMetadata) {
 	stored.GPUMdevUUID = ""
 }
 
-func releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error {
+func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error {
 	path := storedVGPUDevicePath(stored)
 	if path != "" {
-		assignment := devices.VGPUAssignment{
-			Framework:  stored.GPUFramework,
-			DevicePath: path,
-			MdevUUID:   stored.GPUMdevUUID,
-			InstanceID: stored.Id,
-		}
-		if err := devices.DestroyVGPU(ctx, assignment); err != nil {
+		claimed, err := m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path)
+		if err != nil {
 			return err
 		}
+		if claimed {
+			logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance",
+				"instance_id", stored.Id, "device_path", path)
+		} else {
+			assignment := devices.VGPUAssignment{
+				Framework:  stored.GPUFramework,
+				DevicePath: path,
+				MdevUUID:   stored.GPUMdevUUID,
+				InstanceID: stored.Id,
+			}
+			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
+				return err
+			}
+		}
 	}
 	clearStoredVGPUDevice(stored)
 	return nil
 }
 
+func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) {
+	instances, err := m.listInstances(ctx)
+	if err != nil {
+		return false, fmt.Errorf("list instances for vGPU release check: %w", err)
+	}
+	for i := range instances {
+		inst := &instances[i]
+		if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil {
+			continue
+		}
+		if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
 // releaseRetainedVGPULocked releases a vGPU assignment retained on a stopped
 // instance after a failed release during the original stop. It is a no-op
 // when no assignment is retained, and a failed retry only logs so the
@@ -52,7 +79,7 @@ func (m *manager) releaseRetainedVGPULocked(ctx context.Context, id string) {
 	if storedVGPUDevicePath(stored) == "" {
 		return
 	}
-	if err := releaseStoredVGPU(ctx, stored); err != nil {
+	if err := m.releaseStoredVGPU(ctx, stored); err != nil {
 		log.WarnContext(ctx, "failed to destroy retained vGPU; retaining assignment metadata", "instance_id", id, "error", err)
 		return
 	}
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 6f2c4681..7fc824f9 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -5,14 +5,39 @@ import (
 	"testing"
 
 	"github.com/kernel/hypeman/lib/devices"
+	"github.com/kernel/hypeman/lib/paths"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	stored := &StoredMetadata{
+		Id:             "failed-create",
+		Name:           "failed-create",
+		GPUProfile:     "NVIDIA L40S-2Q",
+		GPUFramework:   devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath:  "/sys/bus/pci/devices/0000:82:00.4",
+		HypervisorType: "qemu",
+		DataDir:        m.paths.InstanceDir("failed-create"),
+	}
+
+	m.cleanupFailedCreate(context.Background(), stored.Id, stored)
+
+	retained, err := m.loadMetadata(stored.Id)
+	require.NoError(t, err)
+	assert.Equal(t, stored.GPUProfile, retained.GPUProfile)
+	assert.Equal(t, stored.GPUFramework, retained.GPUFramework)
+	assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath)
+}
+
 func TestStoredVGPUDevicePath(t *testing.T) {
 	t.Parallel()
 
-	assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", storedVGPUDevicePath(&StoredMetadata{
-		GPUDevicePath: "/sys/bus/mdev/devices/new-uuid",
+	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", storedVGPUDevicePath(&StoredMetadata{
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
 		GPUMdevUUID:   "legacy-uuid",
 	}))
 	assert.Equal(t, "/sys/bus/mdev/devices/legacy-uuid", storedVGPUDevicePath(&StoredMetadata{
@@ -24,11 +49,12 @@ func TestStoredVGPUDevicePath(t *testing.T) {
 func TestReleaseStoredVGPURetainsMetadataOnFailure(t *testing.T) {
 	t.Parallel()
 
+	m := &manager{paths: paths.New(t.TempDir())}
 	stored := &StoredMetadata{
 		GPUFramework:  devices.VGPUFramework("future-framework"),
 		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
 	}
-	err := releaseStoredVGPU(context.Background(), stored)
+	err := m.releaseStoredVGPU(context.Background(), stored)
 	assert.Error(t, err)
 	assert.Equal(t, devices.VGPUFramework("future-framework"), stored.GPUFramework)
 	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath)
@@ -39,13 +65,11 @@ func TestSetAndClearStoredVGPUDevice(t *testing.T) {
 
 	stored := &StoredMetadata{}
 	setStoredVGPUDevice(stored, &devices.VGPUDevice{
-		Framework: devices.VGPUFrameworkMdev,
-		SysfsPath: "/sys/bus/mdev/devices/new-uuid",
-		MdevUUID:  "new-uuid",
+		Framework: devices.VGPUFrameworkVendorVFIO,
+		SysfsPath: "/sys/bus/pci/devices/0000:82:00.4",
 	})
-	assert.Equal(t, devices.VGPUFrameworkMdev, stored.GPUFramework)
-	assert.Equal(t, "/sys/bus/mdev/devices/new-uuid", stored.GPUDevicePath)
-	assert.Equal(t, "new-uuid", stored.GPUMdevUUID)
+	assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework)
+	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath)
 
 	clearStoredVGPUDevice(stored)
 	assert.Empty(t, stored.GPUFramework)

From fd5b30df8cf44386cd99a85a5dd79db401a4b51a Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 19:05:42 +0000
Subject: [PATCH 02/17] Reconcile vendor VFIO vGPUs against a fail-closed
 instance inventory

Startup reconciliation protects the VFs of instances whose hypervisor
survived the restart, verified by socket ownership so a reused PID
cannot hold a VF. The inventory behind that protected set must not
silently skip unreadable metadata: a skipped live claimant would leave
its VF unprotected during the pre-VFIO-open boot window. Add
ListInstancesForReconcile, which fails on any unreadable metadata, and
skip vendor VFIO reconciliation when the inventory is unavailable
while keeping mdev reconciliation running.
---
 cmd/api/main.go             | 33 ++++++++++++++++++++++++++++-----
 lib/builds/manager_test.go  |  4 ++++
 lib/instances/manager.go    |  6 ++++++
 lib/instances/query.go      | 13 +++++++++++--
 lib/instances/query_test.go | 22 ++++++++++++++++++++++
 lib/instances/storage.go    |  8 +++++++-
 lib/instances/wait_test.go  |  3 +++
 7 files changed, 81 insertions(+), 8 deletions(-)

diff --git a/cmd/api/main.go b/cmd/api/main.go
index 98d7be72..44fa1b16 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -172,6 +172,24 @@ func configureUFFDGraduationController(cfg *config.Config, instanceManager insta
 	}, logger), nil
 }
 
+func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.Manager) (map[string]struct{}, error) {
+	allInstances, err := instanceManager.ListInstancesForReconcile(ctx)
+	if err != nil {
+		return nil, err
+	}
+	protected := make(map[string]struct{})
+	for _, inst := range allInstances {
+		if inst.GPUDevicePath == "" || inst.HypervisorPID == nil {
+			continue
+		}
+		if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
+			continue
+		}
+		protected[inst.GPUDevicePath] = struct{}{}
+	}
+	return protected, nil
+}
+
 func run() error {
 	// Load config early for OTel initialization
 	// Config path can be specified via CONFIG_PATH env var or defaults to platform-specific locations
@@ -362,11 +380,16 @@ func run() error {
 		return fmt.Errorf("reconcile device state: %w", err)
 	}
 
-	// Reconcile mdev devices (clears orphaned vGPUs from previous runs)
-	logger.Info("Reconciling mdev devices...")
-	if err := devices.ReconcileMdevs(app.Ctx, nil); err != nil {
-		// Log but don't fail - mdev cleanup is best-effort
-		logger.Warn("failed to reconcile mdev devices", "error", err)
+	// Reconcile vGPU devices (clears orphaned vGPUs from previous runs)
+	logger.Info("Reconciling vGPU devices...")
+	protected, err := liveInstanceVGPUDevicePaths(app.Ctx, app.InstanceManager)
+	if err != nil {
+		logger.Warn("failed to list instances for vGPU reconcile protection; skipping vendor VFIO reconciliation", "error", err)
+		protected = nil
+	}
+	if err := devices.ReconcileVGPUs(app.Ctx, protected); err != nil {
+		// Log but don't fail - vGPU cleanup is best-effort
+		logger.Warn("failed to reconcile vGPU devices", "error", err)
 	}
 
 	// Wire up resource validator for aggregate limit checking
diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go
index dfdc9fca..14ccf8c2 100644
--- a/lib/builds/manager_test.go
+++ b/lib/builds/manager_test.go
@@ -51,6 +51,10 @@ func (m *mockInstanceManager) ListInstances(ctx context.Context, filter *instanc
 	return result, nil
 }
 
+func (m *mockInstanceManager) ListInstancesForReconcile(ctx context.Context) ([]instances.Instance, error) {
+	return m.ListInstances(ctx, nil)
+}
+
 func (m *mockInstanceManager) ListSnapshots(ctx context.Context, filter *instances.ListSnapshotsFilter) ([]instances.Snapshot, error) {
 	return nil, nil
 }
diff --git a/lib/instances/manager.go b/lib/instances/manager.go
index 85f75975..d4206c91 100644
--- a/lib/instances/manager.go
+++ b/lib/instances/manager.go
@@ -27,6 +27,7 @@ import (
 
 type Manager interface {
 	ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error)
+	ListInstancesForReconcile(ctx context.Context) ([]Instance, error)
 	ListSnapshots(ctx context.Context, filter *ListSnapshotsFilter) ([]Snapshot, error)
 	GetSnapshot(ctx context.Context, snapshotID string) (*Snapshot, error)
 	CreateInstance(ctx context.Context, req CreateInstanceRequest) (*Instance, error)
@@ -696,6 +697,11 @@ func (m *manager) UpdateInstance(ctx context.Context, id string, req UpdateInsta
 	return inst, err
 }
 
+// ListInstancesForReconcile returns every instance or an invalid metadata error.
+func (m *manager) ListInstancesForReconcile(ctx context.Context) ([]Instance, error) {
+	return m.loadInstances(ctx, false)
+}
+
 // ListInstances returns instances, optionally filtered by the given criteria.
 // Pass nil to return all instances.
 func (m *manager) ListInstances(ctx context.Context, filter *ListInstancesFilter) ([]Instance, error) {
diff --git a/lib/instances/query.go b/lib/instances/query.go
index c7632f93..2311a3fe 100644
--- a/lib/instances/query.go
+++ b/lib/instances/query.go
@@ -882,14 +882,18 @@ func parseSentinelTimestamp(line, sentinelPrefix string) (time.Time, bool) {
 	return time.Time{}, false
 }
 
-// listInstances returns all instances
+// listInstances returns all instances, skipping metadata files that cannot be loaded.
 func (m *manager) listInstances(ctx context.Context) ([]Instance, error) {
+	return m.loadInstances(ctx, true)
+}
+
+func (m *manager) loadInstances(ctx context.Context, skipInvalid bool) ([]Instance, error) {
 	ctx, span := m.tracerOrDefault().Start(ctx, "instances.list_metadata")
 	defer span.End()
 	log := logger.FromContext(ctx)
 	log.DebugContext(ctx, "listing all instances")
 
-	files, err := m.listMetadataFiles()
+	files, err := m.listMetadataFilesWithStatErrors(!skipInvalid)
 	if err != nil {
 		log.ErrorContext(ctx, "failed to list metadata files", "error", err)
 		return nil, err
@@ -907,6 +911,11 @@ func (m *manager) listInstances(ctx context.Context) ([]Instance, error) {
 		)
 		meta, err := m.loadMetadata(id)
 		if err != nil {
+			if !skipInvalid {
+				hydrateSpan.RecordError(err)
+				hydrateSpan.End()
+				return nil, fmt.Errorf("load metadata for instance %s: %w", id, err)
+			}
 			// Skip instances with invalid metadata
 			log.WarnContext(hydrateCtx, "skipping instance with invalid metadata", "instance_id", id, "error", err)
 			hydrateSpan.End()
diff --git a/lib/instances/query_test.go b/lib/instances/query_test.go
index 8bb0bb46..41aba54e 100644
--- a/lib/instances/query_test.go
+++ b/lib/instances/query_test.go
@@ -14,6 +14,28 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestListInstancesForReconcileFailsOnInvalidMetadata(t *testing.T) {
+	m := &manager{paths: paths.New(t.TempDir())}
+
+	require.NoError(t, m.ensureDirectories("valid"))
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:        "valid",
+		Name:      "valid",
+		CreatedAt: time.Now(),
+		DataDir:   m.paths.InstanceDir("valid"),
+	}}))
+	require.NoError(t, m.ensureDirectories("invalid"))
+	require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid"), []byte("{"), 0644))
+
+	listed, err := m.ListInstances(context.Background(), nil)
+	require.NoError(t, err)
+	require.Len(t, listed, 1)
+
+	_, err = m.ListInstancesForReconcile(context.Background())
+	require.Error(t, err)
+	assert.ErrorContains(t, err, "load metadata for instance invalid")
+}
+
 func TestParseExitSentinelLine(t *testing.T) {
 	t.Parallel()
 	tests := []struct {
diff --git a/lib/instances/storage.go b/lib/instances/storage.go
index f33a5962..d44dcbf2 100644
--- a/lib/instances/storage.go
+++ b/lib/instances/storage.go
@@ -177,8 +177,12 @@ func removeAllWithRetry(path string, removeAll func(string) error, sleep func(ti
 	}
 }
 
-// listMetadataFiles returns paths to all instance metadata files
+// listMetadataFiles returns paths to all instance metadata files.
 func (m *manager) listMetadataFiles() ([]string, error) {
+	return m.listMetadataFilesWithStatErrors(false)
+}
+
+func (m *manager) listMetadataFilesWithStatErrors(failOnStatError bool) ([]string, error) {
 	guestsDir := m.paths.GuestsDir()
 
 	// Ensure guests directory exists
@@ -200,6 +204,8 @@ func (m *manager) listMetadataFiles() ([]string, error) {
 		metaPath := filepath.Join(guestsDir, entry.Name(), "metadata.json")
 		if _, err := os.Stat(metaPath); err == nil {
 			metaFiles = append(metaFiles, metaPath)
+		} else if failOnStatError && !os.IsNotExist(err) {
+			return nil, fmt.Errorf("stat metadata for instance %s: %w", entry.Name(), err)
 		}
 	}
 
diff --git a/lib/instances/wait_test.go b/lib/instances/wait_test.go
index bab6f06d..a4246479 100644
--- a/lib/instances/wait_test.go
+++ b/lib/instances/wait_test.go
@@ -32,6 +32,9 @@ func (s *stubManager) GetInstance(ctx context.Context, id string) (*Instance, er
 func (s *stubManager) ListInstances(context.Context, *ListInstancesFilter) ([]Instance, error) {
 	return nil, nil
 }
+func (s *stubManager) ListInstancesForReconcile(context.Context) ([]Instance, error) {
+	return nil, nil
+}
 func (s *stubManager) ListSnapshots(context.Context, *ListSnapshotsFilter) ([]Snapshot, error) {
 	return nil, nil
 }

From 6ab81d0aed1dd788b2445dad2707cd274798a0c2 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 20:19:26 +0000
Subject: [PATCH 03/17] Fail closed on vGPU claim checks

---
 lib/instances/vgpu.go      |  2 +-
 lib/instances/vgpu_test.go | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 02d4ebc4..19767c78 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -48,7 +48,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata)
 }
 
 func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) {
-	instances, err := m.listInstances(ctx)
+	instances, err := m.ListInstancesForReconcile(ctx)
 	if err != nil {
 		return false, fmt.Errorf("list instances for vGPU release check: %w", err)
 	}
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 7fc824f9..5f1a0140 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -2,6 +2,7 @@ package instances
 
 import (
 	"context"
+	"os"
 	"testing"
 
 	"github.com/kernel/hypeman/lib/devices"
@@ -33,6 +34,17 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath)
 }
 
+func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("invalid-instance"))
+	require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644))
+
+	_, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4")
+	require.Error(t, err)
+}
+
 func TestStoredVGPUDevicePath(t *testing.T) {
 	t.Parallel()
 

From 52d3d4b31e06b2021dee6250b99d25649d0ed1c5 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 20:19:50 +0000
Subject: [PATCH 04/17] Retain only vGPU assignment after failed create

---
 lib/instances/create.go    |  8 +++++++-
 lib/instances/vgpu_test.go | 13 ++++++++++++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index 029ad13b..2ba23d7a 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -607,7 +607,13 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
 		return
 	}
-	if err := m.saveMetadata(&metadata{StoredMetadata: *retainedVGPU}); err != nil {
+	retained := StoredMetadata{
+		Id:            id,
+		GPUFramework:  retainedVGPU.GPUFramework,
+		GPUDevicePath: retainedVGPU.GPUDevicePath,
+		GPUMdevUUID:   retainedVGPU.GPUMdevUUID,
+	}
+	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
 	}
 }
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 5f1a0140..55b94ce8 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -21,6 +21,10 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 		GPUProfile:     "NVIDIA L40S-2Q",
 		GPUFramework:   devices.VGPUFrameworkVendorVFIO,
 		GPUDevicePath:  "/sys/bus/pci/devices/0000:82:00.4",
+		GPUMdevUUID:    "mdev-uuid",
+		NetworkEnabled: true,
+		IP:             "192.0.2.1",
+		Volumes:        []VolumeAttachment{{VolumeID: "volume"}},
 		HypervisorType: "qemu",
 		DataDir:        m.paths.InstanceDir("failed-create"),
 	}
@@ -29,9 +33,16 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 
 	retained, err := m.loadMetadata(stored.Id)
 	require.NoError(t, err)
-	assert.Equal(t, stored.GPUProfile, retained.GPUProfile)
+	assert.Equal(t, stored.Id, retained.Id)
 	assert.Equal(t, stored.GPUFramework, retained.GPUFramework)
 	assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath)
+	assert.Equal(t, stored.GPUMdevUUID, retained.GPUMdevUUID)
+	assert.Empty(t, retained.Name)
+	assert.Empty(t, retained.GPUProfile)
+	assert.False(t, retained.NetworkEnabled)
+	assert.Empty(t, retained.IP)
+	assert.Empty(t, retained.Volumes)
+	assert.Empty(t, retained.DataDir)
 }
 
 func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) {

From f73e7e36287e12816bd5028e541f3e0f7f98f5c0 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 20:20:05 +0000
Subject: [PATCH 05/17] Clear released vGPU assignment on start rollback

---
 lib/instances/start.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/instances/start.go b/lib/instances/start.go
index 2c9fe023..8cf7eceb 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -181,6 +181,11 @@ func (m *manager) startInstance(
 				if saveErr := m.saveMetadata(meta); saveErr != nil {
 					log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr)
 				}
+			} else {
+				clearStoredVGPUDevice(stored)
+				if saveErr := m.saveMetadata(meta); saveErr != nil {
+					log.ErrorContext(ctx, "failed to save metadata after vGPU cleanup", "instance_id", id, "error", saveErr)
+				}
 			}
 		})
 		if err := m.saveMetadata(meta); err != nil {

From 6eaea6cae2053f0841c8f00361bf72886b87de3a Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Thu, 6 Aug 2026 20:50:00 +0000
Subject: [PATCH 06/17] Test start rollback vGPU cleanup

---
 lib/instances/vgpu_test.go | 70 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 70 insertions(+)

diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 55b94ce8..df2c265e 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -3,7 +3,10 @@ package instances
 import (
 	"context"
 	"os"
+	"path/filepath"
+	"sync"
 	"testing"
+	_ "unsafe"
 
 	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/paths"
@@ -45,6 +48,73 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	assert.Empty(t, retained.DataDir)
 }
 
+//go:linkname hostVendorVFIO github.com/kernel/hypeman/lib/devices.hostVendorVFIO
+var hostVendorVFIO vendorVFIOSysfs
+
+type vendorVFIOSysfs struct {
+	pciDevicesPath  string
+	procPath        string
+	vfioDevicesPath string
+	owners          map[string]string
+}
+
+func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
+	root := t.TempDir()
+	pciDevicesPath := filepath.Join(root, "sys", "bus", "pci", "devices")
+	vfAddress := "0000:82:00.4"
+	nvidiaPath := filepath.Join(pciDevicesPath, vfAddress, "nvidia")
+	require.NoError(t, os.MkdirAll(nvidiaPath, 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte("0"), 0o644))
+	require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte("ID    : vGPU Name\n1148  : NVIDIA L40S-2Q\n"), 0o644))
+
+	originalVendorVFIO := hostVendorVFIO
+	hostVendorVFIO = vendorVFIOSysfs{
+		pciDevicesPath:  pciDevicesPath,
+		procPath:        filepath.Join(root, "proc"),
+		vfioDevicesPath: filepath.Join(root, "dev", "vfio", "devices"),
+		owners:          make(map[string]string),
+	}
+	t.Cleanup(func() { hostVendorVFIO = originalVendorVFIO })
+	require.NoError(t, os.MkdirAll(hostVendorVFIO.procPath, 0o755))
+
+	m := &manager{
+		paths:           paths.New(t.TempDir()),
+		imageManager:    readyFixtureImageManager{name: "test-image"},
+		instanceLocks:   sync.Map{},
+		bootMarkerScans: sync.Map{},
+	}
+	const id = "start-rollback"
+	require.NoError(t, m.ensureDirectories(id))
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:             id,
+		Name:           id,
+		Image:          "test-image",
+		GPUProfile:     "NVIDIA L40S-2Q",
+		HypervisorType: lifecycleNoopHypervisorType,
+		SocketPath:     m.paths.InstanceSocket(id, "noop.sock"),
+		DataDir:        m.paths.InstanceDir(id),
+	}}))
+
+	t.Setenv("TMPDIR", filepath.Join(root, "missing"))
+	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{})
+	require.Error(t, err)
+
+	stored, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile)
+	assert.Empty(t, stored.GPUFramework)
+	assert.Empty(t, stored.GPUDevicePath)
+	assert.Empty(t, stored.GPUMdevUUID)
+	assertFileContents(t, filepath.Join(nvidiaPath, "current_vgpu_type"), "0")
+}
+
+func assertFileContents(t *testing.T, path, want string) {
+	t.Helper()
+	got, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.Equal(t, want, string(got))
+}
+
 func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) {
 	t.Parallel()
 

From 51427091f179162d9b454636d1fdde0e97e27337 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Fri, 7 Aug 2026 15:02:16 +0000
Subject: [PATCH 07/17] Normalize legacy mdev paths in live-claim check

The claim guard compared raw GPUDevicePath, which is empty on records
persisted before the framework migration; a live claimant with only a
legacy GPUMdevUUID was invisible to the check. Normalize the inventory
side with storedVGPUDevicePath, matching the release subject.
---
 lib/instances/vgpu.go      |  2 +-
 lib/instances/vgpu_test.go | 18 ++++++++++++++++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 19767c78..b16524b3 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -54,7 +54,7 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu
 	}
 	for i := range instances {
 		inst := &instances[i]
-		if inst.Id == excludeID || inst.GPUDevicePath != devicePath || inst.HypervisorPID == nil {
+		if inst.Id == excludeID || storedVGPUDevicePath(&inst.StoredMetadata) != devicePath || inst.HypervisorPID == nil {
 			continue
 		}
 		if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index df2c265e..5a7f566a 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -126,6 +126,24 @@ func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T)
 	require.Error(t, err)
 }
 
+func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("legacy-claimant"))
+	pid := os.Getpid()
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            "legacy-claimant",
+		Name:          "legacy-claimant",
+		GPUMdevUUID:   "legacy-uuid",
+		HypervisorPID: &pid,
+	}}))
+
+	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/mdev/devices/legacy-uuid")
+	require.NoError(t, err)
+	assert.True(t, claimed)
+}
+
 func TestStoredVGPUDevicePath(t *testing.T) {
 	t.Parallel()
 

From f2ded22cf3b36e3510001d24135b6df57ed2699f Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Fri, 7 Aug 2026 20:45:49 +0000
Subject: [PATCH 08/17] Bind the live-claimant test socket under /tmp for macOS

---
 lib/instances/lifecycle_noop_test.go | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go
index 3eb5a46c..db0527a1 100644
--- a/lib/instances/lifecycle_noop_test.go
+++ b/lib/instances/lifecycle_noop_test.go
@@ -200,7 +200,14 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) {
 	claimantID := "inst-live-claimant"
 	require.NoError(t, m.ensureDirectories(claimantID))
 	pid := os.Getpid()
-	socketPath := m.paths.InstanceSocket(claimantID, "noop.sock")
+	// Bind under /tmp: a t.TempDir()-derived path exceeds the macOS AF_UNIX
+	// path limit.
+	socketDir, err := os.MkdirTemp("/tmp", "hypeman-claimant-socket-")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = os.RemoveAll(socketDir)
+	})
+	socketPath := filepath.Join(socketDir, "noop.sock")
 	listener, err := net.Listen("unix", socketPath)
 	require.NoError(t, err)
 	defer listener.Close()

From e3940e9eba408b154655ac79113ae4e977277a86 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Fri, 7 Aug 2026 20:45:49 +0000
Subject: [PATCH 09/17] Surface retained vGPU cleanup through a typed create
 error and manager seam

Replace the go:linkname shadow of devices.hostVendorVFIO with createVGPU/destroyVGPU manager fields, and wrap failed creates whose rollback release also failed in VGPUCleanupPendingError so the API can point callers at the retained instance record.
---
 cmd/api/api/instances.go   |  7 +++
 lib/instances/create.go    | 28 +++++++++---
 lib/instances/manager.go   |  4 ++
 lib/instances/start.go     |  8 ++--
 lib/instances/vgpu.go      | 32 ++++++++++++-
 lib/instances/vgpu_test.go | 92 ++++++++++++++++++++++++--------------
 6 files changed, 126 insertions(+), 45 deletions(-)

diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go
index 3e13f080..e4293ea9 100644
--- a/cmd/api/api/instances.go
+++ b/cmd/api/api/instances.go
@@ -343,6 +343,7 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 
 	inst, err := s.InstanceManager.CreateInstance(ctx, domainReq)
 	if err != nil {
+		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
 		case errors.Is(err, instances.ErrImageNotReady):
 			return oapi.CreateInstance400JSONResponse{
@@ -389,6 +390,12 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 				Code:    "not_found",
 				Message: err.Error(),
 			}, nil
+		case errors.As(err, &vgpuPending):
+			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
+			return oapi.CreateInstance500JSONResponse{
+				Code:    "vgpu_cleanup_pending",
+				Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID),
+			}, nil
 		default:
 			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
 			return oapi.CreateInstance500JSONResponse{
diff --git a/lib/instances/create.go b/lib/instances/create.go
index 2ba23d7a..5ab93bcc 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -268,10 +268,20 @@ func (m *manager) createInstance(
 	var stored *StoredMetadata
 	var retainedVGPU *StoredMetadata
 
-	// Setup cleanup stack early so device attachment errors trigger cleanup
+	// Setup cleanup stack early so device attachment errors trigger cleanup.
+	// When rollback retains a vGPU assignment, surface the retained instance
+	// ID to the caller so the record is discoverable and can be deleted to
+	// retry the release. The wrapping defer is registered first so it runs
+	// after cu.Clean has decided whether metadata was retained.
+	vgpuRetained := false
+	defer func() {
+		if retErr != nil && vgpuRetained {
+			retErr = &VGPUCleanupPendingError{InstanceID: id, Err: retErr}
+		}
+	}()
 	cu := cleanup.Make(func() {
 		log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id)
-		m.cleanupFailedCreate(ctx, id, retainedVGPU)
+		vgpuRetained = m.cleanupFailedCreate(ctx, id, retainedVGPU)
 	})
 	defer cu.Clean()
 
@@ -288,7 +298,7 @@ func (m *manager) createInstance(
 	// Handle vGPU profile request
 	if req.GPU != nil && req.GPU.Profile != "" {
 		log.InfoContext(ctx, "creating vGPU", "instance_id", id, "profile", req.GPU.Profile)
-		gpuDevice, err = devices.CreateVGPU(ctx, req.GPU.Profile, id)
+		gpuDevice, err = m.createVGPUDevice(ctx, req.GPU.Profile, id)
 		if err != nil {
 			log.ErrorContext(ctx, "failed to create vGPU", "profile", req.GPU.Profile, "error", err)
 			return nil, wrapCreateVGPUErr(req.GPU.Profile, err)
@@ -306,7 +316,7 @@ func (m *manager) createInstance(
 				MdevUUID:   gpuDevice.MdevUUID,
 				InstanceID: id,
 			}
-			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
+			if err := m.destroyVGPUAssignment(ctx, assignment); err != nil {
 				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err)
 				retainedVGPU = stored
 				if retainedVGPU == nil {
@@ -596,16 +606,18 @@ func (m *manager) createInstance(
 	return &finalInst, nil
 }
 
-func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) {
+// cleanupFailedCreate reports whether it retained instance metadata for a
+// vGPU assignment whose release failed during rollback.
+func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool {
 	if retainedVGPU == nil {
 		m.deleteInstanceData(id)
-		return
+		return false
 	}
 
 	log := logger.FromContext(ctx)
 	if err := m.ensureDirectories(id); err != nil {
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
-		return
+		return false
 	}
 	retained := StoredMetadata{
 		Id:            id,
@@ -615,7 +627,9 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
+		return false
 	}
+	return true
 }
 
 // validateCreateRequest validates the create instance request.
diff --git a/lib/instances/manager.go b/lib/instances/manager.go
index d4206c91..ebb98081 100644
--- a/lib/instances/manager.go
+++ b/lib/instances/manager.go
@@ -180,6 +180,8 @@ type manager struct {
 	tracer                    trace.Tracer
 	now                       func() time.Time
 	writeFile                 func(string, []byte, os.FileMode) error
+	createVGPU                func(context.Context, string, string) (*devices.VGPUDevice, error)
+	destroyVGPU               func(context.Context, devices.VGPUAssignment) error
 	deleteSnapshotFn          func(context.Context, string) error
 	egressProxy               *egressproxy.Service
 	egressProxyServiceOptions egressproxy.ServiceOptions
@@ -278,6 +280,8 @@ func NewManagerWithConfigE(p *paths.Paths, imageManager images.Manager, systemMa
 		defaultHypervisor:                defaultHypervisor,
 		now:                              time.Now,
 		writeFile:                        os.WriteFile,
+		createVGPU:                       devices.CreateVGPU,
+		destroyVGPU:                      devices.DestroyVGPU,
 		meter:                            meter,
 		tracer:                           tracer,
 		guestMemoryPolicy:                policy,
diff --git a/lib/instances/start.go b/lib/instances/start.go
index 8cf7eceb..abff4ef7 100644
--- a/lib/instances/start.go
+++ b/lib/instances/start.go
@@ -161,11 +161,11 @@ func (m *manager) startInstance(
 	// 4b. Recreate the vGPU if this instance had a GPU profile
 	// Note: GPU availability was already validated in step 2b
 	if stored.GPUProfile != "" {
-		log.InfoContext(ctx, "creating vGPU mdev for start", "instance_id", id, "profile", stored.GPUProfile)
-		device, err := devices.CreateVGPU(ctx, stored.GPUProfile, id)
+		log.InfoContext(ctx, "creating vGPU for start", "instance_id", id, "profile", stored.GPUProfile)
+		device, err := m.createVGPUDevice(ctx, stored.GPUProfile, id)
 		if err != nil {
 			log.ErrorContext(ctx, "failed to create vGPU", "instance_id", id, "profile", stored.GPUProfile, "error", err)
-			return nil, fmt.Errorf("create vGPU mdev for profile %s: %w", stored.GPUProfile, err)
+			return nil, fmt.Errorf("create vGPU for profile %s: %w", stored.GPUProfile, err)
 		}
 		setStoredVGPUDevice(stored, device)
 		// Add vGPU cleanup to stack
@@ -176,7 +176,7 @@ func (m *manager) startInstance(
 				MdevUUID:   device.MdevUUID,
 				InstanceID: id,
 			}
-			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
+			if err := m.destroyVGPUAssignment(ctx, assignment); err != nil {
 				log.WarnContext(ctx, "failed to destroy vGPU on cleanup", "instance_id", id, "error", err)
 				if saveErr := m.saveMetadata(meta); saveErr != nil {
 					log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", saveErr)
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index b16524b3..fe655278 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -9,6 +9,36 @@ import (
 	"github.com/kernel/hypeman/lib/logger"
 )
 
+// VGPUCleanupPendingError reports a failed create whose vGPU release also
+// failed during rollback. The instance record identified by InstanceID is
+// retained so the release can be retried; deleting the instance retries it.
+type VGPUCleanupPendingError struct {
+	InstanceID string
+	Err        error
+}
+
+func (e *VGPUCleanupPendingError) Error() string {
+	return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID)
+}
+
+func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err }
+
+func (m *manager) createVGPUDevice(ctx context.Context, profileName, instanceID string) (*devices.VGPUDevice, error) {
+	create := m.createVGPU
+	if create == nil {
+		create = devices.CreateVGPU
+	}
+	return create(ctx, profileName, instanceID)
+}
+
+func (m *manager) destroyVGPUAssignment(ctx context.Context, assignment devices.VGPUAssignment) error {
+	destroy := m.destroyVGPU
+	if destroy == nil {
+		destroy = devices.DestroyVGPU
+	}
+	return destroy(ctx, assignment)
+}
+
 func setStoredVGPUDevice(stored *StoredMetadata, device *devices.VGPUDevice) {
 	stored.GPUFramework = device.Framework
 	stored.GPUDevicePath = device.SysfsPath
@@ -38,7 +68,7 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata)
 				MdevUUID:   stored.GPUMdevUUID,
 				InstanceID: stored.Id,
 			}
-			if err := devices.DestroyVGPU(ctx, assignment); err != nil {
+			if err := m.destroyVGPUAssignment(ctx, assignment); err != nil {
 				return err
 			}
 		}
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 5a7f566a..4224b6ba 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -2,11 +2,11 @@ package instances
 
 import (
 	"context"
+	"errors"
 	"os"
 	"path/filepath"
 	"sync"
 	"testing"
-	_ "unsafe"
 
 	"github.com/kernel/hypeman/lib/devices"
 	"github.com/kernel/hypeman/lib/paths"
@@ -32,7 +32,7 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 		DataDir:        m.paths.InstanceDir("failed-create"),
 	}
 
-	m.cleanupFailedCreate(context.Background(), stored.Id, stored)
+	assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored))
 
 	retained, err := m.loadMetadata(stored.Id)
 	require.NoError(t, err)
@@ -48,40 +48,43 @@ func TestCleanupFailedCreateRetainsVGPUAssignment(t *testing.T) {
 	assert.Empty(t, retained.DataDir)
 }
 
-//go:linkname hostVendorVFIO github.com/kernel/hypeman/lib/devices.hostVendorVFIO
-var hostVendorVFIO vendorVFIOSysfs
+func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("failed-create"))
 
-type vendorVFIOSysfs struct {
-	pciDevicesPath  string
-	procPath        string
-	vfioDevicesPath string
-	owners          map[string]string
+	assert.False(t, m.cleanupFailedCreate(context.Background(), "failed-create", nil))
+	_, err := m.loadMetadata("failed-create")
+	require.Error(t, err)
 }
 
-func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
-	root := t.TempDir()
-	pciDevicesPath := filepath.Join(root, "sys", "bus", "pci", "devices")
-	vfAddress := "0000:82:00.4"
-	nvidiaPath := filepath.Join(pciDevicesPath, vfAddress, "nvidia")
-	require.NoError(t, os.MkdirAll(nvidiaPath, 0o755))
-	require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "current_vgpu_type"), []byte("0"), 0o644))
-	require.NoError(t, os.WriteFile(filepath.Join(nvidiaPath, "creatable_vgpu_types"), []byte("ID    : vGPU Name\n1148  : NVIDIA L40S-2Q\n"), 0o644))
-
-	originalVendorVFIO := hostVendorVFIO
-	hostVendorVFIO = vendorVFIOSysfs{
-		pciDevicesPath:  pciDevicesPath,
-		procPath:        filepath.Join(root, "proc"),
-		vfioDevicesPath: filepath.Join(root, "dev", "vfio", "devices"),
-		owners:          make(map[string]string),
-	}
-	t.Cleanup(func() { hostVendorVFIO = originalVendorVFIO })
-	require.NoError(t, os.MkdirAll(hostVendorVFIO.procPath, 0o755))
+func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) {
+	t.Parallel()
+
+	cause := errors.New("boot failed")
+	err := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause}
+	assert.ErrorIs(t, err, cause)
+	assert.Contains(t, err.Error(), "inst-1")
+}
 
+func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) {
+	t.Helper()
 	m := &manager{
 		paths:           paths.New(t.TempDir()),
 		imageManager:    readyFixtureImageManager{name: "test-image"},
 		instanceLocks:   sync.Map{},
 		bootMarkerScans: sync.Map{},
+		createVGPU: func(_ context.Context, profileName, _ string) (*devices.VGPUDevice, error) {
+			return &devices.VGPUDevice{
+				Framework:   devices.VGPUFrameworkVendorVFIO,
+				VFAddress:   "0000:82:00.4",
+				ProfileType: "1148",
+				ProfileName: profileName,
+				SysfsPath:   "/sys/bus/pci/devices/0000:82:00.4",
+			}, nil
+		},
+		destroyVGPU: destroy,
 	}
 	const id = "start-rollback"
 	require.NoError(t, m.ensureDirectories(id))
@@ -94,25 +97,48 @@ func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
 		SocketPath:     m.paths.InstanceSocket(id, "noop.sock"),
 		DataDir:        m.paths.InstanceDir(id),
 	}}))
+	return m, id
+}
+
+func TestStartRollbackClearsVGPUAssignmentAfterSuccessfulDestroy(t *testing.T) {
+	var destroyed []devices.VGPUAssignment
+	m, id := newStartRollbackVGPUManager(t, func(_ context.Context, assignment devices.VGPUAssignment) error {
+		destroyed = append(destroyed, assignment)
+		return nil
+	})
 
-	t.Setenv("TMPDIR", filepath.Join(root, "missing"))
+	t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing"))
 	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{})
 	require.Error(t, err)
 
+	require.Len(t, destroyed, 1)
+	assert.Equal(t, devices.VGPUAssignment{
+		Framework:  devices.VGPUFrameworkVendorVFIO,
+		DevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+		InstanceID: id,
+	}, destroyed[0])
+
 	stored, err := m.loadMetadata(id)
 	require.NoError(t, err)
 	assert.Equal(t, "NVIDIA L40S-2Q", stored.GPUProfile)
 	assert.Empty(t, stored.GPUFramework)
 	assert.Empty(t, stored.GPUDevicePath)
 	assert.Empty(t, stored.GPUMdevUUID)
-	assertFileContents(t, filepath.Join(nvidiaPath, "current_vgpu_type"), "0")
 }
 
-func assertFileContents(t *testing.T, path, want string) {
-	t.Helper()
-	got, err := os.ReadFile(path)
+func TestStartRollbackRetainsVGPUAssignmentAfterFailedDestroy(t *testing.T) {
+	m, id := newStartRollbackVGPUManager(t, func(context.Context, devices.VGPUAssignment) error {
+		return errors.New("destroy failed")
+	})
+
+	t.Setenv("TMPDIR", filepath.Join(t.TempDir(), "missing"))
+	_, err := m.startInstance(context.Background(), id, StartInstanceRequest{})
+	require.Error(t, err)
+
+	stored, err := m.loadMetadata(id)
 	require.NoError(t, err)
-	assert.Equal(t, want, string(got))
+	assert.Equal(t, devices.VGPUFrameworkVendorVFIO, stored.GPUFramework)
+	assert.Equal(t, "/sys/bus/pci/devices/0000:82:00.4", stored.GPUDevicePath)
 }
 
 func TestVGPUAssignmentClaimedByLiveInstanceFailsOnInvalidMetadata(t *testing.T) {

From 483af9af537ba346aaed9a149c3fa5deeb0e3f8f Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Fri, 7 Aug 2026 20:45:49 +0000
Subject: [PATCH 10/17] Generalize the create vGPU error text

---
 lib/instances/create.go           | 2 +-
 lib/instances/create_mdev_test.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index 5ab93bcc..938203ca 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -56,7 +56,7 @@ func wrapCreateVGPUErr(profile string, err error) error {
 	if errors.Is(err, devices.ErrVGPUNotSupportedOnMacOS) {
 		return fmt.Errorf("%w: %w", ErrInvalidRequest, err)
 	}
-	return fmt.Errorf("create vGPU mdev for profile %s: %w", profile, err)
+	return fmt.Errorf("create vGPU for profile %s: %w", profile, err)
 }
 
 // generateVsockCID converts first 8 chars of instance ID to a unique CID
diff --git a/lib/instances/create_mdev_test.go b/lib/instances/create_mdev_test.go
index e6e4f55c..05b3e9d8 100644
--- a/lib/instances/create_mdev_test.go
+++ b/lib/instances/create_mdev_test.go
@@ -63,7 +63,7 @@ func TestWrapCreateVGPUErr(t *testing.T) {
 		{
 			name:        "other vGPU error",
 			err:         errors.New("boom"),
-			wantMessage: "create vGPU mdev for profile profile: boom",
+			wantMessage: "create vGPU for profile profile: boom",
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {

From 76ef364b26c4124ba220db7873353a5d6f3c453f Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sat, 8 Aug 2026 00:58:24 +0000
Subject: [PATCH 11/17] Scope vGPU claim scan to vendor VFIO and close
 reconcile gaps

---
 cmd/api/api/instances.go             | 14 +++++++------
 cmd/api/api/instances_test.go        | 31 ++++++++++++++++++++++++++++
 cmd/api/main.go                      |  8 +++++--
 cmd/api/main_test.go                 | 30 +++++++++++++++++++++++++++
 lib/instances/lifecycle_noop_test.go |  4 ++--
 lib/instances/vgpu.go                | 15 +++++++++++---
 lib/instances/vgpu_test.go           | 21 +++++++++++++++++++
 7 files changed, 110 insertions(+), 13 deletions(-)

diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go
index e4293ea9..a4174e11 100644
--- a/cmd/api/api/instances.go
+++ b/cmd/api/api/instances.go
@@ -345,6 +345,14 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 	if err != nil {
 		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
+		// Checked first: it wraps the original create error, so a later
+		// errors.Is case would match the cause and hide the retained instance.
+		case errors.As(err, &vgpuPending):
+			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
+			return oapi.CreateInstance500JSONResponse{
+				Code:    "vgpu_cleanup_pending",
+				Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID),
+			}, nil
 		case errors.Is(err, instances.ErrImageNotReady):
 			return oapi.CreateInstance400JSONResponse{
 				Code:    "image_not_ready",
@@ -390,12 +398,6 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 				Code:    "not_found",
 				Message: err.Error(),
 			}, nil
-		case errors.As(err, &vgpuPending):
-			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
-			return oapi.CreateInstance500JSONResponse{
-				Code:    "vgpu_cleanup_pending",
-				Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID),
-			}, nil
 		default:
 			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
 			return oapi.CreateInstance500JSONResponse{
diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go
index f37ebfbe..4f19d6fe 100644
--- a/cmd/api/api/instances_test.go
+++ b/cmd/api/api/instances_test.go
@@ -16,6 +16,7 @@ import (
 	"github.com/kernel/hypeman/lib/instances"
 	"github.com/kernel/hypeman/lib/instances/phasetracking"
 	mw "github.com/kernel/hypeman/lib/middleware"
+	"github.com/kernel/hypeman/lib/network"
 	"github.com/kernel/hypeman/lib/oapi"
 	"github.com/kernel/hypeman/lib/paths"
 	restartpolicy "github.com/kernel/hypeman/lib/restart-policy"
@@ -46,6 +47,36 @@ func TestGetInstance_NotFound(t *testing.T) {
 	require.Error(t, err)
 }
 
+type createErrorInstanceManager struct {
+	instances.Manager
+	err error
+}
+
+func (m createErrorInstanceManager) CreateInstance(context.Context, instances.CreateInstanceRequest) (*instances.Instance, error) {
+	return nil, m.err
+}
+
+// A retained-assignment error must win over the mapping of the create error
+// it wraps, or the response omits the instance the caller has to delete.
+func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
+		InstanceID: "inst-1",
+		Err:        network.ErrNameExists,
+	}}
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+		Body: &oapi.CreateInstanceRequest{Image: "test-image"},
+	})
+	require.NoError(t, err)
+
+	pending, ok := resp.(oapi.CreateInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "inst-1")
+}
+
 func TestCreateInstance_AutoPullImage(t *testing.T) {
 	t.Parallel()
 	if _, err := os.Stat("/dev/kvm"); os.IsNotExist(err) {
diff --git a/cmd/api/main.go b/cmd/api/main.go
index 44fa1b16..b425eaa2 100644
--- a/cmd/api/main.go
+++ b/cmd/api/main.go
@@ -179,10 +179,14 @@ func liveInstanceVGPUDevicePaths(ctx context.Context, instanceManager instances.
 	}
 	protected := make(map[string]struct{})
 	for _, inst := range allInstances {
-		if inst.GPUDevicePath == "" || inst.HypervisorPID == nil {
+		if inst.GPUDevicePath == "" {
 			continue
 		}
-		if !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
+		// A nil PID does not mean the assignment is orphaned: the PID is
+		// persisted only after the hypervisor starts, so a crash during boot
+		// leaves the device path without one. Only skip protection when the
+		// recorded hypervisor is known to be gone.
+		if inst.HypervisorPID != nil && !instances.HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
 			continue
 		}
 		protected[inst.GPUDevicePath] = struct{}{}
diff --git a/cmd/api/main_test.go b/cmd/api/main_test.go
index 34dbba42..573a404c 100644
--- a/cmd/api/main_test.go
+++ b/cmd/api/main_test.go
@@ -2,15 +2,18 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os/exec"
 	"testing"
 	"time"
 
 	"github.com/getkin/kin-openapi/openapi3filter"
 	"github.com/go-chi/chi/v5"
 	"github.com/golang-jwt/jwt/v5"
+	"github.com/kernel/hypeman/lib/instances"
 	mw "github.com/kernel/hypeman/lib/middleware"
 	"github.com/kernel/hypeman/lib/oapi"
 	nethttpmiddleware "github.com/oapi-codegen/nethttp-middleware"
@@ -338,3 +341,30 @@ func TestImageNameWithSlashes_URLEncoding(t *testing.T) {
 		})
 	}
 }
+
+type vgpuReconcileManagerStub struct {
+	instances.Manager
+	list []instances.Instance
+}
+
+func (s vgpuReconcileManagerStub) ListInstancesForReconcile(context.Context) ([]instances.Instance, error) {
+	return s.list, nil
+}
+
+// The hypervisor PID is persisted only after boot, so an assignment without
+// one may belong to a VM that is still starting and must stay protected.
+func TestLiveInstanceVGPUDevicePathsProtectsAssignmentsWithoutPID(t *testing.T) {
+	dead := exec.Command("true")
+	require.NoError(t, dead.Run())
+	deadPID := dead.Process.Pid
+
+	manager := vgpuReconcileManagerStub{list: []instances.Instance{
+		{StoredMetadata: instances.StoredMetadata{Id: "booting", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4"}},
+		{StoredMetadata: instances.StoredMetadata{Id: "dead", GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.5", HypervisorPID: &deadPID}},
+	}}
+
+	protected, err := liveInstanceVGPUDevicePaths(context.Background(), manager)
+	require.NoError(t, err)
+	assert.Contains(t, protected, "/sys/bus/pci/devices/0000:82:00.4")
+	assert.NotContains(t, protected, "/sys/bus/pci/devices/0000:82:00.5")
+}
diff --git a/lib/instances/lifecycle_noop_test.go b/lib/instances/lifecycle_noop_test.go
index db0527a1..f36aeb84 100644
--- a/lib/instances/lifecycle_noop_test.go
+++ b/lib/instances/lifecycle_noop_test.go
@@ -193,7 +193,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) {
 	meta, err := m.loadMetadata(id)
 	require.NoError(t, err)
 	meta.GPUProfile = "NVIDIA L40S-2Q"
-	meta.GPUFramework = devices.VGPUFramework("future-framework")
+	meta.GPUFramework = devices.VGPUFrameworkVendorVFIO
 	meta.GPUDevicePath = "/sys/bus/pci/devices/0000:82:00.4"
 	require.NoError(t, m.saveMetadata(meta))
 
@@ -221,7 +221,7 @@ func TestDeleteDropsStaleVGPUClaimedByLiveInstance(t *testing.T) {
 		SocketPath:     socketPath,
 		DataDir:        m.paths.InstanceDir(claimantID),
 		GPUProfile:     "NVIDIA L40S-2Q",
-		GPUFramework:   devices.VGPUFramework("future-framework"),
+		GPUFramework:   devices.VGPUFrameworkVendorVFIO,
 		GPUDevicePath:  "/sys/bus/pci/devices/0000:82:00.4",
 	}}))
 
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index fe655278..9ae4028f 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -54,9 +54,18 @@ func clearStoredVGPUDevice(stored *StoredMetadata) {
 func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata) error {
 	path := storedVGPUDevicePath(stored)
 	if path != "" {
-		claimed, err := m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path)
-		if err != nil {
-			return err
+		// Vendor VFIO VFs are reused across instances, so stale metadata can
+		// point at a path claimed by a live instance and the release must fail
+		// closed on an incomplete inventory. mdev UUIDs are unique and never
+		// reused, so skip the scan there — it would let one unreadable
+		// metadata file block every mdev release on the host.
+		claimed := false
+		if stored.GPUFramework == devices.VGPUFrameworkVendorVFIO {
+			var err error
+			claimed, err = m.vgpuAssignmentClaimedByLiveInstance(ctx, stored.Id, path)
+			if err != nil {
+				return err
+			}
 		}
 		if claimed {
 			logger.FromContext(ctx).WarnContext(ctx, "dropping stale vGPU assignment claimed by another live instance",
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 4224b6ba..562c29c7 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -170,6 +170,27 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.
 	assert.True(t, claimed)
 }
 
+func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{
+		paths:       paths.New(t.TempDir()),
+		destroyVGPU: func(context.Context, devices.VGPUAssignment) error { return nil },
+	}
+	require.NoError(t, m.ensureDirectories("invalid-instance"))
+	require.NoError(t, os.WriteFile(m.paths.InstanceMetadata("invalid-instance"), []byte("{"), 0o644))
+
+	stored := &StoredMetadata{
+		Id:            "mdev-instance",
+		GPUFramework:  devices.VGPUFrameworkMdev,
+		GPUMdevUUID:   "uuid-1",
+		GPUDevicePath: "/sys/bus/mdev/devices/uuid-1",
+	}
+	require.NoError(t, m.releaseStoredVGPU(context.Background(), stored),
+		"an unreadable metadata file must not block mdev releases")
+	assert.Empty(t, stored.GPUDevicePath)
+}
+
 func TestStoredVGPUDevicePath(t *testing.T) {
 	t.Parallel()
 

From d80e28e90355546fc7783176644a6f0ce0a2b0e1 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sun, 9 Aug 2026 06:25:10 +0000
Subject: [PATCH 12/17] Harden the vendor VFIO release path

Enable vendor VFIO dispatch in CreateVGPU now that the lifecycle
persists assignments durably and guards releases.

Protect nil-PID claims in the release guard: the hypervisor PID is only
persisted after the claimant boots, so a matching assignment without a
PID must be treated as live, matching the startup reconcile protection.

Scan raw metadata instead of hydrating instances for the claim check.
Hydration derives state through hypervisor queries for every instance
on the host, which every vendor VFIO release would pay; the guard only
needs the stored assignment, PID, and socket. Unreadable metadata still
fails the release closed.

Report pending vGPU cleanup even when retaining the rollback record
fails: the destroy already failed, so the caller must learn about the
outstanding assignment either way.
---
 integration/vgpu_test.go   |  5 ----
 lib/devices/vgpu_linux.go  |  5 +---
 lib/instances/create.go    | 11 +++++---
 lib/instances/vgpu.go      | 36 +++++++++++++++++++++++----
 lib/instances/vgpu_test.go | 51 ++++++++++++++++++++++++++++++++++++++
 5 files changed, 90 insertions(+), 18 deletions(-)

diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go
index 7873aa35..1f1a8277 100644
--- a/integration/vgpu_test.go
+++ b/integration/vgpu_test.go
@@ -324,11 +324,6 @@ func checkVGPUTestPrerequisites() (string, string) {
 	if framework == devices.VGPUFrameworkNone {
 		return "vGPU test requires SR-IOV VFs with an mdev or vendor VFIO vGPU framework", ""
 	}
-	if framework == devices.VGPUFrameworkVendorVFIO {
-		// CreateVGPU rejects vendor VFIO until the instance lifecycle
-		// integration lands.
-		return "vGPU test requires the vendor VFIO instance lifecycle integration", ""
-	}
 
 	// Check for available profiles
 	profiles, err := devices.ListGPUProfiles()
diff --git a/lib/devices/vgpu_linux.go b/lib/devices/vgpu_linux.go
index 72fe3b94..a4ccd2db 100644
--- a/lib/devices/vgpu_linux.go
+++ b/lib/devices/vgpu_linux.go
@@ -73,10 +73,7 @@ func CreateVGPU(ctx context.Context, profileName, instanceID string) (*VGPUDevic
 			MdevUUID:    mdev.UUID,
 		}, nil
 	case VGPUFrameworkVendorVFIO:
-		// The instance lifecycle does not yet persist vendor VFIO assignments
-		// durably or guard their release against live claims, so keep the
-		// backend out of the create path until that integration lands.
-		return nil, fmt.Errorf("vendor VFIO vGPU support is not yet integrated with the instance lifecycle")
+		return hostVendorVFIO.create(ctx, profileName, instanceID)
 	default:
 		return nil, fmt.Errorf("vGPU framework not available")
 	}
diff --git a/lib/instances/create.go b/lib/instances/create.go
index 938203ca..e6954efc 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -606,8 +606,11 @@ func (m *manager) createInstance(
 	return &finalInst, nil
 }
 
-// cleanupFailedCreate reports whether it retained instance metadata for a
-// vGPU assignment whose release failed during rollback.
+// cleanupFailedCreate reports whether a vGPU assignment is still outstanding
+// after a failed create. The vGPU destroy already failed when retainedVGPU is
+// set, so the pending cleanup is reported even when the retention record
+// cannot be persisted — in that case the assignment is orphaned until the
+// next startup reconcile, and the caller must still surface it.
 func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool {
 	if retainedVGPU == nil {
 		m.deleteInstanceData(id)
@@ -617,7 +620,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	log := logger.FromContext(ctx)
 	if err := m.ensureDirectories(id); err != nil {
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
-		return false
+		return true
 	}
 	retained := StoredMetadata{
 		Id:            id,
@@ -627,7 +630,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
-		return false
+		return true
 	}
 	return true
 }
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index 9ae4028f..f9d67844 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 
 	"github.com/kernel/hypeman/lib/devices"
+	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/logger"
 )
 
@@ -86,19 +87,44 @@ func (m *manager) releaseStoredVGPU(ctx context.Context, stored *StoredMetadata)
 	return nil
 }
 
+// vgpuAssignmentClaimedByLiveInstance reports whether another instance's
+// stored metadata claims devicePath. It reads raw metadata instead of
+// hydrating full instances: the scan runs on every vendor VFIO release, and
+// deriving state would query the hypervisor of every instance on the host.
+// It fails closed: unreadable metadata is an error, and a matching claim
+// without a persisted PID counts as live because the PID is only persisted
+// after the claimant's hypervisor starts.
 func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, excludeID, devicePath string) (bool, error) {
-	instances, err := m.ListInstancesForReconcile(ctx)
+	files, err := m.listMetadataFilesWithStatErrors(true)
 	if err != nil {
 		return false, fmt.Errorf("list instances for vGPU release check: %w", err)
 	}
-	for i := range instances {
-		inst := &instances[i]
-		if inst.Id == excludeID || storedVGPUDevicePath(&inst.StoredMetadata) != devicePath || inst.HypervisorPID == nil {
+	for _, file := range files {
+		id := filepath.Base(filepath.Dir(file))
+		if id == excludeID {
 			continue
 		}
-		if HypervisorProcessExists(*inst.HypervisorPID, inst.SocketPath) {
+		meta, err := m.loadMetadata(id)
+		if err != nil {
+			return false, fmt.Errorf("load metadata for vGPU release check: instance %s: %w", id, err)
+		}
+		stored := &meta.StoredMetadata
+		if storedVGPUDevicePath(stored) != devicePath {
+			continue
+		}
+		if stored.HypervisorPID == nil {
 			return true, nil
 		}
+		if HypervisorProcessExists(*stored.HypervisorPID, stored.SocketPath) {
+			return true, nil
+		}
+		// The stored PID can be stale after a hypeman restart; a live owner
+		// of the claimant's socket still marks the claim as live.
+		if stored.SocketPath != "" && !ProcessExists(*stored.HypervisorPID) {
+			if owner, _, err := hypervisor.ResolveProcessPID(stored.SocketPath); err == nil && ProcessExists(owner) {
+				return true, nil
+			}
+		}
 	}
 	return false, nil
 }
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 562c29c7..b456298b 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -59,6 +59,23 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) {
 	require.Error(t, err)
 }
 
+func TestCleanupFailedCreateReportsPendingWhenRetentionFails(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	// A file at the guests directory path makes ensureDirectories fail even
+	// when running as root.
+	require.NoError(t, os.WriteFile(m.paths.GuestsDir(), nil, 0o644))
+
+	stored := &StoredMetadata{
+		Id:            "failed-create",
+		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+	}
+	assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored),
+		"a failed retention must still report the outstanding vGPU assignment")
+}
+
 func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) {
 	t.Parallel()
 
@@ -170,6 +187,40 @@ func TestVGPUAssignmentClaimedByLiveInstanceNormalizesLegacyMdevPath(t *testing.
 	assert.True(t, claimed)
 }
 
+func TestVGPUAssignmentClaimedByLiveInstanceProtectsNilPIDClaim(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("booting-claimant"))
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            "booting-claimant",
+		Name:          "booting-claimant",
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+	}}))
+
+	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4")
+	require.NoError(t, err)
+	assert.True(t, claimed, "a matching claim without a persisted PID must be treated as live: the PID is only persisted after the claimant boots")
+}
+
+func TestVGPUAssignmentClaimedByLiveInstanceIgnoresDeadClaim(t *testing.T) {
+	t.Parallel()
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	require.NoError(t, m.ensureDirectories("dead-claimant"))
+	deadPID := 1 << 30
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            "dead-claimant",
+		Name:          "dead-claimant",
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+		HypervisorPID: &deadPID,
+	}}))
+
+	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", "/sys/bus/pci/devices/0000:82:00.4")
+	require.NoError(t, err)
+	assert.False(t, claimed, "a claim whose hypervisor is gone must not block the release")
+}
+
 func TestReleaseStoredVGPUSkipsClaimScanForMdev(t *testing.T) {
 	t.Parallel()
 

From d7dab392a0b353aa7d348488648be325a52a8f0d Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sun, 9 Aug 2026 07:40:01 +0000
Subject: [PATCH 13/17] Fail closed on retained vGPU cleanup

---
 lib/instances/create.go                      | 26 +++++-------
 lib/instances/process_identity_linux_test.go | 43 ++++++++++++++++++++
 lib/instances/vgpu.go                        | 21 ++++------
 lib/instances/vgpu_test.go                   | 23 ++++++-----
 4 files changed, 76 insertions(+), 37 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index e6954efc..ccdc6f42 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -269,19 +269,18 @@ func (m *manager) createInstance(
 	var retainedVGPU *StoredMetadata
 
 	// Setup cleanup stack early so device attachment errors trigger cleanup.
-	// When rollback retains a vGPU assignment, surface the retained instance
-	// ID to the caller so the record is discoverable and can be deleted to
-	// retry the release. The wrapping defer is registered first so it runs
-	// after cu.Clean has decided whether metadata was retained.
-	vgpuRetained := false
+	// When rollback cannot release a vGPU assignment, report whether its
+	// retention record was persisted. The wrapping defer is registered first
+	// so it runs after cu.Clean has attempted to retain the metadata.
+	vgpuPersisted := false
 	defer func() {
-		if retErr != nil && vgpuRetained {
-			retErr = &VGPUCleanupPendingError{InstanceID: id, Err: retErr}
+		if retErr != nil && retainedVGPU != nil {
+			retErr = &VGPUCleanupPendingError{InstanceID: id, Retained: vgpuPersisted, Err: retErr}
 		}
 	}()
 	cu := cleanup.Make(func() {
 		log.DebugContext(ctx, "cleaning up instance on error", "instance_id", id)
-		vgpuRetained = m.cleanupFailedCreate(ctx, id, retainedVGPU)
+		vgpuPersisted = m.cleanupFailedCreate(ctx, id, retainedVGPU)
 	})
 	defer cu.Clean()
 
@@ -606,11 +605,8 @@ func (m *manager) createInstance(
 	return &finalInst, nil
 }
 
-// cleanupFailedCreate reports whether a vGPU assignment is still outstanding
-// after a failed create. The vGPU destroy already failed when retainedVGPU is
-// set, so the pending cleanup is reported even when the retention record
-// cannot be persisted — in that case the assignment is orphaned until the
-// next startup reconcile, and the caller must still surface it.
+// cleanupFailedCreate reports whether the retention record for a vGPU
+// assignment whose release failed during rollback was persisted.
 func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVGPU *StoredMetadata) bool {
 	if retainedVGPU == nil {
 		m.deleteInstanceData(id)
@@ -620,7 +616,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	log := logger.FromContext(ctx)
 	if err := m.ensureDirectories(id); err != nil {
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
-		return true
+		return false
 	}
 	retained := StoredMetadata{
 		Id:            id,
@@ -630,7 +626,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
-		return true
+		return false
 	}
 	return true
 }
diff --git a/lib/instances/process_identity_linux_test.go b/lib/instances/process_identity_linux_test.go
index ff260f45..f5e91d3f 100644
--- a/lib/instances/process_identity_linux_test.go
+++ b/lib/instances/process_identity_linux_test.go
@@ -14,6 +14,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/kernel/hypeman/lib/devices"
+	"github.com/kernel/hypeman/lib/paths"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -146,6 +148,47 @@ func TestRefreshHypervisorPIDPrefersSocketOwnerOverLiveStoredPID(t *testing.T) {
 	assert.Equal(t, owner.Process.Pid, *stored.HypervisorPID)
 }
 
+func TestVGPUAssignmentClaimedByLiveInstanceProtectsReusedPIDClaim(t *testing.T) {
+	socketPath := filepath.Join(t.TempDir(), "test.sock")
+	owner := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$")
+	owner.Env = append(os.Environ(), "HYPERVISOR_SOCKET_HELPER=1", "HYPERVISOR_SOCKET_PATH="+socketPath)
+	stdin, err := owner.StdinPipe()
+	require.NoError(t, err)
+	stdout, err := owner.StdoutPipe()
+	require.NoError(t, err)
+	require.NoError(t, owner.Start())
+	t.Cleanup(func() {
+		_ = stdin.Close()
+		_ = owner.Process.Kill()
+		_ = owner.Wait()
+	})
+	_, err = bufio.NewReader(stdout).ReadString('\n')
+	require.NoError(t, err)
+
+	stale := exec.Command("sleep", "30")
+	require.NoError(t, stale.Start())
+	t.Cleanup(func() {
+		_ = stale.Process.Kill()
+		_ = stale.Wait()
+	})
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	const devicePath = "/sys/bus/pci/devices/0000:82:00.4"
+	stalePID := stale.Process.Pid
+	require.NoError(t, m.ensureDirectories("live-claimant"))
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: StoredMetadata{
+		Id:            "live-claimant",
+		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath: devicePath,
+		HypervisorPID: &stalePID,
+		SocketPath:    socketPath,
+	}}))
+
+	claimed, err := m.vgpuAssignmentClaimedByLiveInstance(context.Background(), "other-instance", devicePath)
+	require.NoError(t, err)
+	assert.True(t, claimed)
+}
+
 func TestKillHypervisorSurvivesConcurrentReaper(t *testing.T) {
 	socketPath := filepath.Join(t.TempDir(), "test.sock")
 	process := exec.Command(os.Args[0], "-test.run=^TestHypervisorProcessExistsWithReboundSocketPathHelper$")
diff --git a/lib/instances/vgpu.go b/lib/instances/vgpu.go
index f9d67844..81d03222 100644
--- a/lib/instances/vgpu.go
+++ b/lib/instances/vgpu.go
@@ -6,20 +6,23 @@ import (
 	"path/filepath"
 
 	"github.com/kernel/hypeman/lib/devices"
-	"github.com/kernel/hypeman/lib/hypervisor"
 	"github.com/kernel/hypeman/lib/logger"
 )
 
 // VGPUCleanupPendingError reports a failed create whose vGPU release also
-// failed during rollback. The instance record identified by InstanceID is
-// retained so the release can be retried; deleting the instance retries it.
+// failed during rollback. When Retained is true, deleting the retained instance
+// retries the release; otherwise startup reconciliation recovers the assignment.
 type VGPUCleanupPendingError struct {
 	InstanceID string
+	Retained   bool
 	Err        error
 }
 
 func (e *VGPUCleanupPendingError) Error() string {
-	return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID)
+	if e.Retained {
+		return fmt.Sprintf("%v; vGPU release failed during rollback, instance %s retains the assignment", e.Err, e.InstanceID)
+	}
+	return fmt.Sprintf("%v; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", e.Err, e.InstanceID)
 }
 
 func (e *VGPUCleanupPendingError) Unwrap() error { return e.Err }
@@ -115,16 +118,10 @@ func (m *manager) vgpuAssignmentClaimedByLiveInstance(ctx context.Context, exclu
 		if stored.HypervisorPID == nil {
 			return true, nil
 		}
-		if HypervisorProcessExists(*stored.HypervisorPID, stored.SocketPath) {
+		pid, err := resolveLiveHypervisorPID(stored.HypervisorPID, stored.SocketPath)
+		if err != nil || pid > 0 {
 			return true, nil
 		}
-		// The stored PID can be stale after a hypeman restart; a live owner
-		// of the claimant's socket still marks the claim as live.
-		if stored.SocketPath != "" && !ProcessExists(*stored.HypervisorPID) {
-			if owner, _, err := hypervisor.ResolveProcessPID(stored.SocketPath); err == nil && ProcessExists(owner) {
-				return true, nil
-			}
-		}
 	}
 	return false, nil
 }
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index b456298b..11cc2629 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -59,30 +59,33 @@ func TestCleanupFailedCreateDeletesDataWithoutRetainedVGPU(t *testing.T) {
 	require.Error(t, err)
 }
 
-func TestCleanupFailedCreateReportsPendingWhenRetentionFails(t *testing.T) {
+func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) {
 	t.Parallel()
 
 	m := &manager{paths: paths.New(t.TempDir())}
-	// A file at the guests directory path makes ensureDirectories fail even
-	// when running as root.
-	require.NoError(t, os.WriteFile(m.paths.GuestsDir(), nil, 0o644))
+	const id = "failed-create"
+	require.NoError(t, m.ensureDirectories(id))
+	require.NoError(t, os.Mkdir(m.paths.InstanceMetadata(id), 0o755))
 
 	stored := &StoredMetadata{
-		Id:            "failed-create",
+		Id:            id,
 		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
 		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
 	}
-	assert.True(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored),
-		"a failed retention must still report the outstanding vGPU assignment")
+	assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored))
 }
 
 func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) {
 	t.Parallel()
 
 	cause := errors.New("boot failed")
-	err := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause}
-	assert.ErrorIs(t, err, cause)
-	assert.Contains(t, err.Error(), "inst-1")
+	retained := &VGPUCleanupPendingError{InstanceID: "inst-1", Retained: true, Err: cause}
+	assert.ErrorIs(t, retained, cause)
+	assert.Equal(t, "boot failed; vGPU release failed during rollback, instance inst-1 retains the assignment", retained.Error())
+
+	unpersisted := &VGPUCleanupPendingError{InstanceID: "inst-1", Err: cause}
+	assert.ErrorIs(t, unpersisted, cause)
+	assert.Equal(t, "boot failed; vGPU release failed during rollback and the retention record for instance inst-1 could not be saved; the assignment is recovered on the next startup reconcile", unpersisted.Error())
 }
 
 func newStartRollbackVGPUManager(t *testing.T, destroy func(context.Context, devices.VGPUAssignment) error) (*manager, string) {

From 40d5575ad64f6d8262d800ad51471e0d4ddd78c9 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sun, 9 Aug 2026 22:02:02 +0000
Subject: [PATCH 14/17] Report surviving vGPU retention metadata

---
 lib/instances/create.go    | 14 ++++++++++++--
 lib/instances/vgpu_test.go | 28 ++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index ccdc6f42..fc8e936c 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -614,9 +614,19 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 
 	log := logger.FromContext(ctx)
+	retentionFailed := func() bool {
+		meta, err := m.loadMetadata(id)
+		if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" {
+			return true
+		}
+		if err := m.deleteInstanceData(id); err != nil {
+			log.ErrorContext(ctx, "failed to delete stale instance data after retention failure", "instance_id", id, "error", err)
+		}
+		return false
+	}
 	if err := m.ensureDirectories(id); err != nil {
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
-		return false
+		return retentionFailed()
 	}
 	retained := StoredMetadata{
 		Id:            id,
@@ -626,7 +636,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
-		return false
+		return retentionFailed()
 	}
 	return true
 }
diff --git a/lib/instances/vgpu_test.go b/lib/instances/vgpu_test.go
index 11cc2629..3c270bb3 100644
--- a/lib/instances/vgpu_test.go
+++ b/lib/instances/vgpu_test.go
@@ -73,6 +73,34 @@ func TestCleanupFailedCreateReportsUnpersistedRetention(t *testing.T) {
 		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
 	}
 	assert.False(t, m.cleanupFailedCreate(context.Background(), stored.Id, stored))
+	_, err := m.loadMetadata(id)
+	require.Error(t, err)
+}
+
+func TestCleanupFailedCreateReportsRetainedWhenFullMetadataSurvives(t *testing.T) {
+	if os.Geteuid() == 0 {
+		t.Skip("root bypasses directory permissions")
+	}
+
+	m := &manager{paths: paths.New(t.TempDir())}
+	const id = "failed-create"
+	require.NoError(t, m.ensureDirectories(id))
+	stored := &StoredMetadata{
+		Id:            id,
+		GPUFramework:  devices.VGPUFrameworkVendorVFIO,
+		GPUDevicePath: "/sys/bus/pci/devices/0000:82:00.4",
+	}
+	require.NoError(t, m.saveMetadata(&metadata{StoredMetadata: *stored}))
+
+	instanceDir := filepath.Dir(m.paths.InstanceMetadata(id))
+	require.NoError(t, os.Chmod(instanceDir, 0o555))
+	t.Cleanup(func() { _ = os.Chmod(instanceDir, 0o755) })
+
+	assert.True(t, m.cleanupFailedCreate(context.Background(), id, stored))
+	retained, err := m.loadMetadata(id)
+	require.NoError(t, err)
+	assert.Equal(t, stored.GPUFramework, retained.GPUFramework)
+	assert.Equal(t, stored.GPUDevicePath, retained.GPUDevicePath)
 }
 
 func TestVGPUCleanupPendingErrorUnwraps(t *testing.T) {

From aa68476d75eadfecf7fa6f85f0f9101baaf2ef81 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sun, 9 Aug 2026 22:02:38 +0000
Subject: [PATCH 15/17] Return accurate vGPU cleanup guidance

---
 cmd/api/api/instances.go      |  8 ++++++--
 cmd/api/api/instances_test.go | 23 +++++++++++++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go
index a4174e11..93689f2f 100644
--- a/cmd/api/api/instances.go
+++ b/cmd/api/api/instances.go
@@ -346,12 +346,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst
 		var vgpuPending *instances.VGPUCleanupPendingError
 		switch {
 		// Checked first: it wraps the original create error, so a later
-		// errors.Is case would match the cause and hide the retained instance.
+		// errors.Is case would match the cause and hide the pending vGPU cleanup.
 		case errors.As(err, &vgpuPending):
 			log.ErrorContext(ctx, "failed to create instance", "error", err, "image", request.Body.Image)
+			message := fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID)
+			if !vgpuPending.Retained {
+				message = fmt.Sprintf("failed to create instance; vGPU release failed during rollback and the retention record for instance %s could not be saved; the assignment is recovered on the next startup reconcile", vgpuPending.InstanceID)
+			}
 			return oapi.CreateInstance500JSONResponse{
 				Code:    "vgpu_cleanup_pending",
-				Message: fmt.Sprintf("failed to create instance; vGPU release failed during rollback and instance %s retains the assignment, delete it to retry", vgpuPending.InstanceID),
+				Message: message,
 			}, nil
 		case errors.Is(err, instances.ErrImageNotReady):
 			return oapi.CreateInstance400JSONResponse{
diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go
index 4f19d6fe..4d9e5563 100644
--- a/cmd/api/api/instances_test.go
+++ b/cmd/api/api/instances_test.go
@@ -63,6 +63,7 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T)
 	svc := newTestService(t)
 	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
 		InstanceID: "inst-1",
+		Retained:   true,
 		Err:        network.ErrNameExists,
 	}}
 
@@ -75,6 +76,28 @@ func TestCreateInstance_VGPUCleanupPendingBeatsWrappedErrorMapping(t *testing.T)
 	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
 	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
 	assert.Contains(t, pending.Message, "inst-1")
+	assert.Contains(t, pending.Message, "delete it to retry")
+}
+
+func TestCreateInstance_VGPUCleanupPendingWithoutRetentionUsesReconcileGuidance(t *testing.T) {
+	t.Parallel()
+	svc := newTestService(t)
+	svc.InstanceManager = createErrorInstanceManager{err: &instances.VGPUCleanupPendingError{
+		InstanceID: "inst-1",
+		Err:        network.ErrNameExists,
+	}}
+
+	resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{
+		Body: &oapi.CreateInstanceRequest{Image: "test-image"},
+	})
+	require.NoError(t, err)
+
+	pending, ok := resp.(oapi.CreateInstance500JSONResponse)
+	require.True(t, ok, "expected 500 vgpu_cleanup_pending, got %T", resp)
+	assert.EqualValues(t, "vgpu_cleanup_pending", pending.Code)
+	assert.Contains(t, pending.Message, "retention record for instance inst-1 could not be saved")
+	assert.Contains(t, pending.Message, "startup reconcile")
+	assert.NotContains(t, pending.Message, "delete")
 }
 
 func TestCreateInstance_AutoPullImage(t *testing.T) {

From 8807e97f4a4b42bfbb5a3a57267fe6f230e541ab Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Sun, 9 Aug 2026 22:26:00 +0000
Subject: [PATCH 16/17] Clarify vGPU retention fallback

---
 lib/instances/create.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/instances/create.go b/lib/instances/create.go
index fc8e936c..36f67b0f 100644
--- a/lib/instances/create.go
+++ b/lib/instances/create.go
@@ -614,7 +614,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 
 	log := logger.FromContext(ctx)
-	retentionFailed := func() bool {
+	retentionSurvives := func() bool {
 		meta, err := m.loadMetadata(id)
 		if err == nil && storedVGPUDevicePath(&meta.StoredMetadata) != "" {
 			return true
@@ -626,7 +626,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.ensureDirectories(id); err != nil {
 		log.ErrorContext(ctx, "failed to retain instance data after vGPU cleanup failure", "instance_id", id, "error", err)
-		return retentionFailed()
+		return retentionSurvives()
 	}
 	retained := StoredMetadata{
 		Id:            id,
@@ -636,7 +636,7 @@ func (m *manager) cleanupFailedCreate(ctx context.Context, id string, retainedVG
 	}
 	if err := m.saveMetadata(&metadata{StoredMetadata: retained}); err != nil {
 		log.ErrorContext(ctx, "failed to retain vGPU assignment metadata after cleanup failure", "instance_id", id, "error", err)
-		return retentionFailed()
+		return retentionSurvives()
 	}
 	return true
 }

From d6eb4926892b00cbde7f4c7d097dd38b49485e88 Mon Sep 17 00:00:00 2001
From: yummybomb <19238148+yummybomb@users.noreply.github.com>
Date: Mon, 10 Aug 2026 04:53:56 +0000
Subject: [PATCH 17/17] Report vendor VFIO profile availability per free VF

---
 lib/devices/vendor_vfio_linux.go      | 21 +++++++--------------
 lib/devices/vendor_vfio_linux_test.go |  6 +++---
 2 files changed, 10 insertions(+), 17 deletions(-)

diff --git a/lib/devices/vendor_vfio_linux.go b/lib/devices/vendor_vfio_linux.go
index 6f59bed3..ceab85ec 100644
--- a/lib/devices/vendor_vfio_linux.go
+++ b/lib/devices/vendor_vfio_linux.go
@@ -81,13 +81,13 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) {
 	return vfs, nil
 }
 
-// listProfiles aggregates creatable profiles per parent GPU. Free VFs on the
-// same GPU share its framebuffer, so counting each advertising VF overreports
-// availability. The driver only guarantees that a GPU still advertising a
-// type can fit one more instance of it, so report that per-GPU lower bound.
+// listProfiles counts each free VF advertising a type as one creatable
+// instance, matching the driver-reported units that mdev sums through
+// available_instances. This is a best-effort snapshot because creating on one
+// VF may revoke the type from siblings that share its GPU framebuffer.
 func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) {
 	profilesByType := make(map[string]profileMetadata)
-	creatableGPUs := make(map[string]map[string]struct{})
+	creatableVFs := make(map[string]int)
 	for _, vf := range vfs {
 		creatable, err := s.readCreatableProfiles(vf.PCIAddress)
 		if err != nil {
@@ -96,14 +96,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro
 		for _, profile := range creatable {
 			profilesByType[profile.TypeName] = profile
 			if !vf.Allocated {
-				gpu := vf.ParentGPU
-				if gpu == "" {
-					gpu = vf.PCIAddress
-				}
-				if creatableGPUs[profile.TypeName] == nil {
-					creatableGPUs[profile.TypeName] = make(map[string]struct{})
-				}
-				creatableGPUs[profile.TypeName][gpu] = struct{}{}
+				creatableVFs[profile.TypeName]++
 			}
 		}
 	}
@@ -119,7 +112,7 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro
 		profiles = append(profiles, GPUProfile{
 			Name:          profile.Name,
 			FramebufferMB: profile.FramebufferMB,
-			Available:     len(creatableGPUs[profile.TypeName]),
+			Available:     creatableVFs[profile.TypeName],
 		})
 	}
 	return profiles, nil
diff --git a/lib/devices/vendor_vfio_linux_test.go b/lib/devices/vendor_vfio_linux_test.go
index 51668ed1..4555f5c2 100644
--- a/lib/devices/vendor_vfio_linux_test.go
+++ b/lib/devices/vendor_vfio_linux_test.go
@@ -184,7 +184,7 @@ func TestVendorVFIODestroyRetainsAssignmentWhenOneVFIOPathIsMissing(t *testing.T
 	}
 }
 
-func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) {
+func TestVendorVFIOListProfilesCountsFreeVFs(t *testing.T) {
 	t.Parallel()
 
 	sysfs := newTestVendorVFIOSysfs(t)
@@ -196,8 +196,8 @@ func TestVendorVFIOListProfilesCountsPerGPUNotPerVF(t *testing.T) {
 	require.NoError(t, err)
 	profiles, err := sysfs.listProfiles(vfs)
 	require.NoError(t, err)
-	assert.Equal(t, 2, profileAvailability(profiles, "NVIDIA L40S-48Q"),
-		"free VFs share their parent GPU's capacity, so availability is per GPU")
+	assert.Equal(t, 3, profileAvailability(profiles, "NVIDIA L40S-48Q"),
+		"each free VF advertising the type counts as one creatable instance")
 }
 
 func TestVendorVFIOCreateReportsCapacityWhenAllGPUsFull(t *testing.T) {