Detect and quarantine wedged vGPU VFs

cmd/api/main.go

@@ -717,6 +717,12 @@ func run() error {
 			return app.HealthCheckController.Run(gctx)
 		})
 	}
+	if app.VGPUSentinelController != nil {
+		grp.Go(func() error {
+			logger.Info("starting vGPU sentinel controller")
+			return app.VGPUSentinelController.Run(gctx)
+		})
+	}
 	if restartController, ok := app.InstanceManager.(interface {
 		StartRestartPolicyController(context.Context) error
 	}); ok {

cmd/api/wire.go

@@ -29,26 +29,27 @@ import (
 
 // application struct to hold initialized components
 type application struct {
-	Ctx                   context.Context
-	Logger                *slog.Logger
-	Config                *config.Config
-	ImageManager          images.Manager
-	SystemManager         system.Manager
-	NetworkManager        network.Manager
-	DeviceManager         devices.Manager
-	InstanceManager       instances.Manager
-	VolumeManager         volumes.Manager
-	BuilderManager        builders.Manager
-	IngressManager        ingress.Manager
-	BuildManager          builds.Manager
-	PushManager           imagepush.Manager
-	ResourceManager       *resources.Manager
-	GuestMemoryController guestmemory.Controller
-	AutoStandbyController *autostandby.Controller
-	HealthCheckController *instances.HealthCheckController
-	VMMetricsManager      *vm_metrics.Manager
-	Registry              *registry.Registry
-	ApiService            *api.ApiService
+	Ctx                    context.Context
+	Logger                 *slog.Logger
+	Config                 *config.Config
+	ImageManager           images.Manager
+	SystemManager          system.Manager
+	NetworkManager         network.Manager
+	DeviceManager          devices.Manager
+	InstanceManager        instances.Manager
+	VolumeManager          volumes.Manager
+	BuilderManager         builders.Manager
+	IngressManager         ingress.Manager
+	BuildManager           builds.Manager
+	PushManager            imagepush.Manager
+	ResourceManager        *resources.Manager
+	GuestMemoryController  guestmemory.Controller
+	AutoStandbyController  *autostandby.Controller
+	HealthCheckController  *instances.HealthCheckController
+	VGPUSentinelController *instances.VGPUSentinelController
+	VMMetricsManager       *vm_metrics.Manager
+	Registry               *registry.Registry
+	ApiService             *api.ApiService
 }
 
 // initializeApp is the injector function
@@ -72,6 +73,7 @@ func initializeApp() (*application, func(), error) {
 		providers.ProvideGuestMemoryController,
 		providers.ProvideAutoStandbyController,
 		providers.ProvideHealthCheckController,
+		providers.ProvideVGPUSentinelController,
 		providers.ProvideVMMetricsManager,
 		providers.ProvideRegistry,
 		api.New,

lib/devices/GPU.md

@@ -282,13 +282,37 @@ NVRM: GPU 0000:00:03.0: RmInitAdapter failed! (0x22:0x65:884)
 ```
 
 (0x65 = timeout; the guest's init requests are never answered, and
-`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle). Because
-placement is deterministic least-loaded, an idle host re-picks the same VF for
-every request, so one wedged VF presents as all vGPU instances failing while
-`/resources` reports full capacity.
-
-The wedge itself leaves no host-side log: no kernel error, no XID, no plugin
-crash. The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is
+`/proc/interrupts` shows the GPU's MSI-X vectors allocated but idle).
+
+Hypeman detects this automatically: the guest agent watches the guest kernel
+log (`/dev/kmsg`) for that line and reports it as a `HYPEMAN-GPU-INIT-FAILED`
+marker in the instance's `logs/app.log`, which the vGPU sentinel controller
+scans for every vendor VFIO instance. A match quarantines the VF in
+`<data-dir>/gpu/vf-health.json` (it survives restarts): the VF is excluded
+from placement and from advertised profile availability, and its parent GPU
+becomes overflow-only — deprioritized for new placements. The conviction is
+logged at error level (`quarantined wedged vGPU VF`) and counted in
+`hypeman_instances_vgpu_sentinel_convictions_total`;
+`hypeman_instances_vgpu_quarantined_vfs` gauges the current quarantine count.
+There is no rate limit on convictions: a systemic non-wedge init failure
+(e.g. a guest/host driver mismatch) emits the same line on every VF and
+would quarantine the whole host, so such changes must be validated on a
+test host first, and the convictions counter is the signal to alert on if
+one gets through.
+
+Detection requires the hypeman guest agent: an image that skips the agent
+never reports, so a wedge hit exclusively by such images stays undetected in
+v1. The scanner requires the complete standalone guest-agent log envelope, so
+customer commands and ordinary output containing the marker do not match. The
+serial console remains guest-writable: a root guest can deliberately imitate
+the entire line and quarantine the VF its own instance holds; the quarantine
+only ever removes capacity, never touches the instance.
+
+The wedge-creating kill itself leaves no host-side log: no kernel error, no
+XID, no plugin crash. Detection therefore happens on the next boot that lands
+on the VF, whose guest driver starts failing ~27s after spawn.
+
+The trigger is a SIGKILL delivered to QEMU while the vGPU plugin is
 still initializing the VF (roughly the first seconds after process start):
 a single hard kill in that window wedges the VF near-deterministically,
 while QEMU processes that exit voluntarily — error exits, QMP quit, SIGTERM —
@@ -303,18 +327,47 @@ External SIGKILLs (OOM killer, manual `kill -9`) can still trigger it.
 Confirm by assigning the same profile on a different VF: if that guest
 initializes, the VF is wedged, not the driver stack. Remediate by cycling
 SR-IOV on the parent GPU (this destroys and recreates all of its VFs, so it
-requires no vGPU assignments on that GPU):
+requires no vGPU assignments on that GPU). The DCGM quiesce is not optional:
+with `nv-hostengine`/`dcgm-exporter` holding the GPUs open, `sriov-manage -d`
+fails with `Cannot obtain unbindLock` on first contact.
+
+Any manual edit to `vf-health.json` needs an immediate hypeman restart: the
+store loads only at startup, and a conviction landing first re-persists the
+in-memory set over your edit. The restart does not disturb running VMs —
+startup reconciliation protects live VFs.
+
+**Draining the parent GPU.** Overflow-only is a preference, not a cordon:
+under capacity pressure new placements still land on the card's healthy VFs
+and refill it. To drain the card, quarantine all of its VFs by hand — add
+records to `vf-health.json` (the shape of a real conviction: `vf_address`
+plus `quarantined_at`) and restart. Running instances are untouched and
+drain through their normal lifecycle: standby is blocked for vGPU instances,
+so only a running VM pins a VF, and each stop or delete frees one for good.
+Monitor by listing instances whose `gpu.device_path` sits under the parent
+GPU; once none remain, run the cycle below.
 
 ```bash
+# 1. Quiesce the services holding the GPU (required for the unbind lock).
+systemctl stop nvidia-dcgm-exporter nvidia-dcgm
+
+# 2. Cycle SR-IOV on the parent GPU.
 /usr/lib/nvidia/sriov-manage -d <parent-gpu-pci-addr>
 /usr/lib/nvidia/sriov-manage -e <parent-gpu-pci-addr>
+
+# 3. Restart the quiesced services.
+systemctl start nvidia-dcgm nvidia-dcgm-exporter
 ```
 
+After the cycle, clear the quarantine by removing the card's entries from
+`vf-health.json` (restart rule above), then boot a GPU instance as
+verification: placement excludes quarantined VFs, so the recovered VF cannot
+be targeted while its entry exists, and there is no VF-pin API — clearing
+first is safe because the sentinel automatically re-quarantines the VF if the
+cycle did not cure it (every cycle in hardware validation did).
+
 Do not unbind/rebind the VF from the nvidia driver — it breaks the
 nvidia-vgpu-vfio core-device registration (`vfio_pci_core_device not found`)
 and the VF stops accepting assignments entirely until the SR-IOV cycle.
-Services holding the GPU (DCGM, persistenced) must be stopped for the cycle
-to obtain the unbind lock.
 
 ### vGPU assignment fails
 

lib/devices/manager.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"log/slog"
 	"os"
 	"runtime"
 	"strings"
@@ -85,6 +86,9 @@ type manager struct {
 // NewManager creates a new device manager.
 // Use SetLivenessChecker after construction to enable accurate orphan detection.
 func NewManager(p *paths.Paths) Manager {
+	if err := initVFHealthStore(p.VFHealthState()); err != nil {
+		slog.Default().Error("failed to load VF health state; vGPU placement is disabled until the state file is repaired or removed", "error", err)
+	}
 	return &manager{
 		paths:      p,
 		vfioBinder: NewVFIOBinder(),

lib/devices/vendor_vfio_linux.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"log/slog"
+	"math/rand/v2"
 	"os"
 	"path/filepath"
 	"sort"
@@ -29,6 +30,15 @@ type vendorVFIOSysfs struct {
 	vfioDevicesPath   string
 	owners            map[string]string
 	framebufferByType map[string]int
+	pickVFIndex       func(n int) int // overridden in tests; nil means random
+}
+
+// withVGPUPlacementLock runs f under the lock that serializes vendor VFIO
+// vGPU placement, so quarantine updates and VF selection cannot interleave.
+func withVGPUPlacementLock(f func()) {
+	vendorVFIOMu.Lock()
+	defer vendorVFIOMu.Unlock()
+	f()
 }
 
 var (
@@ -102,6 +112,10 @@ func (s vendorVFIOSysfs) discoverVFs() ([]VirtualFunction, error) {
 // available_instances. This is a best-effort snapshot because creating on one
 // VF may revoke the type from siblings that share its GPU framebuffer.
 func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, error) {
+	quarantined, err := vfHealth.checkedAddresses()
+	if err != nil {
+		return nil, err
+	}
 	profilesByType := make(map[string]profileMetadata)
 	creatableVFs := make(map[string]int)
 	for _, vf := range vfs {
@@ -113,9 +127,10 @@ func (s vendorVFIOSysfs) listProfiles(vfs []VirtualFunction) ([]GPUProfile, erro
 			slog.Default().Warn("skipping unreadable creatable vGPU types", "vf", vf.PCIAddress, "error", err)
 			continue
 		}
+		_, bad := quarantined[vf.PCIAddress]
 		for _, profile := range creatable {
 			profilesByType[profile.TypeName] = profile
-			if !vf.Allocated {
+			if !vf.Allocated && !bad {
 				creatableVFs[profile.TypeName]++
 			}
 		}
@@ -304,10 +319,25 @@ func (s vendorVFIOSysfs) reconcile(ctx context.Context, protectedDevicePaths map
 }
 
 func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType string) (string, error) {
+	quarantined, err := vfHealth.checkedAddresses()
+	if err != nil {
+		return "", err
+	}
 	usageByGPU := make(map[string]int)
 	unknownUsageByGPU := make(map[string]bool)
+	quarantinedByGPU := make(map[string]int)
 	freeByGPU := make(map[string][]VirtualFunction)
 	for _, vf := range vfs {
+		// A quarantined VF is never a placement candidate, but its parent GPU
+		// stays usable: the count only deprioritizes the card so it drains
+		// toward the SR-IOV cycle instead of staying warm.
+		_, bad := quarantined[vf.PCIAddress]
+		if bad {
+			quarantinedByGPU[vf.ParentGPU]++
+			if !vf.Allocated {
+				continue
+			}
+		}
 		if vf.Allocated {
 			// framebufferByType only covers currently creatable profiles, so
 			// after a restart an allocated type can be missing when its
@@ -341,6 +371,9 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType
 		gpus = append(gpus, gpu)
 	}
 	sort.Slice(gpus, func(i, j int) bool {
+		if quarantinedByGPU[gpus[i]] != quarantinedByGPU[gpus[j]] {
+			return quarantinedByGPU[gpus[i]] < quarantinedByGPU[gpus[j]]
+		}
 		if unknownUsageByGPU[gpus[i]] != unknownUsageByGPU[gpus[j]] {
 			return !unknownUsageByGPU[gpus[i]]
 		}
@@ -352,7 +385,16 @@ func (s vendorVFIOSysfs) selectLeastLoadedVF(vfs []VirtualFunction, profileType
 	if len(gpus) == 0 {
 		return "", nil
 	}
-	return freeByGPU[gpus[0]][0].PCIAddress, nil
+	// Randomize among the chosen GPU's free VFs. A deterministic
+	// lowest-address pick would route every first create on an idle host to
+	// the same VF, so a single undetected wedged VF presents as every GPU
+	// create failing.
+	candidates := freeByGPU[gpus[0]]
+	pick := s.pickVFIndex
+	if pick == nil {
+		pick = rand.IntN
+	}
+	return candidates[pick(len(candidates))].PCIAddress, nil
 }
 
 func (s vendorVFIOSysfs) profileMetadata(vfs []VirtualFunction) ([]profileMetadata, error) {