From fdb0812fc3a7d67f54b7508c69989cf1a8e73764 Mon Sep 17 00:00:00 2001
From: alexcos20 <alex.coseru@gmail.com>
Date: Sat, 5 Sep 2026 00:33:11 +0300
Subject: [PATCH 1/3] gpu metrics

---
 .../dashboards/ocean-node-compute.json        |  13 +-
 .../grafana/dashboards/ocean-node-p2p.json    |   2 +
 deploy/telemetry/scripts/import-dashboard.sh  |  84 +++++++++++-
 docs/compute.md                               |   8 ++
 docs/env.md                                   |   2 +-
 src/components/c2d/compute_engine_docker.ts   |  43 +++++++
 src/components/c2d/gpu/index.ts               |  59 ++++++++-
 src/components/c2d/gpu/nvml.ts                | 121 +++++++++++++++---
 src/components/c2d/gpu/types.ts               |  10 ++
 src/telemetry/computeGauges.ts                |  74 ++++++++---
 10 files changed, 363 insertions(+), 53 deletions(-)

diff --git a/deploy/telemetry/grafana/dashboards/ocean-node-compute.json b/deploy/telemetry/grafana/dashboards/ocean-node-compute.json
index 64ef3d84b..28a767df1 100644
--- a/deploy/telemetry/grafana/dashboards/ocean-node-compute.json
+++ b/deploy/telemetry/grafana/dashboards/ocean-node-compute.json
@@ -781,7 +781,7 @@
           },
           "editorMode": "code",
           "expr": "ocean_compute_gpu_utilization_percent{service_instance_id=~\"$instance\"}",
-          "legendFormat": "{{gpu}} {{vendor}}",
+          "legendFormat": "{{gpu}} {{vendor}} ({{in_use}})",
           "range": true,
           "instant": false,
           "refId": "A"
@@ -827,7 +827,7 @@
       "id": 15,
       "type": "timeseries",
       "title": "GPU memory used / total",
-      "description": "Per-GPU ocean_compute_gpu_memory_used_bytes vs ocean_compute_gpu_memory_total_bytes; ratio via ocean_node:compute_gpu_mem_used_ratio.",
+      "description": "Per-GPU ocean_compute_gpu_memory_used_bytes vs ocean_compute_gpu_memory_total_bytes; the adjacent stat shows the used/total ratio. Emitted for every host GPU (idle included); in_use shows which are held by a job.",
       "datasource": {
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
@@ -904,7 +904,7 @@
       "id": 16,
       "type": "stat",
       "title": "GPU memory used ratio",
-      "description": "Recording rule ocean_node:compute_gpu_mem_used_ratio.",
+      "description": "GPU memory used / total across the selected node's GPUs, computed inline so it does not depend on the ocean_node:compute_gpu_mem_used_ratio recording rule being loaded on this Prometheus.",
       "datasource": {
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
@@ -922,7 +922,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "ocean_node:compute_gpu_mem_used_ratio",
+          "expr": "sum(ocean_compute_gpu_memory_used_bytes{service_instance_id=~\"$instance\"}) / sum(ocean_compute_gpu_memory_total_bytes{service_instance_id=~\"$instance\"})",
           "legendFormat": "used ratio",
           "range": false,
           "instant": true,
@@ -994,7 +994,7 @@
           },
           "editorMode": "code",
           "expr": "ocean_compute_gpu_temperature_celsius{service_instance_id=~\"$instance\"}",
-          "legendFormat": "{{gpu}} {{vendor}}",
+          "legendFormat": "{{gpu}} {{vendor}} ({{in_use}})",
           "range": true,
           "instant": false,
           "refId": "A"
@@ -1059,7 +1059,7 @@
           },
           "editorMode": "code",
           "expr": "ocean_compute_gpu_power_watts{service_instance_id=~\"$instance\"}",
-          "legendFormat": "{{gpu}} {{vendor}}",
+          "legendFormat": "{{gpu}} {{vendor}} ({{in_use}})",
           "range": true,
           "instant": false,
           "refId": "A"
@@ -1407,6 +1407,7 @@
         "definition": "label_values(ocean_compute_cpu_host_cores, service_instance_id)",
         "hide": 0,
         "includeAll": true,
+        "allValue": ".*",
         "label": "Instance",
         "multi": true,
         "name": "instance",
diff --git a/deploy/telemetry/grafana/dashboards/ocean-node-p2p.json b/deploy/telemetry/grafana/dashboards/ocean-node-p2p.json
index 82218edf9..8cbc57f44 100644
--- a/deploy/telemetry/grafana/dashboards/ocean-node-p2p.json
+++ b/deploy/telemetry/grafana/dashboards/ocean-node-p2p.json
@@ -1254,6 +1254,7 @@
         "definition": "label_values(ocean_p2p_ready, service_instance_id)",
         "hide": 0,
         "includeAll": true,
+        "allValue": ".*",
         "label": "Instance",
         "multi": true,
         "name": "instance",
@@ -1277,6 +1278,7 @@
         "definition": "label_values(ocean_p2p_ready, ocean_node_role)",
         "hide": 0,
         "includeAll": true,
+        "allValue": ".*",
         "label": "Role",
         "multi": true,
         "name": "role",
diff --git a/deploy/telemetry/scripts/import-dashboard.sh b/deploy/telemetry/scripts/import-dashboard.sh
index 11b9e9a62..df0d7a779 100755
--- a/deploy/telemetry/scripts/import-dashboard.sh
+++ b/deploy/telemetry/scripts/import-dashboard.sh
@@ -7,11 +7,22 @@
 #
 #   GRAFANA_URL=https://your-org.grafana.net \
 #   GRAFANA_TOKEN=glsa_xxx \
+#   PROM_NAME=Prometheus \
 #   ./deploy/telemetry/scripts/import-dashboard.sh [p2p|compute]
 #
 # Minting a token: Grafana -> Administration -> Users and access -> Service accounts ->
 # Add service account -> role "Editor" -> Add service account token. Copy it into GRAFANA_TOKEN.
 #
+# Datasource resolution (the dashboards declare a DS_PROMETHEUS variable that must be pinned to a
+# real Prometheus datasource uid). This does NOT blindly take the first Prometheus-type datasource
+# — a Grafana with several (mon.oceanprotocol.io has 7) would then get the wrong one and the
+# dashboard's variables resolve empty, so every panel shows "No data". Precedence:
+#     1. PROM_UID     — explicit uid, used verbatim.
+#     2. PROM_NAME    — exact datasource name (case-insensitive); must match exactly one.
+#     3. the default  — the single Prometheus datasource marked isDefault.
+#     4. the only one — if the Grafana has exactly one Prometheus datasource.
+#     5. otherwise    — fail loudly, listing candidates. No silent guessing.
+#
 # Optional:
 #   GRAFANA_FOLDER_UID   target folder (default: the "General" folder)
 #   DASHBOARD_FILE       path to the dashboard JSON (overrides the p2p|compute shorthand)
@@ -28,7 +39,9 @@ case "$WHICH" in
 esac
 
 # Exported, not just assigned: the final `node -e` block reads it from `process.env`.
+# Strip any trailing slash so composed URLs don't end up with a `//`.
 export GRAFANA_URL="${GRAFANA_URL:-http://localhost:3001}"
+GRAFANA_URL="${GRAFANA_URL%/}"
 DASHBOARD_FILE="${DASHBOARD_FILE:-$DEFAULT_FILE}"
 
 if [ ! -f "$DASHBOARD_FILE" ]; then
@@ -43,16 +56,73 @@ if [ -z "${GRAFANA_TOKEN:-}" ]; then
   exit 1
 fi
 
-# The dashboards declare a DS_PROMETHEUS datasource variable. Resolve it to the datasource uid on
-# the target Grafana so the imported copy is immediately usable.
+# Resolve the Prometheus datasource uid for the DS_PROMETHEUS variable (see the header).
 PROM_UID="${PROM_UID:-}"
-[ -z "$PROM_UID" ] && PROM_UID=$(
-  curl -fsS -H "Authorization: Bearer $GRAFANA_TOKEN" "$GRAFANA_URL/api/datasources" \
-    | node -e "let s='';process.stdin.on('data',d=>s+=d).on('end',()=>{try{const j=JSON.parse(s);const m=j.find(d=>d.type==='prometheus');console.log(m?m.uid:'')}catch{console.log('')}})"
-)
+if [ -z "$PROM_UID" ]; then
+  # The resolver is written to a temp file (rather than `node -e '...'`) so its template
+  # literals and single quotes are not mangled by the shell.
+  RESOLVER="$(mktemp -t import-dashboard-resolver.XXXXXX.js)"
+  trap 'rm -f "$RESOLVER"' EXIT
+  cat >"$RESOLVER" <<'RESOLVER_JS'
+let s = ''
+process.stdin.on('data', (d) => (s += d)).on('end', () => {
+  let list
+  try {
+    list = JSON.parse(s)
+  } catch {
+    console.error('Could not parse the /api/datasources response from Grafana.')
+    process.exit(1)
+  }
+  const proms = (Array.isArray(list) ? list : []).filter((d) => d.type === 'prometheus')
+  const fmt = (ds) =>
+    ds
+      .map((d) => `    - ${d.name}  (uid=${d.uid})${d.isDefault ? '  [default]' : ''}`)
+      .join('\n')
+
+  if (proms.length === 0) {
+    console.error('No Prometheus-type datasource found on this Grafana. Add one, or set PROM_UID.')
+    process.exit(1)
+  }
+
+  const name = (process.env.PROM_NAME || '').trim()
+  if (name) {
+    const byName = proms.filter((d) => String(d.name).toLowerCase() === name.toLowerCase())
+    if (byName.length === 1) return void console.log(byName[0].uid)
+    if (byName.length === 0) {
+      console.error(`PROM_NAME="${name}" matched no Prometheus datasource. Candidates:\n${fmt(proms)}`)
+      process.exit(1)
+    }
+    console.error(`PROM_NAME="${name}" is not unique (${byName.length} matches). Set PROM_UID:\n${fmt(byName)}`)
+    process.exit(1)
+  }
+
+  const defaults = proms.filter((d) => d.isDefault === true)
+  if (defaults.length === 1) return void console.log(defaults[0].uid)
+  if (defaults.length > 1) {
+    console.error(`Multiple Prometheus datasources are marked default. Set PROM_NAME or PROM_UID:\n${fmt(defaults)}`)
+    process.exit(1)
+  }
+
+  if (proms.length === 1) return void console.log(proms[0].uid)
+
+  console.error(
+    `This Grafana has ${proms.length} Prometheus datasources and none is marked default.\n` +
+      `Refusing to guess (that is the bug this avoids). Set PROM_NAME=<name> or PROM_UID=<uid>:\n` +
+      fmt(proms)
+  )
+  process.exit(1)
+})
+RESOLVER_JS
+
+  DATASOURCES="$(curl -fsS -H "Authorization: Bearer $GRAFANA_TOKEN" "$GRAFANA_URL/api/datasources")"
+  PROM_UID="$(printf '%s' "$DATASOURCES" | PROM_NAME="${PROM_NAME:-}" node "$RESOLVER")" || {
+    echo "Could not resolve a Prometheus datasource (see the message above)." >&2
+    exit 1
+  }
+fi
 
 if [ -z "$PROM_UID" ]; then
-  echo "No Prometheus datasource found on $GRAFANA_URL — add one first, or set PROM_UID." >&2
+  echo "No Prometheus datasource uid resolved on $GRAFANA_URL — set PROM_NAME or PROM_UID." >&2
   exit 1
 fi
 
diff --git a/docs/compute.md b/docs/compute.md
index 240d035c2..4147f082f 100644
--- a/docs/compute.md
+++ b/docs/compute.md
@@ -227,6 +227,14 @@ The environment references it by `id`.
 > how to fix it. AMD and Intel GPU metrics are not yet collected. GPU metrics are returned only
 > to the owner of the job/service, alongside its container metrics. Set `GPU_METRICS=off` to
 > disable.
+>
+> Separately, when telemetry is enabled the node also **exports** GPU health as OTel gauges for
+> operator dashboards: utilization, memory, temperature and power are sampled **host-wide** for
+> every GPU visible to the node process — idle ones included, so a card shows up even when no job
+> holds it — each carrying an `in_use` label (`true`/`false`) for whether a job currently holds
+> it, while `ocean_compute_gpu_devices_in_use` counts the allocated devices. This fleet-facing
+> export is independent of the per-owner job metrics above and honours the same `GPU_METRICS`
+> switch and NVML requirements.
 
 ### Multi-GPU workloads (shared memory)
 
diff --git a/docs/env.md b/docs/env.md
index 02a549006..3e6fe9750 100644
--- a/docs/env.md
+++ b/docs/env.md
@@ -198,7 +198,7 @@ Powers the `getNodeMetrics` (live snapshot) and `getNodeMetricsHistory` (hourly
 
 - `C2D_METRICS_INTERVAL_SECONDS`: How often (in seconds) the node samples live Docker runtime metrics (CPU, RAM, disk, network, block I/O, PIDs, exit info — plus NVIDIA GPU utilization/memory) for running compute jobs and services, persisting a snapshot onto the job record in the C2D database. These metrics are **owner-only**: they are never included in the escrow claim proof and never returned to anyone but the authenticated owner of the job/service. To that owner they come back **by default** on `COMPUTE_GET_STATUS` / `SERVICE_GET_STATUS` (no flag needed — see [API.md](API.md) for the `includeMetrics` override); an unauthenticated status call and the node-wide `serviceList` never return them. Set to `0` to disable collection entirely. Metrics are best-effort (up to one interval of staleness). Defaults to `10`. Example: `10`
 
-- `GPU_METRICS`: Controls the GPU metrics collector. `auto` (default) detects and enables the NVIDIA (NVML) backend when a GPU host is available; `off` disables GPU collection. Requires the optional `koffi` dependency and `libnvidia-ml.so.1` reachable **by the node process** — note that a containerized node does not get the NVIDIA driver libraries just because the host has them, so this is the usual reason GPU metrics are missing (`could not bind libnvidia-ml.so.1`); [compute.md → Troubleshooting GPU metrics](compute.md#troubleshooting-gpu-metrics) lists every warning and its fix. If either is missing, GPU metrics are skipped (no `gpu` field) while container-level metrics continue. AMD and Intel backends are not yet implemented. Cadence reuses `C2D_METRICS_INTERVAL_SECONDS`. Defaults to `auto`. Example: `auto`
+- `GPU_METRICS`: Controls the GPU metrics collector. `auto` (default) detects and enables the NVIDIA (NVML) backend when a GPU host is available; `off` disables GPU collection. Requires the optional `koffi` dependency and `libnvidia-ml.so.1` reachable **by the node process** — note that a containerized node does not get the NVIDIA driver libraries just because the host has them, so this is the usual reason GPU metrics are missing (`could not bind libnvidia-ml.so.1`); [compute.md → Troubleshooting GPU metrics](compute.md#troubleshooting-gpu-metrics) lists every warning and its fix. If either is missing, GPU metrics are skipped (no `gpu` field) while container-level metrics continue. GPU utilization/memory/temperature/power are sampled **host-wide** — every GPU visible to the node process is reported, idle ones included, so a card's health shows even when no job holds it — with an `in_use` label (`true`/`false`) marking devices currently allocated to a job; `ocean_compute_gpu_devices_in_use` counts the allocated ones. AMD and Intel backends are not yet implemented. Cadence reuses `C2D_METRICS_INTERVAL_SECONDS`. Defaults to `auto`. Example: `auto`
 
 - `SERVICE_TEMPLATES_PATH`: Path to a folder of operator-published Service-on-Demand template files (`*.json`, validated against the template schema). The folder is re-read on every `serviceTemplates` request, so templates can be added, edited, or removed without restarting the node. Maps to the `serviceTemplatesPath` config field. Defaults to `databases/serviceTemplates/`, which the image does not create — the operator mounts templates into it (a missing folder simply means no templates). See the [Services guide](services.md). Example: `/templates`
 
diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts
index ae77ef676..a4dbb2cb4 100755
--- a/src/components/c2d/compute_engine_docker.ts
+++ b/src/components/c2d/compute_engine_docker.ts
@@ -154,6 +154,12 @@ export class C2DEngineDocker extends C2DEngine {
   // Best-effort GPU metrics collector (NVIDIA/NVML today). Lazily initializes its vendor
   // backends on first use by a GPU job; a pure-CPU node never loads any GPU code.
   private gpuMetrics: GpuMetricsService = new GpuMetricsService()
+  // Host-wide GPU health snapshot: every GPU visible to this process, idle ones included.
+  // Refreshed by refreshHostGpuSnapshot() on the metrics cadence and read (never written) by the
+  // OTel compute gauge callback, same contract as lastAggregate/envResourceSnapshot. Stays
+  // undefined on nodes that declare no GPU resources, so a pure-CPU node never touches NVML.
+  public hostGpuSnapshot?: ComputeGpuAggregate[]
+  private lastHostGpuSampleAt: number = 0
   // Last time the engine-wide metrics roll-up was logged. The loop ticks every 2s but snapshots
   // only refresh once per C2D_METRICS_INTERVAL_SECONDS, so the summary is throttled to match.
   private lastMetricsSummaryAt: number = 0
@@ -1930,6 +1936,10 @@ export class C2DEngineDocker extends C2DEngine {
       // sees the whole live picture without correlating per-container samples by hand.
       this.logMetricsSummary(jobs, runningServices)
 
+      // Host-wide GPU health (every visible device, idle included) — throttled + best-effort so a
+      // hung or absent NVML can never stall the loop.
+      await this.refreshHostGpuSnapshot()
+
       // Service-on-Demand starts: advance pending service jobs through the start pipeline.
       // Fire-and-forget (NOT awaited): an image pull can take minutes and must not block the
       // loop (compute jobs + expiry must keep advancing). The in-progress guard prevents a
@@ -3117,6 +3127,39 @@ export class C2DEngineDocker extends C2DEngine {
   // throttled to C2D_METRICS_INTERVAL_SECONDS — the loop itself ticks every 2s, but snapshots
   // only refresh once per interval, so logging every tick would just repeat numbers.
   // Never throws: a logging failure must not touch the loop.
+  // Sample the host's GPUs (all of them, not just those held by a running job) so the compute
+  // dashboard shows GPU health even while idle. Throttled to the metrics cadence and strictly
+  // best-effort: any failure leaves the previous snapshot in place and never disturbs the loop.
+  // No-ops entirely on a node that declares no GPU resources, so pure-CPU nodes never touch NVML.
+  private async refreshHostGpuSnapshot(): Promise<void> {
+    try {
+      if (!isMetricsCollectionEnabled()) return
+      const now = Date.now()
+      if (now - (this.lastHostGpuSampleAt ?? 0) < getMetricsIntervalSeconds() * 1000) {
+        return
+      }
+      const connection = await this.getC2DConfig().connection
+      const gpuResources = (connection?.resources ?? []).filter(
+        (r: ComputeResource) => String(r.type).toLowerCase() === 'gpu'
+      )
+      if (gpuResources.length === 0) return // pure-CPU node: never load NVML
+      // Stamp the throttle only once we know there are GPUs worth sampling.
+      this.lastHostGpuSampleAt = now
+      const snapshot = await this.gpuMetrics.sampleHost(gpuResources)
+      this.hostGpuSnapshot = (snapshot ?? []).map((g) => ({
+        resourceId: g.resourceId,
+        vendor: g.vendor,
+        utilizationPercent: g.utilizationPercent ?? undefined,
+        memoryUsedBytes: g.memoryUsedBytes ?? undefined,
+        memoryTotalBytes: g.memoryTotalBytes ?? undefined,
+        temperatureC: g.temperatureC,
+        powerWatts: g.powerWatts
+      }))
+    } catch (e: any) {
+      CORE_LOGGER.debug(`[metrics] host gpu sample failed: ${e?.message}`)
+    }
+  }
+
   private logMetricsSummary(
     jobs: DBComputeJob[] = [],
     services: ServiceJob[] = []
diff --git a/src/components/c2d/gpu/index.ts b/src/components/c2d/gpu/index.ts
index f41f1d6b9..fbd52b4d7 100644
--- a/src/components/c2d/gpu/index.ts
+++ b/src/components/c2d/gpu/index.ts
@@ -5,7 +5,12 @@ import type {
 } from '../../../@types/C2D/C2D.js'
 import { CORE_LOGGER } from '../../../utils/logging/common.js'
 import { ENVIRONMENT_VARIABLES } from '../../../utils/constants.js'
-import { GpuDeviceHandle, GpuVendor, GpuVendorCollector } from './types.js'
+import {
+  GpuDeviceHandle,
+  GpuDeviceMetrics,
+  GpuVendor,
+  GpuVendorCollector
+} from './types.js'
 import { NvmlGpuCollector } from './nvml.js'
 
 export { parseMemoryTotalToBytes } from './types.js'
@@ -32,6 +37,36 @@ function inferVendor(res: ComputeResource): GpuVendor | null {
   return null
 }
 
+// Maps host-enumerated device samples (from a collector's sampleAll) to snapshots, aligning each
+// device's stable NVML UUID to its configured resource id ('gpu0') so host-wide GPU series share
+// labels with `ocean_compute_env_resource_*` (and so `in_use` can be derived per device). A device
+// with no configured UUID match falls back to a vendor+index id. Pure and NVML-free by design so
+// the label logic — the one part that silently mis-labels if wrong — is unit-testable.
+export function mapHostGpuDevices(
+  metrics: GpuDeviceMetrics[],
+  gpuResources: ComputeResource[]
+): GpuMetricsSnapshot[] {
+  const idByUuid = new Map<string, string>()
+  for (const res of gpuResources ?? []) {
+    if (String(res.type).toLowerCase() !== 'gpu') continue
+    const ids: string[] = res?.init?.deviceRequests?.DeviceIDs ?? []
+    const uuid = ids.find((id) => /^GPU-/i.test(id)) ?? ids[0]
+    if (uuid) idByUuid.set(uuid, String(res.id))
+  }
+  return (metrics ?? []).map((m) => ({
+    resourceId:
+      (m.uuid && idByUuid.get(m.uuid)) ??
+      (m.index !== undefined ? `${m.vendor}${m.index}` : m.resourceId),
+    vendor: m.vendor,
+    utilizationPercent: m.utilizationPercent,
+    memoryUsedBytes: m.memoryUsedBytes,
+    memoryTotalBytes: m.memoryTotalBytes,
+    temperatureC: m.temperatureC,
+    powerWatts: m.powerWatts,
+    shared: m.shared
+  }))
+}
+
 // Resolves the GPUs a job/service actually holds and samples each one, attaching a per-device
 // entry to the container snapshot. Driven entirely by job.resources → the env resource pool,
 // so pure-CPU jobs never touch any GPU code. Best-effort: any failure yields no `gpu` field
@@ -136,6 +171,28 @@ export class GpuMetricsService {
     }
   }
 
+  // Host-wide GPU sample: every GPU visible to this process, independent of any job. This is what
+  // populates the dashboard's GPU health panels while the box is idle. `gpuResources` are the
+  // declared GPU ComputeResources (from the engine config) — used only to map an enumerated
+  // device's NVML UUID back to its configured resource id ('gpu0') so the emitted series align
+  // with `ocean_compute_env_resource_*`. A device with no matching config falls back to a
+  // vendor+index id. Returns undefined when GPU collection is disabled or nothing was read.
+  async sampleHost(
+    gpuResources: ComputeResource[]
+  ): Promise<GpuMetricsSnapshot[] | undefined> {
+    if (!gpuMetricsEnabled()) return undefined
+    try {
+      // NVIDIA only today; AMD/Intel enumeration lands with their backends.
+      const collector = this.getCollector('nvidia')
+      if (!collector) return undefined
+      const out = mapHostGpuDevices(await collector.sampleAll(), gpuResources)
+      return out.length ? out : undefined
+    } catch (e: any) {
+      CORE_LOGGER.debug(`[metrics] gpu: host sample failed: ${e?.message}`)
+      return undefined
+    }
+  }
+
   dispose(): void {
     for (const c of this.collectors.values()) {
       try {
diff --git a/src/components/c2d/gpu/nvml.ts b/src/components/c2d/gpu/nvml.ts
index 90a39a6c6..967a381ae 100644
--- a/src/components/c2d/gpu/nvml.ts
+++ b/src/components/c2d/gpu/nvml.ts
@@ -30,6 +30,10 @@ interface NvmlBindings {
   getMemoryInfo: (...a: any[]) => number
   getTemperature: (...a: any[]) => number
   getPowerUsage: (...a: any[]) => number
+  // Host-wide enumeration (for sampleAll): count devices, get a handle by index, read its UUID.
+  getCount: (...a: any[]) => number
+  getHandleByIndex: (...a: any[]) => number
+  getUUID: (...a: any[]) => number
 }
 
 export class NvmlGpuCollector implements GpuVendorCollector {
@@ -93,6 +97,20 @@ export class NvmlGpuCollector implements GpuVendorCollector {
         getPowerUsage: lib.func('nvmlDeviceGetPowerUsage', 'int', [
           deviceType,
           koffi.out(koffi.pointer('uint32'))
+        ]),
+        getCount: lib.func('nvmlDeviceGetCount_v2', 'int', [
+          koffi.out(koffi.pointer('uint32'))
+        ]),
+        getHandleByIndex: lib.func('nvmlDeviceGetHandleByIndex_v2', 'int', [
+          'uint32',
+          koffi.out(koffi.pointer(deviceType))
+        ]),
+        // char* out buffer: pass a Node Buffer (mutable, passed by reference) and read the
+        // null-terminated UUID back after the call.
+        getUUID: lib.func('nvmlDeviceGetUUID', 'int', [
+          deviceType,
+          koffi.pointer('char'),
+          'uint32'
         ])
       }
     } catch (e: any) {
@@ -138,6 +156,48 @@ export class NvmlGpuCollector implements GpuVendorCollector {
     }
   }
 
+  // Reads the four live metrics off an already-resolved NVML device handle into `base`. Each
+  // read is independent and guarded: an unreadable metric is left as its `base` default (null for
+  // util/mem, absent for temp/power) rather than failing the whole sample.
+  private applyDeviceReadings(device: any, base: GpuDeviceMetrics): void {
+    const b = this.bindings
+    if (!b) return
+    const util: any = {}
+    if (b.getUtilizationRates(device, util) === NVML_SUCCESS) {
+      base.utilizationPercent = Number(util.gpu)
+    }
+    const mem: any = {}
+    if (b.getMemoryInfo(device, mem) === NVML_SUCCESS) {
+      base.memoryUsedBytes = Number(mem.used)
+      base.memoryTotalBytes = Number(mem.total)
+    }
+    const tempOut: any[] = [0]
+    if (b.getTemperature(device, NVML_TEMPERATURE_GPU, tempOut) === NVML_SUCCESS) {
+      base.temperatureC = Number(tempOut[0])
+    }
+    const powerOut: any[] = [0]
+    if (b.getPowerUsage(device, powerOut) === NVML_SUCCESS) {
+      base.powerWatts = Number((Number(powerOut[0]) / 1000).toFixed(1)) // mW → W
+    }
+  }
+
+  // Reads a device's NVML UUID ("GPU-…") into a JS string. Best-effort: returns undefined if the
+  // binding is missing or the call fails, so the caller falls back to the enumeration index.
+  private readUuid(device: any): string | undefined {
+    const b = this.bindings
+    if (!b?.getUUID) return undefined
+    try {
+      const buf = Buffer.alloc(96)
+      if (b.getUUID(device, buf, buf.length) === NVML_SUCCESS) {
+        const end = buf.indexOf(0)
+        return buf.toString('latin1', 0, end < 0 ? buf.length : end)
+      }
+    } catch {
+      // best-effort
+    }
+    return undefined
+  }
+
   private sampleOne(handle: GpuDeviceHandle): GpuDeviceMetrics {
     const base: GpuDeviceMetrics = {
       resourceId: handle.resourceId,
@@ -152,25 +212,7 @@ export class NvmlGpuCollector implements GpuVendorCollector {
     try {
       const devOut: any[] = [null]
       if (b.getHandleByUUID(handle.uuid, devOut) !== NVML_SUCCESS) return base
-      const device = devOut[0]
-
-      const util: any = {}
-      if (b.getUtilizationRates(device, util) === NVML_SUCCESS) {
-        base.utilizationPercent = Number(util.gpu)
-      }
-      const mem: any = {}
-      if (b.getMemoryInfo(device, mem) === NVML_SUCCESS) {
-        base.memoryUsedBytes = Number(mem.used)
-        base.memoryTotalBytes = Number(mem.total)
-      }
-      const tempOut: any[] = [0]
-      if (b.getTemperature(device, NVML_TEMPERATURE_GPU, tempOut) === NVML_SUCCESS) {
-        base.temperatureC = Number(tempOut[0])
-      }
-      const powerOut: any[] = [0]
-      if (b.getPowerUsage(device, powerOut) === NVML_SUCCESS) {
-        base.powerWatts = Number((Number(powerOut[0]) / 1000).toFixed(1)) // mW → W
-      }
+      this.applyDeviceReadings(devOut[0], base)
     } catch (e: any) {
       CORE_LOGGER.debug(
         `GPU metrics (nvidia): sample of ${handle.resourceId} failed: ${e?.message}`
@@ -184,6 +226,47 @@ export class NvmlGpuCollector implements GpuVendorCollector {
     return handles.map((h) => this.sampleOne(h))
   }
 
+  // Enumerate every GPU visible to this process (nvmlDeviceGetCount + GetHandleByIndex) and
+  // sample each — the host-wide path that surfaces idle GPUs no job is holding. Each entry carries
+  // its NVML UUID (for mapping back to a configured resource id) and index (fallback identity).
+  async sampleAll(): Promise<GpuDeviceMetrics[]> {
+    if (!(await this.detect())) return []
+    const b = this.bindings
+    if (!b?.getCount || !b?.getHandleByIndex) return []
+    const out: GpuDeviceMetrics[] = []
+    try {
+      const countOut: any[] = [0]
+      if (b.getCount(countOut) !== NVML_SUCCESS) return []
+      const count = Number(countOut[0]) || 0
+      for (let i = 0; i < count; i++) {
+        const devOut: any[] = [null]
+        if (b.getHandleByIndex(i, devOut) !== NVML_SUCCESS) continue
+        const device = devOut[0]
+        const uuid = this.readUuid(device)
+        const base: GpuDeviceMetrics = {
+          resourceId: uuid ?? String(i),
+          vendor: 'nvidia',
+          utilizationPercent: null,
+          memoryUsedBytes: null,
+          memoryTotalBytes: null,
+          uuid,
+          index: i
+        }
+        try {
+          this.applyDeviceReadings(device, base)
+        } catch (e: any) {
+          CORE_LOGGER.debug(
+            `GPU metrics (nvidia): sample of device ${i} failed: ${e?.message}`
+          )
+        }
+        out.push(base)
+      }
+    } catch (e: any) {
+      CORE_LOGGER.debug(`GPU metrics (nvidia): host enumeration failed: ${e?.message}`)
+    }
+    return out
+  }
+
   dispose(): void {
     try {
       if (this.initialized && this.bindings) this.bindings.shutdown()
diff --git a/src/components/c2d/gpu/types.ts b/src/components/c2d/gpu/types.ts
index 401b231b0..c208d8886 100644
--- a/src/components/c2d/gpu/types.ts
+++ b/src/components/c2d/gpu/types.ts
@@ -23,6 +23,12 @@ export interface GpuDeviceMetrics {
   temperatureC?: number
   powerWatts?: number
   shared?: boolean
+  // Set by host-wide enumeration (sampleAll), not by job-scoped sampling: the device's stable
+  // vendor identity (nvidia: NVML UUID) and its enumeration index. The GpuMetricsService uses
+  // `uuid` to map an enumerated device back to a configured resource id ('gpu0'); `index` is the
+  // fallback identity when no configured resource matches.
+  uuid?: string
+  index?: number
 }
 
 // Vendor backend contract. Only the NVIDIA (NVML) backend is implemented today; AMD/Intel
@@ -32,6 +38,10 @@ export interface GpuVendorCollector {
   detect(): Promise<boolean> // is this backend usable on this host? Run once, cached.
   resolve(res: ComputeResource): GpuDeviceHandle | null
   sample(handles: GpuDeviceHandle[]): Promise<GpuDeviceMetrics[]>
+  // Enumerate and sample EVERY GPU visible to this process, with no job/handle — this is what
+  // gives idle host GPUs (no running job) live utilization/memory/temperature/power. Returns []
+  // when the backend is unusable on this host.
+  sampleAll(): Promise<GpuDeviceMetrics[]>
   dispose(): void
 }
 
diff --git a/src/telemetry/computeGauges.ts b/src/telemetry/computeGauges.ts
index 5ba1b24f3..77545b3e8 100644
--- a/src/telemetry/computeGauges.ts
+++ b/src/telemetry/computeGauges.ts
@@ -8,7 +8,11 @@
  * chaining and the callback simply observes nothing — it must compile and run either way.
  */
 import type { C2DEngines } from '../components/c2d/compute_engines.js'
-import type { ComputeEngineAggregate, EnvResourceSnapshot } from './computeTypes.js'
+import type {
+  ComputeEngineAggregate,
+  ComputeGpuAggregate,
+  EnvResourceSnapshot
+} from './computeTypes.js'
 import * as M from './metrics.js'
 
 // The runtime-populated fields live on the concrete engine; read them structurally so this module
@@ -17,6 +21,7 @@ type AggregatingEngine = {
   getC2DConfig?: () => { hash?: string }
   lastAggregate?: ComputeEngineAggregate
   envResourceSnapshot?: EnvResourceSnapshot
+  hostGpuSnapshot?: ComputeGpuAggregate[]
 }
 
 export function registerComputeGauges(engines: C2DEngines): void {
@@ -68,24 +73,10 @@ export function registerComputeGauges(engines: C2DEngines): void {
             obs.observe(M.cJobsQueued, a.queuedFreeJobs, { engine, free: 'true' })
           }
 
-          const gpus = a.gpus ?? []
-          obs.observe(M.cGpuDevices, gpus.length, { engine })
-          for (const g of gpus) {
-            const attrs = {
-              engine,
-              gpu: String(g.resourceId),
-              vendor: g.vendor ?? 'unknown'
-            }
-            obs.observe(M.cGpuUtil, g.utilizationPercent ?? 0, attrs)
-            obs.observe(M.cGpuMemUsed, g.memoryUsedBytes ?? 0, attrs)
-            obs.observe(M.cGpuMemTotal, g.memoryTotalBytes ?? 0, attrs)
-            if (typeof g.temperatureC === 'number') {
-              obs.observe(M.cGpuTemp, g.temperatureC, attrs)
-            }
-            if (typeof g.powerWatts === 'number') {
-              obs.observe(M.cGpuPower, g.powerWatts, attrs)
-            }
-          }
+          // Distinct GPUs currently held by running jobs = "devices in use". The per-device
+          // health series (util/mem/temp/power) are emitted below from the host-wide snapshot so
+          // they exist for every GPU even when the box is idle.
+          obs.observe(M.cGpuDevices, (a.gpus ?? []).length, { engine })
         }
 
         const env = eng?.envResourceSnapshot ?? {}
@@ -99,6 +90,51 @@ export function registerComputeGauges(engines: C2DEngines): void {
             }
           }
         }
+
+        // Per-GPU health for every visible device (idle included), sourced from the host-wide
+        // NVML snapshot. `in_use` is derived from the env resource snapshot: a GPU whose resource
+        // id shows inUse>0 in any environment is currently held by a job. The job-scoped a.gpus is
+        // a fallback only for devices the host enumeration could not read (e.g. NVML unavailable),
+        // deduped by resource id so a device is never emitted twice in one tick.
+        const inUseGpuIds = new Set<string>()
+        for (const resources of Object.values(env)) {
+          for (const [rid, v] of Object.entries(resources ?? {})) {
+            if (v && typeof v.inUse === 'number' && v.inUse > 0) inUseGpuIds.add(rid)
+          }
+        }
+        const emittedGpuIds = new Set<string>()
+        const observeGpu = (g: ComputeGpuAggregate, inUse: boolean): void => {
+          const rid = String(g.resourceId)
+          if (emittedGpuIds.has(rid)) return
+          emittedGpuIds.add(rid)
+          const attrs = {
+            engine,
+            gpu: rid,
+            vendor: g.vendor ?? 'unknown',
+            in_use: inUse ? 'true' : 'false'
+          }
+          if (typeof g.utilizationPercent === 'number') {
+            obs.observe(M.cGpuUtil, g.utilizationPercent, attrs)
+          }
+          if (typeof g.memoryUsedBytes === 'number') {
+            obs.observe(M.cGpuMemUsed, g.memoryUsedBytes, attrs)
+          }
+          if (typeof g.memoryTotalBytes === 'number') {
+            obs.observe(M.cGpuMemTotal, g.memoryTotalBytes, attrs)
+          }
+          if (typeof g.temperatureC === 'number') {
+            obs.observe(M.cGpuTemp, g.temperatureC, attrs)
+          }
+          if (typeof g.powerWatts === 'number') {
+            obs.observe(M.cGpuPower, g.powerWatts, attrs)
+          }
+        }
+        for (const g of eng?.hostGpuSnapshot ?? []) {
+          observeGpu(g, inUseGpuIds.has(String(g.resourceId)))
+        }
+        for (const g of a?.gpus ?? []) {
+          observeGpu(g, true)
+        }
       }
     },
     [

From 648e2102fe467000c4cfca3cd5f57e3f317f7238 Mon Sep 17 00:00:00 2001
From: alexcos20 <alex.coseru@gmail.com>
Date: Sat, 5 Sep 2026 00:33:22 +0300
Subject: [PATCH 2/3] add tests

---
 src/test/unit/c2d/gpuHostMetrics.test.ts | 103 +++++++++++++++++++++++
 1 file changed, 103 insertions(+)
 create mode 100644 src/test/unit/c2d/gpuHostMetrics.test.ts

diff --git a/src/test/unit/c2d/gpuHostMetrics.test.ts b/src/test/unit/c2d/gpuHostMetrics.test.ts
new file mode 100644
index 000000000..d6c3046c1
--- /dev/null
+++ b/src/test/unit/c2d/gpuHostMetrics.test.ts
@@ -0,0 +1,103 @@
+import { expect } from 'chai'
+import { mapHostGpuDevices } from '../../../components/c2d/gpu/index.js'
+import type { GpuDeviceMetrics } from '../../../components/c2d/gpu/types.js'
+import type { ComputeResource } from '../../../@types/C2D/C2D.js'
+
+/**
+ * `mapHostGpuDevices` aligns host-enumerated GPU devices (from NVML `sampleAll`) to their
+ * configured resource ids so the host-wide GPU series (`ocean_compute_gpu_*`, which now exist even
+ * when idle) carry the same `gpu` label as `ocean_compute_env_resource_*`. That label alignment is
+ * what lets the dashboard derive per-device `in_use` and dedupe the job-scoped fallback — if it is
+ * wrong, series silently mis-label instead of failing, so it is pinned down here. This is the
+ * NVML-free half of the host-GPU path; end-to-end enumeration needs a real NVIDIA host.
+ */
+
+// Minimal declared GPU resources: id 'gpu0'/'gpu1' each pinned to an NVML UUID, plus a non-GPU
+// resource that must be ignored when building the UUID → id map.
+const gpuResources = [
+  {
+    id: 'gpu0',
+    type: 'gpu',
+    init: { deviceRequests: { DeviceIDs: ['GPU-1111'] } }
+  },
+  {
+    id: 'gpu1',
+    type: 'gpu',
+    init: { deviceRequests: { DeviceIDs: ['GPU-2222'] } }
+  },
+  { id: 'cpu', type: 'cpu' }
+] as unknown as ComputeResource[]
+
+function device(overrides: Partial<GpuDeviceMetrics>): GpuDeviceMetrics {
+  return {
+    resourceId: 'placeholder',
+    vendor: 'nvidia',
+    utilizationPercent: null,
+    memoryUsedBytes: null,
+    memoryTotalBytes: null,
+    ...overrides
+  }
+}
+
+describe('mapHostGpuDevices (host-wide GPU label mapping)', () => {
+  it('maps an enumerated device to its configured resource id by NVML UUID', () => {
+    const out = mapHostGpuDevices(
+      [device({ uuid: 'GPU-2222', index: 3, utilizationPercent: 42 })],
+      gpuResources
+    )
+    expect(out).to.have.length(1)
+    expect(out[0].resourceId).to.equal('gpu1')
+    expect(out[0].utilizationPercent).to.equal(42)
+  })
+
+  it('falls back to vendor+index when the UUID matches no configured resource', () => {
+    const out = mapHostGpuDevices(
+      [device({ uuid: 'GPU-UNKNOWN', index: 5 })],
+      gpuResources
+    )
+    expect(out[0].resourceId).to.equal('nvidia5')
+  })
+
+  it('falls back to vendor+index when the device carries no UUID at all', () => {
+    const out = mapHostGpuDevices([device({ index: 0 })], gpuResources)
+    expect(out[0].resourceId).to.equal('nvidia0')
+  })
+
+  it('ignores non-GPU resources when building the UUID map', () => {
+    // 'cpu' resource has no DeviceIDs; a device claiming to be it must not resolve to 'cpu'.
+    const out = mapHostGpuDevices([device({ uuid: 'GPU-1111', index: 0 })], [
+      { id: 'cpu', type: 'cpu', init: { deviceRequests: { DeviceIDs: ['GPU-1111'] } } }
+    ] as unknown as ComputeResource[])
+    expect(out[0].resourceId).to.equal('nvidia0')
+  })
+
+  it('preserves the sampled metric values, shared flag, and null-vs-number distinction', () => {
+    const out = mapHostGpuDevices(
+      [
+        device({
+          uuid: 'GPU-1111',
+          index: 0,
+          utilizationPercent: 0,
+          memoryUsedBytes: 1024,
+          memoryTotalBytes: 4096,
+          temperatureC: 55,
+          powerWatts: 120.5,
+          shared: true
+        })
+      ],
+      gpuResources
+    )
+    const g = out[0]
+    expect(g.resourceId).to.equal('gpu0')
+    expect(g.utilizationPercent).to.equal(0) // 0 (idle) is kept, not treated as "missing"
+    expect(g.memoryUsedBytes).to.equal(1024)
+    expect(g.memoryTotalBytes).to.equal(4096)
+    expect(g.temperatureC).to.equal(55)
+    expect(g.powerWatts).to.equal(120.5)
+    expect(g.shared).to.equal(true)
+  })
+
+  it('returns an empty array for no devices (host has no visible GPUs)', () => {
+    expect(mapHostGpuDevices([], gpuResources)).to.deep.equal([])
+  })
+})

From 3e783f103b2531d322d30a912ded77874228ebd4 Mon Sep 17 00:00:00 2001
From: alexcos20 <alex.coseru@gmail.com>
Date: Sat, 5 Sep 2026 00:53:23 +0300
Subject: [PATCH 3/3] fix review

---
 .../dashboards/ocean-node-compute.json        |  4 ++--
 docs/compute.md                               |  4 +++-
 docs/env.md                                   |  2 +-
 src/components/c2d/compute_engine_docker.ts   | 22 ++++++++++---------
 4 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/deploy/telemetry/grafana/dashboards/ocean-node-compute.json b/deploy/telemetry/grafana/dashboards/ocean-node-compute.json
index 28a767df1..60985d9f8 100644
--- a/deploy/telemetry/grafana/dashboards/ocean-node-compute.json
+++ b/deploy/telemetry/grafana/dashboards/ocean-node-compute.json
@@ -846,7 +846,7 @@
           },
           "editorMode": "code",
           "expr": "ocean_compute_gpu_memory_used_bytes{service_instance_id=~\"$instance\"}",
-          "legendFormat": "used {{gpu}} {{vendor}}",
+          "legendFormat": "used {{gpu}} {{vendor}} ({{in_use}})",
           "range": true,
           "instant": false,
           "refId": "A"
@@ -858,7 +858,7 @@
           },
           "editorMode": "code",
           "expr": "ocean_compute_gpu_memory_total_bytes{service_instance_id=~\"$instance\"}",
-          "legendFormat": "total {{gpu}} {{vendor}}",
+          "legendFormat": "total {{gpu}} {{vendor}} ({{in_use}})",
           "range": true,
           "instant": false,
           "refId": "B"
diff --git a/docs/compute.md b/docs/compute.md
index 4147f082f..bd1eeab9a 100644
--- a/docs/compute.md
+++ b/docs/compute.md
@@ -234,7 +234,9 @@ The environment references it by `id`.
 > holds it — each carrying an `in_use` label (`true`/`false`) for whether a job currently holds
 > it, while `ocean_compute_gpu_devices_in_use` counts the allocated devices. This fleet-facing
 > export is independent of the per-owner job metrics above and honours the same `GPU_METRICS`
-> switch and NVML requirements.
+> switch and NVML requirements. It runs **only when the node declares GPU `ComputeResource`s**
+> (`type: "gpu"`) in `DOCKER_COMPUTE_ENVIRONMENTS`: a node that has GPUs but declares no GPU
+> resource emits no host GPU gauges, even when NVML is available.
 
 ### Multi-GPU workloads (shared memory)
 
diff --git a/docs/env.md b/docs/env.md
index 3e6fe9750..97609f2c4 100644
--- a/docs/env.md
+++ b/docs/env.md
@@ -198,7 +198,7 @@ Powers the `getNodeMetrics` (live snapshot) and `getNodeMetricsHistory` (hourly
 
 - `C2D_METRICS_INTERVAL_SECONDS`: How often (in seconds) the node samples live Docker runtime metrics (CPU, RAM, disk, network, block I/O, PIDs, exit info — plus NVIDIA GPU utilization/memory) for running compute jobs and services, persisting a snapshot onto the job record in the C2D database. These metrics are **owner-only**: they are never included in the escrow claim proof and never returned to anyone but the authenticated owner of the job/service. To that owner they come back **by default** on `COMPUTE_GET_STATUS` / `SERVICE_GET_STATUS` (no flag needed — see [API.md](API.md) for the `includeMetrics` override); an unauthenticated status call and the node-wide `serviceList` never return them. Set to `0` to disable collection entirely. Metrics are best-effort (up to one interval of staleness). Defaults to `10`. Example: `10`
 
-- `GPU_METRICS`: Controls the GPU metrics collector. `auto` (default) detects and enables the NVIDIA (NVML) backend when a GPU host is available; `off` disables GPU collection. Requires the optional `koffi` dependency and `libnvidia-ml.so.1` reachable **by the node process** — note that a containerized node does not get the NVIDIA driver libraries just because the host has them, so this is the usual reason GPU metrics are missing (`could not bind libnvidia-ml.so.1`); [compute.md → Troubleshooting GPU metrics](compute.md#troubleshooting-gpu-metrics) lists every warning and its fix. If either is missing, GPU metrics are skipped (no `gpu` field) while container-level metrics continue. GPU utilization/memory/temperature/power are sampled **host-wide** — every GPU visible to the node process is reported, idle ones included, so a card's health shows even when no job holds it — with an `in_use` label (`true`/`false`) marking devices currently allocated to a job; `ocean_compute_gpu_devices_in_use` counts the allocated ones. AMD and Intel backends are not yet implemented. Cadence reuses `C2D_METRICS_INTERVAL_SECONDS`. Defaults to `auto`. Example: `auto`
+- `GPU_METRICS`: Controls the GPU metrics collector. `auto` (default) detects and enables the NVIDIA (NVML) backend when a GPU host is available; `off` disables GPU collection. Requires the optional `koffi` dependency and `libnvidia-ml.so.1` reachable **by the node process** — note that a containerized node does not get the NVIDIA driver libraries just because the host has them, so this is the usual reason GPU metrics are missing (`could not bind libnvidia-ml.so.1`); [compute.md → Troubleshooting GPU metrics](compute.md#troubleshooting-gpu-metrics) lists every warning and its fix. If either is missing, GPU metrics are skipped (no `gpu` field) while container-level metrics continue. GPU utilization/memory/temperature/power are sampled **host-wide** — every GPU visible to the node process is reported, idle ones included, so a card's health shows even when no job holds it — with an `in_use` label (`true`/`false`) marking devices currently allocated to a job; `ocean_compute_gpu_devices_in_use` counts the allocated ones. The host-wide export runs only when the node declares GPU `ComputeResource`s (`type: "gpu"`) in `DOCKER_COMPUTE_ENVIRONMENTS`; a node with GPUs but no declared GPU resource emits no host GPU gauges even if NVML is available. AMD and Intel backends are not yet implemented. Cadence reuses `C2D_METRICS_INTERVAL_SECONDS`. Defaults to `auto`. Example: `auto`
 
 - `SERVICE_TEMPLATES_PATH`: Path to a folder of operator-published Service-on-Demand template files (`*.json`, validated against the template schema). The folder is re-read on every `serviceTemplates` request, so templates can be added, edited, or removed without restarting the node. Maps to the `serviceTemplatesPath` config field. Defaults to `databases/serviceTemplates/`, which the image does not create — the operator mounts templates into it (a missing folder simply means no templates). See the [Services guide](services.md). Example: `/templates`
 
diff --git a/src/components/c2d/compute_engine_docker.ts b/src/components/c2d/compute_engine_docker.ts
index a4dbb2cb4..4a636b18d 100755
--- a/src/components/c2d/compute_engine_docker.ts
+++ b/src/components/c2d/compute_engine_docker.ts
@@ -3118,18 +3118,11 @@ export class C2DEngineDocker extends C2DEngine {
     }
   }
 
-  // ONE line per sampling interval with the engine's whole live resource picture, plus a
-  // "pressure" line for each workload that is close to a limit. This is the admin's entry
-  // point into the metrics: `grep '\[metrics\]'` for everything, `grep '\[metrics\] summary'`
-  // for the roll-up, `grep '\[metrics\] pressure'` for what is about to hurt.
-  //
-  // Reads the snapshots already sampled this tick (no extra Docker or DB calls) and is
-  // throttled to C2D_METRICS_INTERVAL_SECONDS — the loop itself ticks every 2s, but snapshots
-  // only refresh once per interval, so logging every tick would just repeat numbers.
-  // Never throws: a logging failure must not touch the loop.
   // Sample the host's GPUs (all of them, not just those held by a running job) so the compute
   // dashboard shows GPU health even while idle. Throttled to the metrics cadence and strictly
-  // best-effort: any failure leaves the previous snapshot in place and never disturbs the loop.
+  // best-effort: a mid-tick throw never disturbs the loop and leaves the previous snapshot in
+  // place, while a completed sample that read no device clears the snapshot — so a GPU that has
+  // dropped off the bus (or NVML gone unavailable) surfaces as a gap, not frozen stale readings.
   // No-ops entirely on a node that declares no GPU resources, so pure-CPU nodes never touch NVML.
   private async refreshHostGpuSnapshot(): Promise<void> {
     try {
@@ -3160,6 +3153,15 @@ export class C2DEngineDocker extends C2DEngine {
     }
   }
 
+  // ONE line per sampling interval with the engine's whole live resource picture, plus a
+  // "pressure" line for each workload that is close to a limit. This is the admin's entry
+  // point into the metrics: `grep '\[metrics\]'` for everything, `grep '\[metrics\] summary'`
+  // for the roll-up, `grep '\[metrics\] pressure'` for what is about to hurt.
+  //
+  // Reads the snapshots already sampled this tick (no extra Docker or DB calls) and is
+  // throttled to C2D_METRICS_INTERVAL_SECONDS — the loop itself ticks every 2s, but snapshots
+  // only refresh once per interval, so logging every tick would just repeat numbers.
+  // Never throws: a logging failure must not touch the loop.
   private logMetricsSummary(
     jobs: DBComputeJob[] = [],
     services: ServiceJob[] = []