GPU Metal: make command-buffer failures fatal

A failed Metal command buffer (e.g. killed by the macOS GPU watchdog,
kIOGPUCommandBufferCallbackErrorImpactingInteractivity / ...Timeout) was
only logged when the stream was drained, and the reconstruction carried on
without its work: on 20 kHz Pb-Pb time frames this gave silently wrong
outputs (271,967-296,957 tracks vs 297,669 on CPU for the same TF).
Worse, a failed buffer never signals the events it was to signal, so
later buffers waiting for them on the GPU, and the host waiting for those,
blocked forever: one such run hung at 0% CPU.

Signal the stream-order event and the recorded event of a buffer from its
completion handler when it fails, so everything queued behind it
completes, and fail in DrainStream through GPUChkErr, like the CUDA and
HIP backends do on a failed synchronisation ("GPU Backend Failure").
Failures seen while the device is torn down are only logged, so the
destructor does not throw.

pbpb30 (M1 Max, abp.sh, 3 rounds, --runs 10): 966.7 ms/ev before, 962.5
after; Metal dumps byte-identical before and after; sim5pp dump still
identical to CPU.

Author: ktf

GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm

@@ -212,7 +212,8 @@ uint32_t metalGPUCoreCount()
 
 int32_t GPUReconstructionMetal::ExitDevice_Runtime()
 {
-  // Tear down Metal
+  // Tear down Metal, also after a failure, without failing again
+  mInternals->exiting = true;
   SynchronizeGPU();
 
   for (uint32_t i = 0; i < mEvents.size(); i++) {
@@ -380,7 +381,9 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const
   // Does not change the retain count, so it's important we manage
   // the lifetime of the events outside here.
   auto mtlEvent = (__bridge id<MTLSharedEvent>)(ev->get<void*>());
-  [cb encodeSignalEvent:mtlEvent value:EventValue(*ev, true)];
+  const uint64_t value = EventValue(*ev, true);
+  [cb encodeSignalEvent:mtlEvent value:value];
+  metalSignalOnError(cb, mtlEvent, value);
   CommitStream(stream);
 }
 
@@ -392,6 +395,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const
   // in constant memory. So each buffer of a stream waits for the previous one.
   const uint64_t value = ++mInternals->streamOrderValue[stream];
   [cb encodeSignalEvent:mInternals->streamOrder[stream] value:value];
+  metalSignalOnError(cb, mInternals->streamOrder[stream], value);
   [cb commit];
   // Hand the committed buffer over to the stream's submission list, which owns
   // it until it is waited for, and open a fresh one to encode the next work in.
@@ -420,16 +424,22 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const
   const auto t0 = std::chrono::steady_clock::now();
   while (!mInternals->submittedBuffers[stream].empty() && !(done = mInternals->streamOrder[stream].signaledValue >= mInternals->streamOrderValue[stream]) && std::chrono::steady_clock::now() - t0 < std::chrono::milliseconds(20)) {
   }
+  int64_t error = 0;
   for (id<MTLCommandBuffer> cb : mInternals->submittedBuffers[stream]) {
     if (!done) {
       [cb waitUntilCompleted];
     }
     if (cb.status == MTLCommandBufferStatusError) {
       GPUError("Metal command buffer on stream %d failed: %s", stream, [[cb.error description] UTF8String]);
+      error = error ? error : cb.error.code;
     }
     [cb release];
   }
   mInternals->submittedBuffers[stream].clear();
+  // The work of a failed buffer is lost, so the results are wrong: fail like the other backends.
+  if (error && !mInternals->exiting) {
+    GPUChkErr(error);
+  }
 }
 
 void GPUReconstructionMetal::SynchronizeGPU()

GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h

@@ -66,6 +66,8 @@ struct GPUReconstructionMetalInternals {
   // collects the stream's work until something has to wait for it.
   std::array<id<MTLFence>, constants::GPU_MAX_STREAMS> streamFence;
   std::array<bool, constants::GPU_MAX_STREAMS> streamHasWork;
+  // Set while the device is torn down, when a failed command buffer is no longer fatal
+  bool exiting = false;
 
   std::vector<id<MTLFunction>> functions;
   std::vector<id<MTLComputePipelineState>> pipelines;
@@ -90,6 +92,17 @@ inline void metalBeginEncoder(GPUReconstructionMetalInternals* internals, E enco
 {
   [encoder waitForFence:internals->streamFence[stream]];
 }
+// A failed command buffer does not signal its events, and the work waiting for
+// them would wait forever: signal them from the host, so that everything
+// completes and the failure is reported when the host synchronises.
+inline void metalSignalOnError(id<MTLCommandBuffer> cb, id<MTLSharedEvent> event, uint64_t value)
+{
+  [cb addCompletedHandler:^(id<MTLCommandBuffer> c) {
+    if (c.status == MTLCommandBufferStatusError && event.signaledValue < value) {
+      event.signaledValue = value;
+    }
+  }];
+}
 template <class E>
 inline void metalEndEncoder(GPUReconstructionMetalInternals* internals, E encoder, int32_t stream)
 {