GPU Metal: give the device events CUDA's semantics

The events signalled and waited for the fixed value 1, and ReleaseEvent
reset them to 0, so a wait issued after a release, or for an event not
recorded yet, never completed: the GPU watchdog killed each such command
buffer after about 10 s, and the queue then dropped all later work. On a
5-event pp time frame this made the TPC reconstruction take 80 s and
return nothing. Count up per event instead: RecordMarker signals the next
value, a wait asks for the last recorded one and is a no-op if there is
none, as cudaStreamWaitEvent is, and ReleaseEvent does nothing, as in
CUDA.

Author: ktf

GPU/GPUTracking/Base/metal/GPUReconstructionMetal.h

@@ -59,6 +59,7 @@ class GPUReconstructionMetal : public GPUReconstructionProcessing::KernelInterfa
 
   void CommitStream(int32_t stream);
   void DrainStream(int32_t stream);
+  uint64_t EventValue(deviceEvent ev, bool next);
 
  private:
   int32_t AddKernels();

GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm

@@ -291,10 +291,14 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const
 
 void GPUReconstructionMetal::ReleaseEvent(deviceEvent ev)
 {
-  // FIXME: is this supposed to reset the event for it to be repurposed
-  //        or to decrease the ref count?
-  auto mtlEvent = (__bridge id<MTLSharedEvent>)(ev.get<void*>());
-  [mtlEvent setSignaledValue:0];
+  // Nothing to do, as in CUDA: the next RecordMarker supersedes the last one.
+}
+
+uint64_t GPUReconstructionMetal::EventValue(deviceEvent ev, bool next)
+{
+  std::lock_guard<std::mutex> lock(mInternals->eventMutex);
+  auto& value = mInternals->eventValues[ev.get<void*>()];
+  return next ? ++value : value;
 }
 
 void GPUReconstructionMetal::RecordMarker(deviceEvent* ev, int32_t stream)
@@ -303,7 +307,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const
   // Does not change the retain count, so it's important we manage
   // the lifetime of the events outside here.
   auto mtlEvent = (__bridge id<MTLSharedEvent>)(ev->get<void*>());
-  [cb encodeSignalEvent:mtlEvent value:1];
+  [cb encodeSignalEvent:mtlEvent value:EventValue(*ev, true)];
   CommitStream(stream);
 }
 
@@ -353,7 +357,9 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const
   id<MTLCommandBuffer> cb = mInternals->commandBuffers[stream];
   for (int32_t ei = 0; ei < nEvents; ei++) {
     auto mtlEvent = (__bridge id<MTLSharedEvent>)(evList[ei].get<void*>());
-    [cb encodeWaitForEvent:mtlEvent value:1];
+    if (const uint64_t value = EventValue(evList[ei], false)) {
+      [cb encodeWaitForEvent:mtlEvent value:value];
+    }
   }
   CommitStream(stream);
   DrainStream(stream);
@@ -363,7 +369,7 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const
 {
   for (int32_t i = 0; i < nEvents; i++) {
     auto mtlEvent = (__bridge id<MTLSharedEvent>)(evList[i].get<void*>());
-    if (mtlEvent.signaledValue == 0) {
+    if (mtlEvent.signaledValue < EventValue(evList[i], false)) {
       return false;
     }
   }

GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h

@@ -32,6 +32,8 @@ using half = _Float16;
 #include <vector>
 #include <string>
 #include <memory>
+#include <mutex>
+#include <unordered_map>
 #include "GPULogging.h"
 
 #include "GPUReconstructionMetal.h"
@@ -67,6 +69,12 @@ struct GPUReconstructionMetalInternals {
   id<MTLBuffer> mem_host;
 
   id<MTLLibrary> library;
+
+  // The value each event was last recorded with. An MTLSharedEvent only counts
+  // up, so recording signals the next value and waiting asks for the last one,
+  // which makes waiting for a never recorded event a no-op, as in CUDA.
+  std::unordered_map<void*, uint64_t> eventValues;
+  std::mutex eventMutex;
 };
 } // namespace o2::gpu