GPU Metal: run the work of a stream in order

Consecutive command buffers of one Metal queue are only ordered as far as
the hazard tracking of the bound buffers sees, and the kernels reach most
memory through addresses stored in constant memory. A kernel could thus
start before the one before it on the same stream had finished: on a
5-event pp time frame the cluster finder found no clusters, and between
12,125 and 12,142 instead of 19,009 with every kernel synchronised.
Make each command buffer of a stream wait on a per-stream event that the
previous one signals, which gives a CUDA stream's in-order semantics
without blocking the host.

Author: ktf

GPU/GPUTracking/Base/metal/GPUReconstructionMetal.mm

@@ -105,6 +105,8 @@
       if (mInternals->commandBuffers[i] == nil) {
         GPUErrorReturn("Error creating Metal command buffer");
       }
+      mInternals->streamOrder[i] = [mInternals->device newSharedEvent];
+      mInternals->streamOrderValue[i] = 0;
     }
 
     mInternals->mem_host = [mInternals->device newBufferWithLength:mHostMemorySize options:MTLResourceStorageModeShared];
@@ -179,6 +181,7 @@
         DrainStream(i);
         [mInternals->commandBuffers[i] release];
         [mInternals->commandQueues[i] release];
+        [mInternals->streamOrder[i] release];
       }
       [mInternals->mem_host release];
     }
@@ -323,11 +326,17 @@ MetalMemRef metalResolve(const GPUReconstructionMetalInternals* internals, const
 void GPUReconstructionMetal::CommitStream(int32_t stream)
 {
   id<MTLCommandBuffer> cb = mInternals->commandBuffers[stream];
+  // A CUDA stream runs its work in order, a Metal queue only as far as its
+  // hazard tracking sees, and the kernels reach most memory through addresses
+  // in constant memory. So each buffer of a stream waits for the previous one.
+  const uint64_t value = ++mInternals->streamOrderValue[stream];
+  [cb encodeSignalEvent:mInternals->streamOrder[stream] value:value];
   [cb commit];
   // Hand the committed buffer over to the stream's submission list, which owns
   // it until it is waited for, and open a fresh one to encode the next work in.
   mInternals->submittedBuffers[stream].emplace_back(cb);
   mInternals->commandBuffers[stream] = [[mInternals->commandQueues[stream] commandBuffer] retain];
+  [mInternals->commandBuffers[stream] encodeWaitForEvent:mInternals->streamOrder[stream] value:value];
 }
 
 void GPUReconstructionMetal::DrainStream(int32_t stream)

GPU/GPUTracking/Base/metal/GPUReconstructionMetalIncludesHost.h

@@ -59,6 +59,9 @@ struct GPUReconstructionMetalInternals {
   // encoded into afterwards, so the two cannot be the same object.
   std::array<id<MTLCommandBuffer>, constants::GPU_MAX_STREAMS> commandBuffers;
   std::array<std::vector<id<MTLCommandBuffer>>, constants::GPU_MAX_STREAMS> submittedBuffers;
+  // Signalled by each committed buffer and waited for by the next one, see CommitStream
+  std::array<id<MTLSharedEvent>, constants::GPU_MAX_STREAMS> streamOrder;
+  std::array<uint64_t, constants::GPU_MAX_STREAMS> streamOrderValue;
 
   std::vector<id<MTLFunction>> functions;
   std::vector<id<MTLComputePipelineState>> pipelines;