Add a multithreaded Geant4 mode to o2-sim

This adds the --isMT mode, in which one sim process runs Geant4 with one transport thread per sim worker.

- Each Geant4 worker thread asks the primary server for its sub-events and sends its hits to the merger, as a worker process does.
- The VMC application is cloned per thread, with its own stack, random generator, external decayer and data channel.
- The threads share one request channel to the primary server.
- Their data channels are closed before the process exits, so that the merger receives all queued hits.
- A worker leaves its event loop once the server has no more work, after an empty event.
- Detector clones get their own hit buffers, and the worker volume map points at the worker's own modules.
- The media energy cuts and the worker-only Geant4 commands are applied on the worker threads.
- The cached magnetic field values are invalidated at the start of each track.
- ShmManager takes a mutex for its allocations, and the single sim process occupies the whole shared region.
- 16 detectors are enabled for MT; EMCAL, HMPID, ZDC, FOCAL, FT0 and FV0 needed fixes to their copy constructor or destructor.
- --isMT stops with an error for other engines, for Geant4 built without multithreading, for VecGeom navigation, and for detectors not yet enabled for it.

https://its.cern.ch/jira/browse/O2-7305
vmc-project/geant4_vmc#92

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Author: sawenzel

Common/Utils/include/CommonUtils/ShmManager.h

@@ -22,6 +22,7 @@
 #include <list>
 #include <cstddef>
 #include <atomic>
+#include <mutex>
 
 #if !defined(__CINT__) && !defined(__MAKECINT__) && !defined(__ROOTCLING__) && !defined(__CLING__)
 #include <boost/interprocess/managed_external_buffer.hpp>
@@ -70,7 +71,8 @@ class ShmManager
 
   // create the local segment
   // this will occupy a subregion of an already created global shared mem segment
-  void occupySegment();
+  // (all of it for the single process of multithreaded Geant4, whose threads share it)
+  void occupySegment(bool wholeRegion = false);
 
   // simply attaches to the global segment
   bool attachToGlobalSegment();
@@ -88,7 +90,7 @@ class ShmManager
   // returns if pointer is part of the shm region under control of this manager
   bool isPointerOk(void* ptr) const
   {
-    return mBufferPtr && getPointerOffset(ptr) < SHMPOOLSIZE;
+    return mBufferPtr && getPointerOffset(ptr) < mPoolSize;
   }
 
   // returns if shared mem setup is correctly setup/operational
@@ -114,6 +116,8 @@ class ShmManager
   ShmMetaInfo* mSegInfoPtr = nullptr; // pointing to the meta information object
   bool mIsMaster = false;             // true if the manager who allocated the region
   bool mIsOperational = false;
+  size_t mPoolSize = SHMPOOLSIZE; // size of the region occupied by this process
+  std::mutex mAllocMutex;         // the boost buffer has no locking of its own; threads of multithreaded Geant4 share it
   // helper function
   void* tryAttach(bool& success);
   size_t getPointerOffset(void* ptr) const { return (size_t)((char*)ptr - (char*)mBufferPtr); }

Common/Utils/src/ShmManager.cxx

@@ -178,7 +178,7 @@ bool ShmManager::attachToGlobalSegment()
   return b;
 }
 
-void ShmManager::occupySegment()
+void ShmManager::occupySegment(bool wholeRegion)
 {
   LOG(info) << "OCCUPYING A SEGMENT IN A SHARED REGION";
   // get information about the global shared mem in which to occupy a region
@@ -197,15 +197,17 @@ void ShmManager::occupySegment()
   if (b) {
     // read meta information in the segment to determine the id of this segment
     auto info = static_cast<o2::utils::ShmMetaInfo*>(addr);
-    const int segmentcounter = info->counter.fetch_add(1);
+    const int npools = wholeRegion ? (info->allocedbytes - sizeof(ShmMetaInfo)) / SHMPOOLSIZE : 1;
+    const int segmentcounter = info->counter.fetch_add(npools);
     LOG(info) << "SEGMENTCOUNT " << segmentcounter;
 
     const auto offset_in_bytes = sizeof(ShmMetaInfo) + segmentcounter * SHMPOOLSIZE;
     mBufferPtr = (void*)(((char*)addr) + offset_in_bytes);
+    mPoolSize = npools * SHMPOOLSIZE;
 
-    assert((unsigned long long)((char*)mBufferPtr - (char*)addr) + SHMPOOLSIZE <= info->allocedbytes);
+    assert((unsigned long long)((char*)mBufferPtr - (char*)addr) + mPoolSize <= info->allocedbytes);
 
-    boostmanagedbuffer = new boost::interprocess::wmanaged_external_buffer(create_only, mBufferPtr, SHMPOOLSIZE);
+    boostmanagedbuffer = new boost::interprocess::wmanaged_external_buffer(create_only, mBufferPtr, mPoolSize);
     boostallocator = new boost::interprocess::allocator<char, wmanaged_external_buffer::segment_manager>(
       boostmanagedbuffer->get_segment_manager());
 
@@ -225,6 +227,7 @@ ShmManager::~ShmManager()
 void* ShmManager::getmemblock(size_t size)
 {
   void* addr = nullptr;
+  std::lock_guard<std::mutex> lock(mAllocMutex);
   try {
     addr = (void*)boostallocator->allocate(size).get();
   } catch (const std::exception& e) {
@@ -235,6 +238,7 @@ void* ShmManager::getmemblock(size_t size)
 
 void ShmManager::freememblock(void* ptr, size_t s)
 {
+  std::lock_guard<std::mutex> lock(mAllocMutex);
   boostallocator->deallocate((char*)ptr, s);
 }
 

Detectors/3rdparty/FairRootBase/include/FairMCApplication.h

@@ -57,6 +57,7 @@ enum class FairMCApplicationState
  * @since 12.01.04
  */
 
+class FairModule;
 class FairMCApplication : public TVirtualMCApplication
 {
   public:
@@ -304,16 +305,24 @@ class FairMCApplication : public TVirtualMCApplication
 
     ClassDef(FairMCApplication, 4);
 
-  private:
-    /** Protected copy constructor */
+  protected:
+    /** Copy constructor, used to create the application of a worker thread */
     FairMCApplication(const FairMCApplication&);
+
+  private:
     /** Protected assignment operator */
     FairMCApplication& operator=(const FairMCApplication&);
 
     FairRunInfo fRunInfo;   //!
     Bool_t fGeometryIsInitialized;
 
     static FairMCApplication* fgMasterInstance;
+
+  protected:
+    /** True for the application on the master thread, false for a worker clone */
+    Bool_t IsMasterApplication() const { return !fgMasterInstance || fgMasterInstance == this; }
+    /** The worker clone of a module of the master application (the module itself on the master) */
+    FairModule* OwnModule(FairModule* module) const;
 };
 
 // inline functions

Detectors/3rdparty/FairRootBase/include/FairModule.h

@@ -159,6 +159,10 @@ class FairModule : public TNamed
     Bool_t fGeoSaved;   //! flag for initialisation
     TVirtualMC* fMC;    //! cahed pointer to MC (available only after initialization)
 
+  public:
+    /** Sets the cached MC, for a module cloned for a worker thread before its MC existed */
+    void SetMC(TVirtualMC* mc) { fMC = mc; }
+
     ClassDef(FairModule, 4);
 };
 

Detectors/3rdparty/FairRootBase/src/FairMCApplication.cxx

@@ -613,6 +613,24 @@ TVirtualMCApplication* FairMCApplication::CloneForWorker() const
     return workerApplication;
 }
 
+//_____________________________________________________________________________
+FairModule* FairMCApplication::OwnModule(FairModule* module) const
+{
+    // On a worker, the sensitive volumes still refer to the master's modules; the clones are in the same order
+    if (!module || IsMasterApplication()) {
+        return module;
+    }
+    // volumes registered by the worker's own clones (in their Initialize) already point at them
+    if (fModules->IndexOf(module) >= 0) {
+        return module;
+    }
+    Int_t index = fgMasterInstance->fModules->IndexOf(module);
+    if (index < 0 || !fModules->At(index)) {
+        LOG(fatal) << "FairMCApplication: no worker clone for module " << module->GetName();
+    }
+    return static_cast<FairModule*>(fModules->At(index));
+}
+
 //_____________________________________________________________________________
 void FairMCApplication::InitOnWorker()
 {
@@ -1014,7 +1032,8 @@ void FairMCApplication::InitGeometry()
     while ((obj = fModIter->Next())) {
         detector = dynamic_cast<FairDetector*>(obj);
         module = dynamic_cast<FairModule*>(obj);
-        if (module) {
+        if (module && IsMasterApplication()) {
+            // media cuts are set once on the master; multithreaded Geant4 workers share them
             module->SetSpecialPhysicsCuts();
         }
         if (detector) {
@@ -1101,21 +1120,30 @@ void FairMCApplication::InitGeometry()
                     }
                     FairVolume* fNewV = new FairVolume(fv->GetName(), id);
                     fNewV->setModId(fv->getModId());
-                    fNewV->SetModule(fv->GetModule());
+                    fNewV->SetModule(OwnModule(fv->GetModule()));
                     fNewV->setCopyNo(fN->GetNumber());
                     fNewV->setMCid(id);
                     fVolMap.insert(pair<Int_t, FairVolume*>(id, fNewV));
                 }
             } else {
                 FairVolume* fNewV = new FairVolume(fv->GetName(), id);
                 fNewV->setModId(fv->getModId());
-                fNewV->SetModule(fv->GetModule());
+                fNewV->SetModule(OwnModule(fv->GetModule()));
                 fNewV->setCopyNo(1);
                 fNewV->setMCid(id);
                 fVolMap.insert(pair<Int_t, FairVolume*>(id, fNewV));
             }
-        } else {
+        } else if (IsMasterApplication()) {
             fVolMap.insert(pair<Int_t, FairVolume*>(id, fv));
+        } else {
+            // the volume object is shared with the master; a worker needs its own, pointing at its own module
+            FairVolume* fNewV = new FairVolume(fv->GetName(), id);
+            fNewV->setModId(fv->getModId());
+            fNewV->SetModule(OwnModule(fv->GetModule()));
+            fNewV->setCopyNo(fv->getCopyNo());
+            fNewV->setMCid(id);
+            fNewV->setGeoNode(fv->getGeoNode());
+            fVolMap.insert(pair<Int_t, FairVolume*>(id, fNewV));
         }
     }   // end off loop Fill sensitive volumes
 

Detectors/Base/CMakeLists.txt

@@ -35,6 +35,7 @@ o2_add_library(DetectorsBase
                        src/Stack.cxx
                        src/VMCSeederService.cxx
                        src/SimRandom.cxx
+                       src/MTWorkerHooks.cxx
                        src/GlobalParams.cxx
                        src/O2Tessellated.cxx
                        src/TGeoGeometryUtils.cxx
@@ -97,6 +98,13 @@ o2_add_test(
   PUBLIC_LINK_LIBRARIES O2::DetectorsBase
   LABELS detectorsbase)
 
+o2_add_test(
+  DetImplClone
+  SOURCES test/testDetImplClone.cxx
+  COMPONENT_NAME DetectorsBase
+  PUBLIC_LINK_LIBRARIES O2::DetectorsBase
+  LABELS detectorsbase)
+
 o2_add_test(
   O2Tessellated
   SOURCES test/testO2Tessellated.cxx

Detectors/Base/include/DetectorsBase/Detector.h

@@ -219,6 +219,9 @@ class Detector : public FairDetector
   // ... for things that should be setup in each simulation worker separately)
   virtual void initializeLate() = 0;
 
+  /// whether this detector may run in multithreaded Geant4, with one clone per worker thread
+  virtual bool isMTSafe() const { return false; }
+
   /// helper wrapper function to register a geometry volume given by name with FairRoot
   /// @returns The MonteCarlo ID for the volume
   int registerSensitiveVolumeAndGetVolID(std::string const& name);
@@ -334,6 +337,10 @@ class DetImpl : public o2::base::Detector
   // offer same constructors as base
   using Detector::Detector;
 
+  /// copy for a worker thread of multithreaded Geant4; the copy creates its own hit buffers in initializeLate
+  DetImpl(const DetImpl& rhs) : Detector(rhs) {}
+  DetImpl& operator=(const DetImpl&) = delete;
+
   // default implementation for getHitBranchNames
   std::string getHitBranchNames(int probe) const override
   {

Detectors/Base/include/DetectorsBase/MTWorkerHooks.h

@@ -0,0 +1,52 @@
+// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#ifndef ALICEO2_BASE_MTWORKERHOOKS_H
+#define ALICEO2_BASE_MTWORKERHOOKS_H
+
+#include <functional>
+#include <memory>
+#include <fairmq/FwdDecls.h>
+
+class TVirtualMCDecayer;
+
+namespace o2::data
+{
+class PrimaryChunk;
+}
+
+namespace o2::base
+{
+
+/// What the simulation setup provides to the transport threads of multithreaded Geant4.
+/// Installed once on the master thread before transport starts; each function is called on a worker thread.
+struct MTWorkerHooks {
+  /// Returns the next chunk of primaries for the calling thread, or nullptr when there is no more work
+  std::function<std::unique_ptr<o2::data::PrimaryChunk>()> fetchChunk;
+  /// Returns the calling thread's channel to the hit merger
+  std::function<fair::mq::Channel*()> dataChannel;
+  /// Creates the external decayer of the calling thread (empty if there is none)
+  std::function<TVirtualMCDecayer*()> makeDecayer;
+  /// Ends the event loop of the calling thread after the current event
+  std::function<void()> stopWorker;
+  /// Called on a worker thread at the start of each run, after its user actions exist
+  std::function<void()> beginRunOnWorker;
+  /// Called once on the master thread after initialisation and before the first transport
+  std::function<void()> beforeTransport;
+  /// Called on a worker thread at the start of each track
+  std::function<void()> beginTrack;
+
+  static MTWorkerHooks& instance();
+};
+
+} // namespace o2::base
+
+#endif

Detectors/Base/src/MTWorkerHooks.cxx

@@ -0,0 +1,23 @@
+// Copyright 2019-2026 CERN and copyright holders of ALICE O2.
+// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
+// All rights not expressly granted are reserved.
+//
+// This software is distributed under the terms of the GNU General Public
+// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
+//
+// In applying this license CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include "DetectorsBase/MTWorkerHooks.h"
+
+namespace o2::base
+{
+
+MTWorkerHooks& MTWorkerHooks::instance()
+{
+  static MTWorkerHooks hooks;
+  return hooks;
+}
+
+} // namespace o2::base

Detectors/Base/src/Stack.cxx

@@ -120,6 +120,7 @@ Stack::Stack(const Stack& rhs)
   : FairGenericStack(rhs),
     mStack(),
     mParticles(),
+    mTrackIDtoParticlesEntry(rhs.mTrackIDtoParticlesEntry.size(), -1),
     mTracks(nullptr),
     mIndexMap(),
     mIndexOfCurrentTrack(-1),
@@ -131,10 +132,15 @@ Stack::Stack(const Stack& rhs)
     mNumberOfPrimariesPopped(0),
     mStoreMothers(rhs.mStoreMothers),
     mStoreSecondaries(rhs.mStoreSecondaries),
+    mPruneKinematics(rhs.mPruneKinematics),
     mMinHits(rhs.mMinHits),
     mEnergyCut(rhs.mEnergyCut),
-    mTrackRefs(new std::vector<o2::TrackReference>),
-    mIsG4Like(rhs.mIsG4Like)
+    mCleanupThreshold(rhs.mCleanupThreshold),
+    mIsG4Like(rhs.mIsG4Like),
+    mIsExternalMode(rhs.mIsExternalMode),
+    mDoTrackSeeding(rhs.mDoTrackSeeding),
+    mTransportPrimary(rhs.mTransportPrimary),
+    mTrackRefs(new std::vector<o2::TrackReference>)
 {
   LOG(debug) << "copy constructor called";
   mTracks = new std::vector<MCTrack>();