Enable user to take ownership of MPI #722
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| #ifndef SUBCOMMUNICATOR_H | ||
| #define SUBCOMMUNICATOR_H | ||
|
|
||
| #include "quest/include/config.h" | ||
|
|
||
| #if QUEST_COMPILE_MPI && QUEST_COMPILE_SUBCOMM | ||
|
Member
There was a problem hiding this comment. I'm not thrilled about guarding the entire header, and the contextualising of exposing subcommunicator as "compiling". Semantics like "expose subcommunicator" might have been better, since all the internal trickery of "don't expose internal subcommunicator-specific functions to other TUs" is brittle when This is a half-thought for a future me |
||
|
|
||
| #include <mpi.h> | ||
|
|
||
| #ifdef __cplusplus | ||
| extern "C" { | ||
| #endif | ||
|
|
||
| /** @notyetdoced | ||
| * Advanced initialiser which allows the user to provide an MPI communicator for QuEST to use. | ||
| * Use of this initialiser implies userOwnsMpi = true, (exposed by initCustomMpiQuESTEnv) and | ||
| * therefore that they have already initialised MPI, and they will call MPI_Finalize at the | ||
| * appropriate time. | ||
| * | ||
| * The user-provided MPI communicator undergoes the same validation procedure as any that QuEST | ||
| * would use, and so must contain a power-of-2 number of processes. | ||
| */ | ||
| void initCustomMpiCommQuESTEnv(MPI_Comm questComm, int useGpuAccel, int useMultithread); | ||
|
|
||
| #ifdef __cplusplus | ||
| } | ||
| #endif | ||
|
|
||
| #endif | ||
|
|
||
| #endif | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,6 +12,7 @@ target_sources(QuEST | |
| operations.cpp | ||
| paulis.cpp | ||
| qureg.cpp | ||
| subcommunicator.cpp | ||
| trotterisation.cpp | ||
| types.cpp | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -71,7 +71,7 @@ static bool hasEnvBeenFinalized = false; | |
| */ | ||
|
|
||
|
|
||
| void validateAndInitCustomQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuAccel, int useMultithread, const char* caller) { | ||
|
|
||
|
Member
There was a problem hiding this comment. This is missing validation that when |
||
| // ensure that we are never re-initialising QuEST (even after finalize) because | ||
| // this leads to undefined behaviour in distributed mode, as per the MPI | ||
|
|
@@ -93,8 +93,7 @@ void validateAndInitCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMulti | |
| // and before any GPU initialisation and validation, since we will | ||
| // perform that specifically upon the MPI-process-bound GPU(s). Further, | ||
| // we can make sure validation errors are reported only by the root node. | ||
| comm_init(useDistrib, userOwnsMpi); | ||
|
|
||
| validate_newEnvDistributedBetweenPower2Nodes(caller); | ||
|
|
||
|
|
@@ -145,6 +144,7 @@ void validateAndInitCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMulti | |
| globalEnvPtr->isMultithreaded = useMultithread; | ||
| globalEnvPtr->isGpuAccelerated = useGpuAccel; | ||
| globalEnvPtr->isDistributed = useDistrib; | ||
| globalEnvPtr->userOwnsMpi = userOwnsMpi; | ||
| globalEnvPtr->isCuQuantumEnabled = useCuQuantum; | ||
| globalEnvPtr->isGpuSharingEnabled = permitGpuSharing; | ||
|
|
||
|
|
@@ -153,6 +153,11 @@ void validateAndInitCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMulti | |
| globalEnvPtr->numNodes = (useDistrib)? comm_getNumNodes() : 1; | ||
| } | ||
|
|
||
| void updateQuESTEnvDistInfo() { | ||
| globalEnvPtr->rank = (globalEnvPtr->isDistributed)? comm_getRank() : 0; | ||
| globalEnvPtr->numNodes = (globalEnvPtr->isDistributed)? comm_getNumNodes() : 1; | ||
| return; | ||
| } | ||
|
|
||
|
|
||
| /* | ||
|
|
@@ -187,10 +192,11 @@ void printCompilationInfo() { | |
|
|
||
| print_table( | ||
| "compilation", { | ||
| {"isMpiCompiled", comm_isMpiCompiled()}, | ||
| {"isMpiSubCommunicatorCompiled", comm_isMpiSubCommunicatorCompiled()}, | ||
|
Member
There was a problem hiding this comment. Should abbreviate |
||
| {"isGpuCompiled", gpu_isGpuCompiled()}, | ||
| {"isOmpCompiled", cpu_isOpenmpCompiled()}, | ||
| {"isCuQuantumCompiled", gpu_isCuQuantumCompiled()}, | ||
| }); | ||
| } | ||
|
|
||
|
|
@@ -200,6 +206,7 @@ void printDeploymentInfo() { | |
| print_table( | ||
| "deployment", { | ||
| {"isMpiEnabled", globalEnvPtr->isDistributed}, | ||
| {"doesUserOwnMpi", globalEnvPtr->userOwnsMpi}, | ||
|
Member
There was a problem hiding this comment.
|
||
| {"isGpuEnabled", globalEnvPtr->isGpuAccelerated}, | ||
| {"isOmpEnabled", globalEnvPtr->isMultithreaded}, | ||
| {"isCuQuantumEnabled", globalEnvPtr->isCuQuantumEnabled}, | ||
|
|
@@ -397,13 +404,19 @@ extern "C" { | |
|
|
||
| void initCustomQuESTEnv(int useDistrib, int useGpuAccel, int useMultithread) { | ||
|
|
||
| const bool userOwnsMpi = false; | ||
| validateAndInitCustomQuESTEnv(useDistrib, userOwnsMpi, useGpuAccel, useMultithread, __func__); | ||
| } | ||
|
|
||
|
|
||
| void initCustomMpiQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuAccel, int useMultithread) { | ||
| validateAndInitCustomQuESTEnv(useDistrib, userOwnsMpi, useGpuAccel, useMultithread, __func__); | ||
| } | ||
|
|
||
| void initQuESTEnv() { | ||
|
|
||
| const bool userOwnsMpi = false; | ||
| validateAndInitCustomQuESTEnv(modeflag::USE_AUTO, userOwnsMpi, modeflag::USE_AUTO, modeflag::USE_AUTO, __func__); | ||
| } | ||
|
|
||
|
|
||
|
|
@@ -436,7 +449,7 @@ void finalizeQuESTEnv() { | |
|
|
||
| if (globalEnvPtr->isDistributed) { | ||
| comm_sync(); | ||
| comm_end(globalEnvPtr->userOwnsMpi); | ||
| } | ||
|
|
||
| // free global env's heap memory and flag it as unallocated | ||
|
|
@@ -454,8 +467,12 @@ void syncQuESTEnv() { | |
| if (globalEnvPtr->isGpuAccelerated) | ||
| gpu_sync(); | ||
|
|
||
| if (globalEnvPtr->isDistributed) { | ||
| comm_sync(); | ||
| #if QUEST_COMPILE_SUBCOMM | ||
| updateQuESTEnvDistInfo(); | ||
|
Member
There was a problem hiding this comment. This function needn't exist. It invokes globalEnvPtr->rank = (globalEnvPtr->isDistributed)? comm_getRank() : 0;
globalEnvPtr->numNodes = (globalEnvPtr->isDistributed)? comm_getNumNodes() : 1;which is called from nowhere else; the two lines are clearer than the vagueness of "QuESTEnvDistInfo". Further, the ternary in those lines are dead-code, given this function si called only when But more importantly, I'm unsure of the semantics of this function; why should
Member
There was a problem hiding this comment. @otbrown is this just a dev artefact? (double checking since I've removed in cleanup)
Collaborator
Author
There was a problem hiding this comment. Apologies, yes, there was a time when that call was needed to allow the user to change the subcommunicator after initialisation. That way madness lies however, so that plan was aborted. |
||
| #endif | ||
| } | ||
| } | ||
|
|
||
|
|
||
|
|
@@ -498,10 +515,11 @@ void getQuESTEnvironmentString(char str[200]) { | |
| int cuQuantum = env.isGpuAccelerated && gpu_isCuQuantumCompiled(); | ||
| int gpuDirect = env.isGpuAccelerated && gpu_isDirectGpuCommPossible(); | ||
|
|
||
| snprintf(str, 200, "CUDA=%d OpenMP=%d MPI=%d userOwnsMPI=%d threads=%d ranks=%d cuQuantum=%d gpuDirect=%d", | ||
| env.isGpuAccelerated, | ||
| env.isMultithreaded, | ||
| env.isDistributed, | ||
| env.userOwnsMpi, | ||
| numThreads, | ||
| env.numNodes, | ||
| cuQuantum, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| #include "quest/include/config.h" | ||
| #include "quest/include/environment.h" | ||
| #include "quest/include/subcommunicator.h" | ||
|
|
||
| #include "quest/src/comm/comm_config.hpp" | ||
| #include "quest/src/core/errors.hpp" | ||
|
|
||
| #if QUEST_COMPILE_MPI && QUEST_COMPILE_SUBCOMM | ||
|
|
||
| #include <stdbool.h> | ||
|
Member
There was a problem hiding this comment. Why is the C |
||
| #include <mpi.h> | ||
|
|
||
| void initCustomMpiCommQuESTEnv(MPI_Comm userQuestComm, int useGpuAccel, int useMultithread) { | ||
| // useDistrib and userOwnsMpi are implied by the user of this initialiser | ||
| const int useDistrib = 1; | ||
| const bool userOwnsMpi = true; | ||
|
|
||
| // set mpiCommQuest to user provided communicator | ||
| if (comm_isInit()) { | ||
| comm_setMpiComm(userQuestComm); | ||
| } else { | ||
| error_commNotInit(); | ||
|
Member
There was a problem hiding this comment. This is an internal errror throw to indicate a QuEST bug, not user-input validation. See here |
||
| } | ||
|
|
||
| // initialise QuEST around that communicator | ||
| initCustomMpiQuESTEnv(useDistrib, userOwnsMpi, useGpuAccel, useMultithread); | ||
|
otbrown marked this conversation as resolved.
Member
There was a problem hiding this comment. This function ( |
||
|
|
||
| return; | ||
| } | ||
|
|
||
| #endif | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This should be an
int, or the other fields updated tobool, since they are all on equal footing. The existingis*fields are always set to0or1. Possibly there was confusion with the arguments ofinitCustomQuESTEnv()which accepts similarly named arguments as-1in order for the auto-deployer to override them - that never reaches theQuESTEnv.