From 263e5fd4262fa0ff6f460293609cf68e9a57be7f Mon Sep 17 00:00:00 2001 From: Randolph Settgast Date: Wed, 16 Oct 2024 20:56:24 -0700 Subject: [PATCH] Make relationship between MPI_iCommData and CommID unique/one-to-one (#3402) --- .../mesh/mpiCommunications/CommID.hpp | 5 +++-- .../mpiCommunications/CommunicationTools.cpp | 16 ++++++++-------- .../mesh/mpiCommunications/MPI_iCommData.cpp | 5 +++-- .../mesh/mpiCommunications/MPI_iCommData.hpp | 4 ++-- .../SolidMechanicsLagrangianFEM.cpp | 2 +- .../SolidMechanicsLagrangianFEM.hpp | 2 +- .../solidMechanics/SolidMechanicsMPM.cpp | 6 +++++- .../solidMechanics/SolidMechanicsMPM.hpp | 1 - .../EmbeddedSurfacesParallelSynchronization.cpp | 7 +++---- .../surfaceGeneration/ParallelTopologyChange.cpp | 4 ++-- 10 files changed, 28 insertions(+), 24 deletions(-) diff --git a/src/coreComponents/mesh/mpiCommunications/CommID.hpp b/src/coreComponents/mesh/mpiCommunications/CommID.hpp index df56b0ca61..2f8997c1ba 100644 --- a/src/coreComponents/mesh/mpiCommunications/CommID.hpp +++ b/src/coreComponents/mesh/mpiCommunications/CommID.hpp @@ -44,8 +44,6 @@ class CommID */ ~CommID(); - /// default copy constructor - CommID( CommID const & ) = default; /** * Move constructor @@ -53,6 +51,9 @@ class CommID */ CommID( CommID && src ); + /// deleted default copy constructor + CommID( CommID const & ) = delete; + /// deleted copy assignment operator CommID & operator=( CommID const & ) = delete; diff --git a/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp b/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp index fdb2ac4170..87dd288c16 100644 --- a/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp +++ b/src/coreComponents/mesh/mpiCommunications/CommunicationTools.cpp @@ -121,7 +121,7 @@ void CommunicationTools::assignGlobalIndices( ObjectManagerBase & manager, integer const numNeighbors = LvArray::integerConversion< integer >( neighbors.size() ); - MPI_iCommData commData( getCommID() ); + MPI_iCommData commData; commData.resize( numNeighbors ); array1d< int > receiveBufferSizes( numNeighbors ); @@ -340,11 +340,11 @@ CommunicationTools::assignNewGlobalIndices( ElementRegionManager & elementManage * @return The data received from all the @p neighbors. Data at index @p i coming from neighbor at index @p i in the list of @p neighbors. */ template< class DATA_PROVIDER > -array1d< array1d< globalIndex > > exchange( int commId, - std::vector< NeighborCommunicator > & neighbors, +array1d< array1d< globalIndex > > exchange( std::vector< NeighborCommunicator > & neighbors, DATA_PROVIDER const & data ) { - MPI_iCommData commData( commId ); + MPI_iCommData commData; + int commId = commData.commID(); integer const numNeighbors = LvArray::integerConversion< integer >( neighbors.size() ); commData.resize( numNeighbors ); for( integer i = 0; i < numNeighbors; ++i ) @@ -391,7 +391,7 @@ CommunicationTools::buildNeighborPartitionBoundaryObjects( ObjectManagerBase & m { return std::cref( globalPartitionBoundaryObjectsIndices ); }; - array1d< array1d< globalIndex > > const neighborPartitionBoundaryObjects = exchange( getCommID(), allNeighbors, data ); + array1d< array1d< globalIndex > > const neighborPartitionBoundaryObjects = exchange( allNeighbors, data ); integer const numNeighbors = LvArray::integerConversion< integer >( allNeighbors.size() ); for( integer i = 0; i < numNeighbors; ++i ) @@ -566,7 +566,7 @@ void CommunicationTools::findMatchedPartitionBoundaryNodes( NodeManager & nodeMa { return req.at( allNeighbors[i].neighborRank() ); }; - array1d< array1d< globalIndex > > const nodesRequestedByNeighbors = exchange( getCommID(), allNeighbors, data ); + array1d< array1d< globalIndex > > const nodesRequestedByNeighbors = exchange( allNeighbors, data ); // Then we store the requested nodes for each receiver. for( integer i = 0; i < numNeighbors; ++i ) @@ -804,7 +804,7 @@ void CommunicationTools::setupGhosts( MeshLevel & meshLevel, bool const unorderedComms ) { GEOS_MARK_FUNCTION; - MPI_iCommData commData( getCommID() ); + MPI_iCommData commData; commData.resize( neighbors.size() ); NodeManager & nodeManager = meshLevel.getNodeManager(); @@ -1092,7 +1092,7 @@ void CommunicationTools::synchronizeFields( FieldIdentifiers const & fieldsToBeS std::vector< NeighborCommunicator > & neighbors, bool onDevice ) { - MPI_iCommData icomm( getCommID() ); + MPI_iCommData icomm; icomm.resize( neighbors.size() ); synchronizePackSendRecvSizes( fieldsToBeSync, mesh, neighbors, icomm, onDevice ); synchronizePackSendRecv( fieldsToBeSync, mesh, neighbors, icomm, onDevice ); diff --git a/src/coreComponents/mesh/mpiCommunications/MPI_iCommData.cpp b/src/coreComponents/mesh/mpiCommunications/MPI_iCommData.cpp index 55609a0e0f..2e0d792043 100644 --- a/src/coreComponents/mesh/mpiCommunications/MPI_iCommData.cpp +++ b/src/coreComponents/mesh/mpiCommunications/MPI_iCommData.cpp @@ -14,13 +14,14 @@ */ #include "MPI_iCommData.hpp" +#include "CommunicationTools.hpp" namespace geos { -MPI_iCommData::MPI_iCommData( int const inputCommID ): +MPI_iCommData::MPI_iCommData(): m_size( 0 ), - m_commID( inputCommID ), // CommunicationTools::getInstance().getCommID() ), + m_commID( CommunicationTools::getInstance().getCommID() ), m_mpiSendBufferRequest(), m_mpiRecvBufferRequest(), m_mpiSendBufferStatus(), diff --git a/src/coreComponents/mesh/mpiCommunications/MPI_iCommData.hpp b/src/coreComponents/mesh/mpiCommunications/MPI_iCommData.hpp index e749a81872..24f5e8cdba 100644 --- a/src/coreComponents/mesh/mpiCommunications/MPI_iCommData.hpp +++ b/src/coreComponents/mesh/mpiCommunications/MPI_iCommData.hpp @@ -36,7 +36,7 @@ class MPI_iCommData * @param inputCommID The CommID integer that indicates what communication * pipeline to use for a set of neighbor communications. */ - MPI_iCommData( int const inputCommID ); + MPI_iCommData(); /// Default destructor ~MPI_iCommData(); @@ -96,7 +96,7 @@ class MPI_iCommData int m_size; /// The integer ID for the set of communication pipelines - int m_commID; + CommID m_commID; /// A collection of field names keyed on object keys to pack/unpack from /// communication pipeline. diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp index 0ad7c46f77..1c1c5cf978 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.cpp @@ -59,7 +59,6 @@ SolidMechanicsLagrangianFEM::SolidMechanicsLagrangianFEM( const string & name, m_maxForce( 0.0 ), m_maxNumResolves( 10 ), m_strainTheory( 0 ), - m_iComm( CommunicationTools::getInstance().getCommID() ), m_isFixedStressPoromechanicsUpdate( false ) { @@ -571,6 +570,7 @@ real64 SolidMechanicsLagrangianFEM::explicitStep( real64 const & time_n, solidMechanics::arrayView2dLayoutIncrDisplacement const & uhat = nodes.getField< solidMechanics::incrementalDisplacement >(); solidMechanics::arrayView2dLayoutAcceleration const & acc = nodes.getField< solidMechanics::acceleration >(); + MPI_iCommData m_iComm; FieldIdentifiers fieldsToBeSync; fieldsToBeSync.addFields( FieldLocation::Node, { solidMechanics::velocity::key(), diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.hpp b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.hpp index c59227142b..3106018022 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.hpp +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsLagrangianFEM.hpp @@ -294,7 +294,7 @@ class SolidMechanicsLagrangianFEM : public SolverBase real64 m_maxForce = 0.0; integer m_maxNumResolves; integer m_strainTheory; - MPI_iCommData m_iComm; +// MPI_iCommData m_iComm; bool m_isFixedStressPoromechanicsUpdate; /// Rigid body modes diff --git a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp index b0cb8c16a2..a08777484f 100644 --- a/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp +++ b/src/coreComponents/physicsSolvers/solidMechanics/SolidMechanicsMPM.cpp @@ -58,7 +58,7 @@ SolidMechanicsMPM::SolidMechanicsMPM( const string & name, SolverBase( name, parent ), m_solverProfiling( 0 ), m_timeIntegrationOption( TimeIntegrationOption::ExplicitDynamic ), - m_iComm( CommunicationTools::getInstance().getCommID() ), +// m_iComm( CommunicationTools::getInstance().getCommID() ), m_prescribedBcTable( 0 ), m_boundaryConditionTypes(), m_bcTable(), @@ -940,6 +940,7 @@ real64 SolidMechanicsMPM::explicitStep( real64 const & time_n, MeshLevel & mesh = grid.getBaseDiscretization(); NodeManager & nodeManager = mesh.getNodeManager(); + MPI_iCommData m_iComm; m_iComm.resize( domain.getNeighbors().size() ); @@ -1258,6 +1259,9 @@ void SolidMechanicsMPM::syncGridFields( std::vector< std::string > const & field fieldsToBeSynced.addFields( FieldLocation::Node, fieldNames ); std::vector< NeighborCommunicator > & neighbors = domain.getNeighbors(); + MPI_iCommData m_iComm; + m_iComm.resize( neighbors.size() ); + // (2) Swap send and receive indices so we can sum from ghost to master for( size_t n=0; n m_profilingLabels; TimeIntegrationOption m_timeIntegrationOption; - MPI_iCommData m_iComm; int m_prescribedBcTable; array1d< int > m_boundaryConditionTypes; // TODO: Surely there's a way to have just one variable here diff --git a/src/coreComponents/physicsSolvers/surfaceGeneration/EmbeddedSurfacesParallelSynchronization.cpp b/src/coreComponents/physicsSolvers/surfaceGeneration/EmbeddedSurfacesParallelSynchronization.cpp index df8d00264e..3e41eb295f 100644 --- a/src/coreComponents/physicsSolvers/surfaceGeneration/EmbeddedSurfacesParallelSynchronization.cpp +++ b/src/coreComponents/physicsSolvers/surfaceGeneration/EmbeddedSurfacesParallelSynchronization.cpp @@ -383,7 +383,7 @@ void synchronizeNewNodes( MeshLevel & mesh, //************************************************************************************************ // We need to send over the new embedded surfaces and related objects for those whose parents are ghosts on neighbors. - MPI_iCommData commData( CommunicationTools::getInstance().getCommID() ); + MPI_iCommData commData; commData.resize( neighbors.size()); for( unsigned int neighborIndex=0; neighborIndex & neighbors, string const fractureRegionName ) { - MPI_iCommData commDataJunk( CommunicationTools::getInstance().getCommID() ); - MPI_iCommData commData( CommunicationTools::getInstance().getCommID() ); + MPI_iCommData commData; commData.resize( neighbors.size()); for( unsigned int neighborIndex=0; neighborIndex