Skip to content

Commit

Permalink
some cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
rrsettgast committed Oct 25, 2024
1 parent f50d560 commit bb05c54
Show file tree
Hide file tree
Showing 2 changed files with 34 additions and 40 deletions.
52 changes: 18 additions & 34 deletions src/coreComponents/common/MpiWrapper.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -612,11 +612,11 @@ struct MpiWrapper

/**
* @brief Convenience function for MPI_Gather using a MPI_MAX operation on struct of value and location
* @brief Max is performed on value and location (global indice)
* @brief Max is performed on value and location (global index) is returned
* @param[in] struct to send into the max gather.
* @return struct with max val and location
*/
template< typename valLoc_type > static valLoc_type maxValLoc( valLoc_type l_valLoc, MPI_Comm comm = MPI_COMM_GEOS );
template< typename T > static T maxValLoc( T localValueLocation, MPI_Comm comm = MPI_COMM_GEOS );

};

Expand Down Expand Up @@ -1126,47 +1126,31 @@ void MpiWrapper::reduce( Span< T const > const src, Span< T > const dst, Reducti
}

// Mpi helper function to return struct containing the max value and location across ranks
template< typename valLoc_type >
valLoc_type
MpiWrapper::maxValLoc( valLoc_type l_valLoc, MPI_Comm comm )
template< typename T >
T MpiWrapper::maxValLoc( T localValueLocation, MPI_Comm comm )
{
// Force struct to have only 2 data members
GEOS_ASSERT_EQ((sizeof(valLoc_type::value)+sizeof(valLoc_type::location)), sizeof(valLoc_type));
// Ensure members are not pointers
if constexpr ( std::is_pointer_v< decltype(&valLoc_type::value) >) {
GEOS_ERROR_IF( false, "Attempting to use pointer for maxValLoc comparison" );
}
if constexpr ( std::is_pointer_v< decltype(&valLoc_type::location) >) {
GEOS_ERROR_IF( false, "Attempting to use pointer for maxValLoc comparison" );
}
//These generate compile error
//GEOS_ERROR_IF(constexpr (std::is_pointer_v<decltype(&valLoc_type::value)>),"Attempting to use pointer for maxValLoc comparison" );
//GEOS_ERROR_IF(constexpr (std::is_pointer_v<decltype(&valLoc_type::location)>),"Attempting to use pointer for maxValLoc comparison" );
// Ensure T is trivially copyable
static_assert( std::is_trivially_copyable< T >::value, "maxValLoc requires a trivially copyable type" );

// Serialize to char
unsigned char *sendbuf = new unsigned char[sizeof(valLoc_type)];
std::memcpy( sendbuf, &l_valLoc, sizeof(valLoc_type));
// T to have only 2 data members named value and location
static_assert( (sizeof(T::value)+sizeof(T::location)) == sizeof(T) );

// recieve buffer
int const numProcs = commSize( comm );
unsigned char *recvbuf = new unsigned char[numProcs*sizeof(valLoc_type)];
// Ensure T has value and location members are scalars
static_assert( std::is_scalar_v< decltype(T::value) > || std::is_scalar_v< decltype(T::location) >, "members of struct should be scalar" );
static_assert( !std::is_pointer_v< decltype(T::value) > && !std::is_pointer_v< decltype(T::location) >, "members of struct should not be pointers" );

MPI_Allgather( sendbuf, sizeof(valLoc_type), MPI_BYTE, recvbuf, sizeof(valLoc_type), MPI_BYTE, comm );
// receive "buffer"
int const numProcs = commSize( comm );
std::vector< T > recvValLoc( numProcs );

// Unpack buffer and find max
std::vector< valLoc_type > g_valLoc( numProcs );
for( int i = 0; i< numProcs; i++ )
{
std::memcpy( &g_valLoc[i], &recvbuf[sizeof(valLoc_type)*i], sizeof(valLoc_type));
}
valLoc_type maxValLoc= *max_element( begin( g_valLoc ), end( g_valLoc ), []( auto & lhs, auto & rhs ) -> integer {return lhs.value < rhs.value; } );
MPI_Allgather( &localValueLocation, sizeof(T), MPI_BYTE, recvValLoc.data(), sizeof(T), MPI_BYTE, comm );

delete[] sendbuf;
delete[] recvbuf;
T maxValLoc= *std::max_element( recvValLoc.begin(),
recvValLoc.end(),
[]( auto & lhs, auto & rhs ) -> bool {return lhs.value < rhs.value; } );

Check warning on line 1150 in src/coreComponents/common/MpiWrapper.hpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/common/MpiWrapper.hpp#L1150

Added line #L1150 was not covered by tests

return maxValLoc;
}

} /* namespace geos */

#endif /* GEOS_COMMON_MPIWRAPPER_HPP_ */
Original file line number Diff line number Diff line change
Expand Up @@ -552,10 +552,17 @@ real64 CompositionalMultiphaseFVM::scalingForSystemSolution( DomainPartition & d
minCompDensScalingFactor = MpiWrapper::min( minCompDensScalingFactor );

string const massUnit = m_useMass ? "kg/m3" : "mol/m3";
GEOS_LOG_LEVEL_RANK_0( logInfo::Solution, GEOS_FMT( " {}: Max pressure change = {} Pa (before scaling) at cell {}",
getName(), GEOS_FMT( "{:.{}f}", globalDeltaPresMax.value, 3 ), globalDeltaPresMax.location ) );
GEOS_LOG_LEVEL_RANK_0( logInfo::Solution, GEOS_FMT( " {}: Max component density change = {} {} (before scaling) at cell {}",
getName(), GEOS_FMT( "{:.{}f}", globalDeltaCompDensMax.value, 3 ), massUnit, globalDeltaCompDensMax.location ) );
GEOS_LOG_LEVEL_INFO_RANK_0( logInfo::Solution,
GEOS_FMT( " {}: Max pressure change = {:.3f} Pa (before scaling) at cell {}",
getName(),
globalDeltaPresMax.value,
globalDeltaPresMax.location ) );
GEOS_LOG_LEVEL_INFO_RANK_0( logInfo::Solution,
GEOS_FMT( " {}: Max component density change = {:.3f} {:.3f} (before scaling) at cell {}",
getName(),
globalDeltaCompDensMax.value,
massUnit,
globalDeltaCompDensMax.location ) );

if( m_isThermal )
{
Expand All @@ -565,8 +572,11 @@ real64 CompositionalMultiphaseFVM::scalingForSystemSolution( DomainPartition & d
auto globalMaxDeltaTemp = MpiWrapper::maxValLoc( valueLocType( localDeltaTempMax, localDeltaTempMaxLoc ));

Check warning on line 572 in src/coreComponents/physicsSolvers/fluidFlow/CompositionalMultiphaseFVM.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/physicsSolvers/fluidFlow/CompositionalMultiphaseFVM.cpp#L569-L572

Added lines #L569 - L572 were not covered by tests

minTempScalingFactor = MpiWrapper::min( minTempScalingFactor );
GEOS_LOG_LEVEL_RANK_0( logInfo::Solution, GEOS_FMT( " {}: Max temperature change = {} K (before scaling) at cell maxRegionDeltaTempLoc {}",
getName(), GEOS_FMT( "{:.{}f}", globalMaxDeltaTemp.value, 3 ), globalMaxDeltaTemp.location ) );
GEOS_LOG_LEVEL_INFO_RANK_0( logInfo::Solution,

Check warning on line 575 in src/coreComponents/physicsSolvers/fluidFlow/CompositionalMultiphaseFVM.cpp

View check run for this annotation

Codecov / codecov/patch

src/coreComponents/physicsSolvers/fluidFlow/CompositionalMultiphaseFVM.cpp#L575

Added line #L575 was not covered by tests
GEOS_FMT( " {}: Max temperature change = {:.3f} K (before scaling) at cell maxRegionDeltaTempLoc {}",
getName(),
globalMaxDeltaTemp.value,
globalMaxDeltaTemp.location ) );
}

if( m_scalingType == ScalingType::Local )
Expand Down

0 comments on commit bb05c54

Please sign in to comment.