34 #define DMSG( LVL , MSG ) { if ( verbose > LVL ) { G4cout << MSG << G4endl; } }
43 inline void Pack(
void*
buffer,
int bufferSize,
int*
position , MPI::Intracomm& comm )
const
45 DMSG(4,
"Packing G4StatDouble(n,scale,sum_w,sum_w2,sum_wx,sum_wx2): "
46 <<m_n<<
" "<<m_scale<<
" "<<m_sum_w<<
" "<<m_sum_w2
47 <<
" "<<m_sum_wx<<
" "<<m_sum_wx2);
52 inline void UnPack(
void* buffer,
int bufferSize,
int* position , MPI::Intracomm& comm ) {
61 DMSG(4,
"UnPacking G4StatDouble(n,scale,sum_w,sum_w2,sum_wx,sum_wx2): "
62 <<m_n<<
" "<<m_scale<<
" "<<m_sum_w<<
" "<<m_sum_w2
63 <<
" "<<m_sum_wx<<
" "<<m_sum_wx2);
65 MPIStatDouble(
G4int ver = 0) : verbose(ver) {}
74 outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
75 ownsBuffer(false),scoringManager(nullptr),commSize(0),
82 outputBuffer(nullptr),outputBufferSize(0),outputBufferPosition(0),bytesSent(0),
84 scoringManager(mgr), commSize(0), destinationRank(destination),
124 DMSG(0,
"G4MPIscorerMerger::Merge called");
125 const unsigned int myrank = MPI::COMM_WORLD.Get_rank();
126 commSize = MPI::COMM_WORLD.Get_size();
128 DMSG(1,
"Comm world size is 1, nothing to do");
131 comm = MPI::COMM_WORLD.Dup();
150 const G4double sttime = MPI::Wtime();
153 typedef std::function<void(unsigned int)> handler_t;
154 using std::placeholders::_1;
157 std::function<void(void)> barrier = std::bind(&MPI::Intracomm::Barrier,&
comm);
176 const G4double elapsed = MPI::Wtime() - sttime;
181 G4cout<<
"G4MPIscorerMerger::Merge() -data transfer performances: "
182 <<double(total)/1000./elapsed<<
" kB/s"
183 <<
" (Total Data Transfer= "<<double(total)/1000.<<
" kB in "
184 <<elapsed<<
" s)."<<
G4endl;
201 DMSG(0,
"G4MPIscorerMerger::Merge done.");
205 DMSG(1,
"Receiving scorers");
207 DMSG(2,
"Receiving from: "<<source);
210 const G4int newbuffsize = status.Get_count(MPI::PACKED);
211 DMSG(2,
"Preparing to receive buffer of size: "<<newbuffsize);
214 DMSG(3,
"New larger buffer expected, resize");
218 buffer =
new char[newbuffsize];
221 std::fill( buffer , buffer + newbuffsize , 0 );
225 comm.Recv(buffer, newbuffsize, MPI::PACKED, source,
229 DMSG(1,
"Receiving of comamnd line scorers done");
233 DMSG(1,
"Sending scorers "<<
this);
240 buffer =
new char[newbuffsize];
243 std::fill( buffer , buffer+newbuffsize,0);
255 DMSG(1,
"Sending done");
261 G4Exception(
"G4MPIscorerMerger::Pack(const G4ScoringManager*)",
263 "Call SetOututBuffer before trying to pack");
268 MPI_Pack(&numMeshes,1,MPI::UNSIGNED,
272 for (
size_t i = 0; i <numMeshes; ++i)
284 G4Exception(
"G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
286 "Call SetOututBuffer before trying to un-pack");
291 &numMeshes,1,MPI::UNSIGNED,
comm);
294 msg <<
"Number of meshes to unpack ("<<numMeshes;
295 msg <<
") does not correspond to expected number ("<<sm->
GetNumberOfMesh();
297 G4Exception(
"G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
303 for (
size_t i = 0 ; i < numMeshes ; ++i ) {
305 &meshid,1,MPI::UNSIGNED,
comm);
308 msg<<
"Cannot unpack: expecting mesh "<<i<<
" and found "<<meshid;
309 msg<<
" during unpack.";
310 G4Exception(
"G4MPIscorerMerger::UnPack(const G4ScroingManager*)",
320 assert(mesh!=
nullptr);
323 DMSG(3,
"Packing mesh: "<<mesh);
326 size_t nummaps = map.size();
330 for (
const auto&
ele: map ) {
332 size_t ss = name.size();
336 #ifdef G4MPI_USE_MPI_PACK_NOT_CONST
337 char*
nn =
new char[name.length()];
338 std::copy(name.begin(),name.end(),
nn);
340 const char* nn = name.c_str();
344 #ifdef G4MPI_USE_MPI_PACK_NOT_CONST
353 assert(inmesh!=
nullptr);
354 DMSG(3,
"Preparing to unpack a mesh and merge into: "<<inmesh);
358 &nummaps,1,MPI::UNSIGNED,
comm);
359 for (
size_t i = 0 ; i < nummaps ; ++i ) {
362 &nameSize,1,MPI::UNSIGNED,
comm);
365 char*
name =
new char[nameSize+1];
366 std::fill(name,name+nameSize+1,0);
368 name,nameSize,MPI::CHAR,
comm);
369 const G4String colname(name,nameSize);
411 DMSG(3,
"Packing hitmap: "<<sm<<
" with: "<<sm->
GetSize()<<
" elements.");
416 const auto& theMap = *sm->
GetMap();
417 std::vector<G4int> ids;
418 std::transform(theMap.begin(),theMap.end(),std::back_inserter(ids),
419 [](decltype(*theMap.begin())&
e){
return e.first;});
420 assert(ids.size()==numEl);
423 for(
const auto&
e : theMap) {
424 const MPIStatDouble sd(*
e.second,
verbose);
425 sd.Pack(outputBuffer,outputBufferSize,&outputBufferPosition,
comm);
455 DMSG(3,
"Preparing to unpack a hit map for: "<<detName<<
","<<colName);
458 &numEl,1,MPI::UNSIGNED,
comm);
459 DMSG(3,
"Will receive "<<numEl<<
" values");
464 for (
unsigned int i = 0; i<numEl;++i) {
467 result->
set(ids[i],sd);
476 DMSG(3,
"Calculating dimension of data to send");
477 if ( sm ==
nullptr )
return 0;
487 size +=
sizeof(
unsigned int);
495 DMSG(3,
"Calculating size for mesh: "<<mesh);
499 for (
const auto&
ele : map ) {
501 size +=
sizeof(
unsigned int);
503 size +=
sizeof(char)*name.size();
506 DMSG(3,
"mesh "<<mesh<<
" size: "<<size);
522 size +=
sizeof(
G4int)*numEls;
526 DMSG(3,
"HitStatDoubleMap "<<map<<
" size: "<<size<<
" in "<<numEls<<
" elements.");
MeshScoreMap GetScoreMap() const
G4double total(Particle const *const p1, Particle const *const p2)
int MPI_Unpack(const void *, int, int *, void *, int, MPI_Datatype, MPI_Comm)
std::ostringstream G4ExceptionDescription
size_t GetNumberOfMesh() const
G4THitsMap< G4StatDouble > HitStatDoubleMap
const G4String & GetWorldName() const
void Accumulate(G4THitsMap< G4double > *map)
G4int CalculatePackSize(const G4ScoringManager *) const
void Pack(const G4ScoringManager *)
Pack all meshes into buffer.
HitStatDoubleMap * UnPackHitStatDoubleMap(const G4String &detName, const G4String &colName)
void Send(const unsigned int destination)
const XML_Char const XML_Char * data
unsigned int destinationRank
int MPI_Pack(const void *, int, MPI_Datatype, void *, int, int *, MPI_Comm)
typedef int(XMLCALL *XML_NotStandaloneHandler)(void *userData)
void Merge(std::function< void(unsigned int)> senderF, std::function< void(unsigned int)> receiverF, std::function< void(void)> barrierF, unsigned int commSize, unsigned int myrank)
virtual ~G4MPIscorerMerger()
void SetupOutputBuffer(char *buff, G4int size, G4int position)
G4double G4ParticleHPJENDLHEData::G4double result
G4ScoringManager * scoringManager
void G4Exception(const char *originOfException, const char *exceptionCode, G4ExceptionSeverity severity, const char *description)
virtual size_t GetSize() const
G4int outputBufferPosition
void UnPackAndMerge(const G4ScoringManager *)
static G4String Status(G4StepStatus stps)
G4int set(const G4int &key, U *&aHit) const
G4GLOB_DLL std::ostream G4cout
G4StatDouble & operator=(const G4double &rhs)
G4VScoringMesh * GetMesh(G4int i) const
void Receive(const unsigned int source)
static const G4double ele
std::map< G4String, RunScore * > MeshScoreMap