From e5bb4918536a58bb13a3930c8b99babb7fa519d9 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 3 Sep 2018 17:49:42 +0200 Subject: [PATCH 1/8] Lulesh: Properly initialize communication control variables --- lulesh/lulesh-comm-dash-onesided.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lulesh/lulesh-comm-dash-onesided.cc b/lulesh/lulesh-comm-dash-onesided.cc index 74c6aea..88ef833 100644 --- a/lulesh/lulesh-comm-dash-onesided.cc +++ b/lulesh/lulesh-comm-dash-onesided.cc @@ -54,16 +54,13 @@ void DASHCommPut(Domain& domain, DASHComm& comm, { Real_t *destAddr; bool rowNotMin, rowNotMax, colNotMin, colNotMax, planeNotMin, planeNotMax; - - Index_t maxPlaneComm = xferFields * domain.maxPlaneSize(); - Index_t maxEdgeComm = xferFields * domain.maxEdgeSize(); + // assume communication to 6 neighbors by default + rowNotMin = rowNotMax = colNotMin = colNotMax = planeNotMin = planeNotMax = true; Index_t pmsg = 0; // plane comm msg Index_t emsg = 0; // edge comm msg Index_t cmsg = 0; // corner comm msg - // assume communication to 6 neighbors by default - rowNotMin = rowNotMax = colNotMin = colNotMax = planeNotMin = planeNotMax = true; if( domain.rowLoc() == 0 ) { rowNotMin = false; } if( domain.rowLoc() == (domain.tp()-1) ) { rowNotMax = false; } @@ -1085,6 +1082,8 @@ void DASHCommSyncPosVel(Domain& domain, DASHComm& comm) //MPI_Status status; Real_t *srcAddr; bool rowNotMin, rowNotMax, colNotMin, colNotMax, planeNotMin, planeNotMax; + // assume communication to 6 neighbors by default + rowNotMin = rowNotMax = colNotMin = colNotMax = planeNotMin = planeNotMax = true; /* assume communication to 6 neighbors by default */ if( domain.rowLoc() == 0 ) { rowNotMin = false; } @@ -1431,6 +1430,8 @@ void DASHCommMonoQ(Domain& domain, DASHComm& comm) //MPI_Status status; Real_t *srcAddr; bool rowNotMin, rowNotMax, colNotMin, colNotMax, planeNotMin, planeNotMax; + // assume communication to 6 neighbors by default + rowNotMin = rowNotMax = colNotMin = colNotMax = planeNotMin = planeNotMax = true; /* assume communication to 6 neighbors by default */ if( domain.rowLoc() == 0 ) { rowNotMin = false; } From 0f33fe12e549fff18f54363abb49a96a0d062f8c Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 3 Sep 2018 17:50:13 +0200 Subject: [PATCH 2/8] Lulesh: barrier after initialization --- lulesh/lulesh-comm-dash.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lulesh/lulesh-comm-dash.cc b/lulesh/lulesh-comm-dash.cc index d3b49e2..7e590d8 100644 --- a/lulesh/lulesh-comm-dash.cc +++ b/lulesh/lulesh-comm-dash.cc @@ -30,6 +30,8 @@ DASHComm::DASHComm(Domain& dom) : m_dom(dom) memset( m_commDataSend->lbegin(), 0, comBufSize*sizeof(Real_t) ); memset( m_commDataRecv->lbegin(), 0, comBufSize*sizeof(Real_t) ); + + m_commDataRecv->barrier(); } DASHComm::~DASHComm() From 5ddc98c112762ddb8e44654ec8695dfe507b5fbd Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 3 Sep 2018 17:57:55 +0200 Subject: [PATCH 3/8] Lulesh: re-add missing max*Comm variables --- lulesh/lulesh-comm-dash-onesided.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lulesh/lulesh-comm-dash-onesided.cc b/lulesh/lulesh-comm-dash-onesided.cc index 88ef833..403bda9 100644 --- a/lulesh/lulesh-comm-dash-onesided.cc +++ b/lulesh/lulesh-comm-dash-onesided.cc @@ -57,6 +57,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, // assume communication to 6 neighbors by default rowNotMin = rowNotMax = colNotMin = colNotMax = planeNotMin = planeNotMax = true; + Index_t maxPlaneComm = xferFields * domain.maxPlaneSize(); + Index_t maxEdgeComm = xferFields * domain.maxEdgeSize(); + Index_t pmsg = 0; // plane comm msg Index_t emsg = 0; // edge comm msg Index_t cmsg = 0; // corner comm msg From 93df843e7314feb2b4d186225764fcf8cae913f1 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Tue, 4 Sep 2018 11:30:33 +0200 Subject: [PATCH 4/8] Lulesh: Add required synchronization to avoid overwriting data --- lulesh/lulesh-comm-dash.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lulesh/lulesh-comm-dash.cc b/lulesh/lulesh-comm-dash.cc index 7e590d8..db16a34 100644 --- a/lulesh/lulesh-comm-dash.cc +++ b/lulesh/lulesh-comm-dash.cc @@ -61,6 +61,7 @@ void DASHComm::ExchangeNodalMass() dash::barrier(); DASHCommSBN(dom, comm, 1, &fieldData); + dash::barrier(); } void DASHComm::Recv_PosVel() @@ -100,6 +101,7 @@ void DASHComm::Sync_PosVel() dash::barrier(); DASHCommSyncPosVel(dom, comm); + dash::barrier(); } @@ -139,6 +141,7 @@ void DASHComm::Sync_Force() dash::barrier(); DASHCommSBN(dom, comm, 3, fieldData); + dash::barrier(); } void DASHComm::Recv_MonoQ() @@ -176,6 +179,7 @@ void DASHComm::Sync_MonoQ() dash::barrier(); DASHCommMonoQ(dom,comm); + dash::barrier(); } Int_t DASHComm::offset(Int_t desc) From 28443dc4653a8dd688aaf2ee3826bbbebe1d7db4 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Tue, 4 Sep 2018 11:50:39 +0200 Subject: [PATCH 5/8] Lulesh: Fix offset computations --- lulesh/lulesh-comm-dash-onesided.cc | 165 +++++++++------------------- lulesh/lulesh-comm-dash.cc | 4 +- 2 files changed, 56 insertions(+), 113 deletions(-) diff --git a/lulesh/lulesh-comm-dash-onesided.cc b/lulesh/lulesh-comm-dash-onesided.cc index 403bda9..a97be45 100644 --- a/lulesh/lulesh-comm-dash-onesided.cc +++ b/lulesh/lulesh-comm-dash-onesided.cc @@ -60,9 +60,7 @@ void DASHCommPut(Domain& domain, DASHComm& comm, Index_t maxPlaneComm = xferFields * domain.maxPlaneSize(); Index_t maxEdgeComm = xferFields * domain.maxEdgeSize(); - Index_t pmsg = 0; // plane comm msg - Index_t emsg = 0; // edge comm msg - Index_t cmsg = 0; // corner comm msg + Index_t futcntr = 0; if( domain.rowLoc() == 0 ) { rowNotMin = false; } @@ -79,7 +77,7 @@ void DASHCommPut(Domain& domain, DASHComm& comm, int sendCount = dx * dy; if (planeNotMin) { - destAddr = &comm.commDataSend()[pmsg * maxPlaneComm]; + destAddr = &comm.commDataSend()[comm.offset(Z1)]; for (Index_t fi=0; fi Date: Tue, 4 Sep 2018 11:56:45 +0200 Subject: [PATCH 6/8] Lulesh: add possible buffer dump to debug output --- lulesh/lulesh-comm-dash-onesided.cc | 251 ++++++++++++++++++++++++---- 1 file changed, 217 insertions(+), 34 deletions(-) diff --git a/lulesh/lulesh-comm-dash-onesided.cc b/lulesh/lulesh-comm-dash-onesided.cc index a97be45..7f72d60 100644 --- a/lulesh/lulesh-comm-dash-onesided.cc +++ b/lulesh/lulesh-comm-dash-onesided.cc @@ -37,16 +37,45 @@ char tagname[][8] = "X1Y1Z1 ", }; +#ifdef WITH_DEBUG +static void +dump_buffer(Real_t *buf, size_t nelem) +{ + for (size_t i = 0; i < nelem; ++i) { + std::cout << std::fixed << std::setprecision(5) << std::setw(3) << buf[i] << " "; + } + std::cout << std::endl; +} // debug output for syncs performed below -void DBGSYNC(int fields, int elem, int tag) +void DBGSYNC_(const char *fname, Real_t *buf, int fields, int elem, int tag) { - /* - cout << "DBGSYNC [" << dash::myid() << "] " << elem - << " " << tagname[tag] << endl; - */ + cout << "DBGSYNC [" << dash::myid() << "] " << fields << "x" << elem << " (" + << fields*elem << ") " << tagname[tag] << " in " << fname << endl; + dump_buffer(buf, fields*elem); } +#define DBGSYNC(__buf, __fields, __elem, __tag) DBGSYNC_(__func__, __buf, __fields, __elem, __tag) + +// debug output for syncs performed below +void DBGPUT_(const char *fname, const dash::GlobIter > src, + Real_t *buf, int fields, int elem, int tag) +{ + cout << "DBGPUT [" << dash::myid() << "] " << fields << "x" << elem << " (" + << fields*elem << ") " << tagname[tag] << " into " << src.dart_gptr() + << " in " << fname << endl; + dump_buffer(buf, fields*elem); +} + +#define DBGPUT(__src, __buf, __fields, __elem, __tag) DBGPUT_(__func__, __src, __buf, __fields, __elem, __tag) + +#else + +#define DBGSYNC(...) +#define DBGPUT(...) + +#endif // WITH_DEBUG + void DASHCommPut(Domain& domain, DASHComm& comm, Index_t xferFields, Domain_member *fieldData, Index_t dx, Index_t dy, Index_t dz, @@ -87,6 +116,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*sendCount; + DBGPUT(comm.dest(myRank - (domain.tp() * domain.tp()), Z1), + destAddr, xferFields, sendCount, Z1); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, @@ -110,6 +142,10 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*sendCount; + DBGPUT(comm.dest(myRank + (domain.tp() * domain.tp()), Z0), + destAddr, xferFields, sendCount, Z0); + + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, @@ -139,6 +175,10 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*sendCount; + DBGPUT(comm.dest(myRank - domain.tp(), Y1), + destAddr, xferFields, sendCount, Y1); + + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, @@ -163,6 +203,10 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*sendCount; + DBGPUT(comm.dest(myRank + domain.tp(), Y0), + destAddr, xferFields, sendCount, Y0); + + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*sendCount, @@ -192,6 +236,10 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*sendCount; + DBGPUT(comm.dest(myRank - 1, X1), + destAddr, xferFields, sendCount, X1); + + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*sendCount, @@ -215,6 +263,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*sendCount; + DBGPUT(comm.dest(myRank + 1, X0), + destAddr, xferFields, sendCount, X0); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*sendCount, @@ -240,6 +291,10 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dz; + DBGPUT(comm.dest(toRank, X1Y1), + destAddr, xferFields, dz, X1Y1); + + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dz, @@ -263,6 +318,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dx; + DBGPUT(comm.dest(toRank, Y1Z1), + destAddr, xferFields, dx, Y1Z1); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dx, @@ -285,6 +343,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dy; + DBGPUT(comm.dest(toRank, X1Z1), + destAddr, xferFields, dy, X1Z1); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dy, @@ -307,6 +368,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dz; + DBGPUT(comm.dest(toRank, X0Y0), + destAddr, xferFields, dz, X0Y0); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dz, @@ -329,6 +393,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dx; + DBGPUT(comm.dest(toRank, Y0Z0), + destAddr, xferFields, dx, Y0Z0); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dx, @@ -351,6 +418,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dy; + DBGPUT(comm.dest(toRank, X0Z0), + destAddr, xferFields, dy, X0Z0); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dy, @@ -373,6 +443,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dz; + DBGPUT(comm.dest(toRank, X1Y0), + destAddr, xferFields, dz, X1Y0); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dz, @@ -395,6 +468,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dx; + DBGPUT(comm.dest(toRank, Y1Z0), + destAddr, xferFields, dx, Y1Z0); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dx, @@ -417,6 +493,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dy; + DBGPUT(comm.dest(toRank, X1Z0), + destAddr, xferFields, dy, X1Z0); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dy, @@ -439,6 +518,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dz; + DBGPUT(comm.dest(toRank, X0Y1), + destAddr, xferFields, dz, X0Y1); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dz, @@ -461,6 +543,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dx; + DBGPUT(comm.dest(toRank, Y0Z1), + destAddr, xferFields, dx, Y0Z1); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dx, @@ -484,6 +569,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, } destAddr -= xferFields*dy; + DBGPUT(comm.dest(toRank, X0Z1), + destAddr, xferFields, dy, X0Z1); + comm.sendRequest[futcntr++] = dash::copy_async( destAddr, destAddr+xferFields*dy, @@ -503,6 +591,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, comBuf[fi] = (domain.*fieldData[fi])(0); } + DBGPUT(comm.dest(toRank, X1Y1Z1), + comBuf, xferFields, 1, X1Y1Z1); + comm.sendRequest[futcntr++] = dash::copy_async( comBuf, comBuf+xferFields, @@ -521,6 +612,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, comBuf[fi] = (domain.*fieldData[fi])(idx); } + DBGPUT(comm.dest(toRank, X1Y1Z0), + comBuf, xferFields, 1, X1Y1Z0); + comm.sendRequest[futcntr++] = dash::copy_async( comBuf, comBuf+xferFields, @@ -539,6 +633,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, comBuf[fi] = (domain.*fieldData[fi])(idx); } + DBGPUT(comm.dest(toRank, X0Y1Z1), + comBuf, xferFields, 1, X0Y1Z1); + comm.sendRequest[futcntr++] = dash::copy_async( comBuf, comBuf+xferFields, @@ -557,6 +654,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, comBuf[fi] = (domain.*fieldData[fi])(idx); } + DBGPUT(comm.dest(toRank, X0Y1Z0), + comBuf, xferFields, 1, X0Y1Z0); + comm.sendRequest[futcntr++] = dash::copy_async( comBuf, comBuf+xferFields, @@ -575,6 +675,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, comBuf[fi] = (domain.*fieldData[fi])(idx); } + DBGPUT(comm.dest(toRank, X1Y0Z1), + comBuf, xferFields, 1, X1Y0Z1); + comm.sendRequest[futcntr++] = dash::copy_async( comBuf, comBuf+xferFields, @@ -593,6 +696,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, comBuf[fi] = (domain.*fieldData[fi])(idx); } + DBGPUT(comm.dest(toRank, X1Y0Z0), + comBuf, xferFields, 1, X1Y0Z0); + comm.sendRequest[futcntr++] = dash::copy_async( comBuf, comBuf+xferFields, @@ -611,6 +717,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, comBuf[fi] = (domain.*fieldData[fi])(idx); } + DBGPUT(comm.dest(toRank, X0Y0Z1), + comBuf, xferFields, 1, X0Y0Z1); + comm.sendRequest[futcntr++] = dash::copy_async( comBuf, comBuf+xferFields, @@ -629,6 +738,9 @@ void DASHCommPut(Domain& domain, DASHComm& comm, comBuf[fi] = (domain.*fieldData[fi])(idx); } + DBGPUT(comm.dest(toRank, X0Y0Z0), + comBuf, xferFields, 1, X0Y0Z0); + comm.sendRequest[futcntr++] = dash::copy_async( comBuf, comBuf+xferFields, @@ -656,7 +768,6 @@ void DASHCommSBN(Domain& domain, DASHComm& comm, int xferFields, Index_t dy = domain.sizeY() + 1; Index_t dz = domain.sizeZ() + 1; - Real_t *srcAddr; Index_t rowNotMin, rowNotMax, colNotMin, colNotMax, planeNotMin, planeNotMax; // assume communication to 6 neighbors by default @@ -677,9 +788,10 @@ void DASHCommSBN(Domain& domain, DASHComm& comm, int xferFields, if (planeNotMin) { /* contiguous memory */ + Real_t *srcAddr; srcAddr = &comm.commDataRecv()[comm.offset(Z0)]; //MPI_Wait(&comm.recvRequest[pmsg], &status); - DBGSYNC(xferFields, opCount, Z0); + DBGSYNC(srcAddr, xferFields, opCount, Z0); for (Index_t fi=0; fi Date: Tue, 4 Sep 2018 11:57:06 +0200 Subject: [PATCH 7/8] Lulesh: Use dart_abort instead of exit() in case of error --- lulesh/lulesh-dash.cc | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/lulesh/lulesh-dash.cc b/lulesh/lulesh-dash.cc index 70b93ce..efef4fc 100644 --- a/lulesh/lulesh-dash.cc +++ b/lulesh/lulesh-dash.cc @@ -590,12 +590,14 @@ void Domain::SetupThreadSupportStructures() for (Index_t i=0; i < clSize; ++i) { Index_t clv = m_nodeElemCornerList[i] ; if ((clv < 0) || (clv > numElem()*8)) { - fprintf(stderr, - "AllocateNodeElemIndexes(): nodeElemCornerList entry out of range!\n"); + fprintf(stderr, + "AllocateNodeElemIndexes(): nodeElemCornerList entry out of range!\n"); #if USE_MPI - MPI_Abort(MPI_COMM_WORLD, -1); + MPI_Abort(MPI_COMM_WORLD, -1); +#elif USE_DASH + dart_abort(-1); #else - exit(-1); + exit(-1); #endif } } @@ -869,6 +871,8 @@ void Domain::CalcLagrangeElements(Real_t* vnew) { #if USE_MPI MPI_Abort(MPI_COMM_WORLD, VolumeError) ; +#elif USE_DASH + dart_abort(VolumeError); #else exit(VolumeError); #endif @@ -919,9 +923,11 @@ void Domain::CalcQForElems(Real_t vnew[]) if(idx >= 0) { #if USE_MPI - MPI_Abort(MPI_COMM_WORLD, QStopError) ; + MPI_Abort(MPI_COMM_WORLD, QStopError) ; +#elif USE_DASH + dart_abort(QStopError); #else - exit(QStopError); + exit(QStopError); #endif } } @@ -972,9 +978,11 @@ void Domain::ApplyMaterialPropertiesForElems(Real_t vnew[]) } if (vc <= 0.) { #if USE_MPI - MPI_Abort(MPI_COMM_WORLD, VolumeError) ; + MPI_Abort(MPI_COMM_WORLD, VolumeError) ; +#elif USE_DASH + dart_abort(VolumeError); #else - exit(VolumeError); + exit(VolumeError); #endif } } From 549f06dd29e0c087c9aa2840d18558e433a610f3 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Wed, 5 Sep 2018 13:33:43 +0200 Subject: [PATCH 8/8] Lulesh: Remove unneeded variables from communication functions --- lulesh/lulesh-comm-dash-onesided.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/lulesh/lulesh-comm-dash-onesided.cc b/lulesh/lulesh-comm-dash-onesided.cc index 7f72d60..b4243fc 100644 --- a/lulesh/lulesh-comm-dash-onesided.cc +++ b/lulesh/lulesh-comm-dash-onesided.cc @@ -86,9 +86,6 @@ void DASHCommPut(Domain& domain, DASHComm& comm, // assume communication to 6 neighbors by default rowNotMin = rowNotMax = colNotMin = colNotMax = planeNotMin = planeNotMax = true; - Index_t maxPlaneComm = xferFields * domain.maxPlaneSize(); - Index_t maxEdgeComm = xferFields * domain.maxEdgeSize(); - Index_t futcntr = 0;