From bc112a659722633bbf8c954ea4e457baa788b052 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 18 Apr 2014 12:25:50 +0100 Subject: [PATCH 01/99] Add basic wrapping of asynchronous MPI calls --- Code/extraction/LocalPropertyOutput.cc | 2 +- .../decomposition/OptimisedDecomposition.cc | 5 +- Code/net/CMakeLists.txt | 1 + Code/net/MpiCommunicator.h | 25 ++++- Code/net/MpiCommunicator.hpp | 72 ++++++++++-- Code/net/MpiRequest.cc | 106 ++++++++++++++++++ Code/net/MpiRequest.h | 57 ++++++++++ Code/net/MpiStatus.cc | 25 +++++ Code/net/MpiStatus.h | 44 ++++++++ Code/net/mpi.h | 2 + 10 files changed, 322 insertions(+), 17 deletions(-) create mode 100644 Code/net/MpiRequest.cc create mode 100644 Code/net/MpiRequest.h create mode 100644 Code/net/MpiStatus.cc create mode 100644 Code/net/MpiStatus.h diff --git a/Code/extraction/LocalPropertyOutput.cc b/Code/extraction/LocalPropertyOutput.cc index 2649e5fec..fea0bbb4d 100644 --- a/Code/extraction/LocalPropertyOutput.cc +++ b/Code/extraction/LocalPropertyOutput.cc @@ -145,7 +145,7 @@ namespace hemelb else { // Receive the writing start position from the previous core. - comms.Receive(localDataOffsetIntoFile, comms.Rank()-1, 1); + comms.Recv(localDataOffsetIntoFile, comms.Rank()-1, 1); // Send the next core its start position. if (comms.Rank() != (comms.Size() - 1)) diff --git a/Code/geometry/decomposition/OptimisedDecomposition.cc b/Code/geometry/decomposition/OptimisedDecomposition.cc index 720c599e3..0b3c2ad17 100644 --- a/Code/geometry/decomposition/OptimisedDecomposition.cc +++ b/Code/geometry/decomposition/OptimisedDecomposition.cc @@ -132,7 +132,6 @@ namespace hemelb idx_t weightFlag = 2; idx_t numberingFlag = 0; idx_t edgesCut = 0; - idx_t nDims = 3; idx_t options[4] = { 0, 0, 0, 0 }; if (ShouldValidate()) { @@ -1193,9 +1192,9 @@ namespace hemelb // If this is a greater rank number than the neighbouringProc, receive the data. if (neighbouringProc > comms.Rank()) { - comms.Receive(neighboursAdjacencyCount, neighbouringProc, 42); + comms.Recv(neighboursAdjacencyCount, neighbouringProc, 42); neighboursAdjacencyData.resize(neighboursAdjacencyCount); - comms.Receive(neighboursAdjacencyData, neighbouringProc, 43); + comms.Recv(neighboursAdjacencyData, neighbouringProc, 43); } else // Neigh == mTopologyRank, i.e. neighbouring vertices on the same proc // Duplicate the data. diff --git a/Code/net/CMakeLists.txt b/Code/net/CMakeLists.txt index 76fc3ffe4..f0844905f 100644 --- a/Code/net/CMakeLists.txt +++ b/Code/net/CMakeLists.txt @@ -1,6 +1,7 @@ add_library(hemelb_net MpiDataType.cc MpiEnvironment.cc MpiError.cc MpiCommunicator.cc MpiGroup.cc MpiFile.cc + MpiRequest.cc MpiStatus.cc IteratedAction.cc BaseNet.cc IOCommunicator.cc mixins/pointpoint/CoalescePointPoint.cc diff --git a/Code/net/MpiCommunicator.h b/Code/net/MpiCommunicator.h index c6f147f71..a76efa756 100644 --- a/Code/net/MpiCommunicator.h +++ b/Code/net/MpiCommunicator.h @@ -21,6 +21,7 @@ namespace hemelb namespace net { class MpiGroup; + class MpiRequest; class MpiCommunicator { @@ -115,15 +116,33 @@ namespace hemelb template std::vector AllToAll(const std::vector& vals) const; + template + void Send(const std::vector& val, int dest, int tag=0) const; template void Send(const T& val, int dest, int tag=0) const; template - void Send(const std::vector& val, int dest, int tag=0) const; + void Send(const T* valPtr, int count, int dest, int tag) const; + + template + void Recv(std::vector& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + template + void Recv(T& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + template + void Recv(T* val, int count, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + + template + MpiRequest Isend(const T& val, int dest, int tag=0) const; + template + MpiRequest Isend(const std::vector& vals, int dest, int tag=0) const; + template + MpiRequest Isend(const T* valPtr, int count, int dest, int tag=0) const; template - void Receive(T& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + MpiRequest Irecv(T& val, int source, int tag=0) const; + template + MpiRequest Irecv(std::vector& vals, int source, int tag=0) const; template - void Receive(std::vector& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + MpiRequest Irecv(T* valPtr, int count, int source, int tag=0) const; protected: /** diff --git a/Code/net/MpiCommunicator.hpp b/Code/net/MpiCommunicator.hpp index a53140859..c2ac69dd5 100644 --- a/Code/net/MpiCommunicator.hpp +++ b/Code/net/MpiCommunicator.hpp @@ -11,6 +11,7 @@ #include "net/MpiDataType.h" #include "net/MpiConstness.h" +#include "net/MpiRequest.h" namespace hemelb { @@ -136,39 +137,90 @@ namespace hemelb return ans; } + // Send implementations template - void MpiCommunicator::Send(const T& val, int dest, int tag) const + void MpiCommunicator::Send(const T* valPtr, int count, int dest, int tag) const { HEMELB_MPI_CALL( MPI_Send, - (MpiConstCast(&val), 1, MpiDataType(), dest, tag, *this) + (valPtr, count, MpiDataType(), dest, tag, *this) ); } template + void MpiCommunicator::Send(const T& val, int dest, int tag) const + { + Send(&val, 1, dest, tag); + } + template void MpiCommunicator::Send(const std::vector& vals, int dest, int tag) const + { + Send(&vals[0], vals.size(), dest, tag); + } + + // Recv implementations + template + void MpiCommunicator::Recv(T* valPtr, int count, int src, int tag, MPI_Status* stat) const { HEMELB_MPI_CALL( - MPI_Send, - (MpiConstCast(&vals[0]), vals.size(), MpiDataType(), dest, tag, *this) + MPI_Recv, + (valPtr, count, MpiDataType(), src, tag, *this, stat) ); } + template + void MpiCommunicator::Recv(T& val, int src, int tag, MPI_Status* stat) const + { + Recv(&val, 1, src, tag); + } + template + void MpiCommunicator::Recv(std::vector& vals, int src, int tag, MPI_Status* stat) const + { + Recv(&vals[0], vals.size(), src, tag); + } + // Isend implementations template - void MpiCommunicator::Receive(T& val, int src, int tag, MPI_Status* stat) const + MpiRequest MpiCommunicator::Isend(const T* valPtr, int count, int dest, int tag) const { + MPI_Request req; HEMELB_MPI_CALL( - MPI_Recv, - (&val, 1, MpiDataType(), src, tag, *this, stat) + MPI_Isend, + (const_cast(valPtr), count, MpiDataType(), dest, tag, *this, &req) ); + return MpiRequest(req); + } + template + MpiRequest MpiCommunicator::Isend(const T& val, int dest, int tag) const + { + return Isend(&val, 1, dest, tag); } template - void MpiCommunicator::Receive(std::vector& vals, int src, int tag, MPI_Status* stat) const + MpiRequest MpiCommunicator::Isend(const std::vector& vals, int dest, int tag) const + { + return Isend(&vals[0], vals.size(), dest, tag); + } + + // Irecv implementations + template + MpiRequest MpiCommunicator::Irecv(T* valPtr, int count, int source, int tag) const { + MPI_Request req; HEMELB_MPI_CALL( - MPI_Recv, - (&vals, vals.size(), MpiDataType(), src, tag, *this, stat) + MPI_Irecv, + (valPtr, count, MpiDataType(), source, tag, *this, &req) ); + return MpiRequest(req); + } + template + MpiRequest MpiCommunicator::Irecv(T& val, int source, int tag) const + { + return Irecv(&val, 1, source, tag); } + template + MpiRequest MpiCommunicator::Irecv(std::vector& vals, int source, int tag) const + { + return Irecv(&vals[0], vals.size(), source, tag); + } + } } diff --git a/Code/net/MpiRequest.cc b/Code/net/MpiRequest.cc new file mode 100644 index 000000000..477346ec4 --- /dev/null +++ b/Code/net/MpiRequest.cc @@ -0,0 +1,106 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#include "net/MpiRequest.h" +#include "net/MpiStatus.h" + +namespace hemelb +{ + namespace net + { + MpiRequest::MpiRequest() : + reqPtr() + { + } + MpiRequest::MpiRequest(MPI_Request req) : + reqPtr() + { + reqPtr.reset(new MPI_Request(req)); + } + + void MpiRequest::Wait() + { + HEMELB_MPI_CALL(MPI_Wait, (reqPtr.get(), MPI_STATUS_IGNORE)); + } + void MpiRequest::Wait(MpiStatus& stat) + { + HEMELB_MPI_CALL(MPI_Wait, (reqPtr.get(), stat.statPtr.get())); + } + + void MpiRequest::WaitAll(ReqVec& reqs) + { + size_t n = reqs.size(); + MPI_Request* rawreqs = new MPI_Request[n]; + try + { + for (size_t i = 0; i < n; ++i) + { + rawreqs[i] = reqs[i]; + } + + HEMELB_MPI_CALL( + MPI_Waitall, + (n, rawreqs, MPI_STATUSES_IGNORE) + ); + } + catch (const std::exception& e) + { + delete[] rawreqs; + throw; + } + + delete[] rawreqs; + } + + void MpiRequest::WaitAll(ReqVec& reqs, StatVec& stats) + { + size_t n = reqs.size(); + MPI_Request* rawreqs = new MPI_Request[n]; + MPI_Status* rawstats = new MPI_Status[n]; + try + { + for (size_t i = 0; i < n; ++i) + { + rawreqs[i] = reqs[i]; + } + + HEMELB_MPI_CALL( + MPI_Waitall, + (n, rawreqs, rawstats) + ); + + for (size_t i = 0; i < n; ++i) + { + stats[i] = MpiStatus(rawstats[i]); + } + } + catch (const std::exception& e) + { + delete[] rawreqs; + delete[] rawstats; + throw; + } + + delete[] rawreqs; + delete[] rawstats; + } + /* + void MpiRequest::WaitAll(ReqVec& reqs, StatVec& stats) + { + stats.resize(reqs.size()); + ReqVec::iterator reqIt = reqs.begin(); + StatVec::iterator statIt = stats.begin(); + for (; reqIt != reqs.end(); ++reqIt, ++statIt) + { + *statIt = reqIt ->Wait(); + } + } + */ + } +} diff --git a/Code/net/MpiRequest.h b/Code/net/MpiRequest.h new file mode 100644 index 000000000..3ba7fb807 --- /dev/null +++ b/Code/net/MpiRequest.h @@ -0,0 +1,57 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_NET_MPIREQUEST_H +#define HEMELB_NET_MPIREQUEST_H + +#include +#include "net/MpiError.h" +#include + +namespace hemelb +{ + namespace net + { + class MpiStatus; + /** + * + */ + class MpiRequest + { + private: + typedef std::vector ReqVec; + typedef std::vector StatVec; + + public: + MpiRequest(); + MpiRequest(MPI_Request req); + + /** + * Allow implicit casts to MPI_Request + * @return The underlying MPI_Request + */ + operator MPI_Request() const + { + return *reqPtr; + } + + void Wait(); + void Wait(MpiStatus& stat); + + static void WaitAll(ReqVec& reqs); + static void WaitAll(ReqVec& reqs, StatVec& stats); + + private: + + boost::shared_ptr reqPtr; + }; + + } +} +#endif // HEMELB_NET_MPIREQUEST_H diff --git a/Code/net/MpiStatus.cc b/Code/net/MpiStatus.cc new file mode 100644 index 000000000..62ff3fc92 --- /dev/null +++ b/Code/net/MpiStatus.cc @@ -0,0 +1,25 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#include "net/MpiStatus.h" + +namespace hemelb +{ + namespace net + { + MpiStatus::MpiStatus() : + statPtr() + { + } + MpiStatus::MpiStatus(MPI_Status stat) : statPtr() + { + statPtr.reset(new MPI_Status(stat)); + } + } +} diff --git a/Code/net/MpiStatus.h b/Code/net/MpiStatus.h new file mode 100644 index 000000000..2d08c07e6 --- /dev/null +++ b/Code/net/MpiStatus.h @@ -0,0 +1,44 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_NET_MPISTATUS_H +#define HEMELB_NET_MPISTATUS_H + +#include "net/MpiError.h" +#include + +namespace hemelb +{ + namespace net + { + class MpiRequest; + + class MpiStatus + { + public: + MpiStatus(); + MpiStatus(MPI_Status stat); + + /** + * Allow implicit casts to MPI_Status + * @return The underlying MPI_Status + */ + operator MPI_Status() const + { + return *statPtr; + } + + private: + boost::shared_ptr statPtr; + // Request needs to be able to access statPtr. + friend class MpiRequest; + }; + } +} +#endif // HEMELB_NET_MPISTATUS_H diff --git a/Code/net/mpi.h b/Code/net/mpi.h index 3e2336b30..2d2e36175 100644 --- a/Code/net/mpi.h +++ b/Code/net/mpi.h @@ -15,6 +15,8 @@ #include "net/MpiEnvironment.h" #include "net/MpiCommunicator.h" #include "net/MpiGroup.h" +#include "net/MpiStatus.h" +#include "net/MpiRequest.h" #include "net/MpiError.h" #endif // HEMELB_NET_MPI_H From 314fee8195aa5c77363dbed181b5f4acd6d3fcf2 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 18 Apr 2014 12:25:51 +0100 Subject: [PATCH 02/99] Add a helper function MapAllToAll that takes a std::map keyed on rank and only sends to ranks with associated values. --- Code/net/MapAllToAll.h | 77 ++++++++++++++++++++++++ Code/net/MpiCommunicator.cc | 24 ++++++++ Code/net/MpiCommunicator.h | 13 ++++ Code/net/MpiCommunicator.hpp | 22 +++++++ Code/net/MpiRequest.cc | 38 ++++++++++++ Code/net/MpiRequest.h | 3 + Code/net/mixins/InterfaceDelegationNet.h | 8 +-- 7 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 Code/net/MapAllToAll.h diff --git a/Code/net/MapAllToAll.h b/Code/net/MapAllToAll.h new file mode 100644 index 000000000..065619165 --- /dev/null +++ b/Code/net/MapAllToAll.h @@ -0,0 +1,77 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_NET_MAPALLTOALL_H +#define HEMELB_NET_MAPALLTOALL_H + +#include +#include "net/mpi.h" + +namespace hemelb +{ + namespace net + { + template + void MapAllToAll(const MpiCommunicator& comm, + const std::map& valsToSend, + std::map& receivedVals, + const int tag = 1234) + { + std::vector sendReqs(valsToSend.size()); + receivedVals.clear(); + int localRank = comm.Rank(); + + typename std::map::const_iterator iter = valsToSend.cbegin(), + end = valsToSend.cend(); + for (int i = 0; iter != end; ++iter, ++i) { + int rank = iter->first; + const T& val = iter->second; + if (rank == localRank) + { + receivedVals[localRank] = val; + } + else + { + // Synchronous to ensure we know when this is matched + sendReqs[i] = comm.Issend(val, rank, tag); + } + } + + bool allProcsHaveHadAllSendsMatched = false; + bool mySendsMatched = false; + MpiRequest barrierReq; + + // Allocate outside of loop to ensure compiler isn't reinitialising this + // all the time. + MPI_Status status; + while(!mySendsMatched || !allProcsHaveHadAllSendsMatched) + { + if (comm.Iprobe(MPI_ANY_SOURCE, tag, &status)) { + // There is a message waiting + comm.Recv(receivedVals[status.MPI_SOURCE], status.MPI_SOURCE, status.MPI_TAG); + } + + if (!mySendsMatched) + { + if (MpiRequest::TestAll(sendReqs)) + { + mySendsMatched = true; + barrierReq = comm.Ibarrier(); + } + } + else + { + allProcsHaveHadAllSendsMatched = barrierReq.Test(); + } + } + } + } +} + +#endif /* HEMELB_NET_MAPALLTOALL_H */ diff --git a/Code/net/MpiCommunicator.cc b/Code/net/MpiCommunicator.cc index a046547d4..da06d32d5 100644 --- a/Code/net/MpiCommunicator.cc +++ b/Code/net/MpiCommunicator.cc @@ -9,6 +9,7 @@ #include "net/MpiCommunicator.h" #include "net/MpiGroup.h" +#include "net/MpiRequest.h" namespace hemelb { @@ -114,5 +115,28 @@ namespace hemelb HEMELB_MPI_CALL(MPI_Comm_dup, (*commPtr, &newComm)); return MpiCommunicator(newComm, true); } + + void MpiCommunicator::Barrier() const + { + HEMELB_MPI_CALL(MPI_Barrier, (*commPtr)); + } + + MpiRequest MpiCommunicator::Ibarrier() const + { + MPI_Request req; + HEMELB_MPI_CALL(MPI_Ibarrier, (*commPtr, &req)); + return MpiRequest(req); + } + + bool MpiCommunicator::Iprobe(int source, int tag, MPI_Status* stat) const + { + int flag; + HEMELB_MPI_CALL( + MPI_Iprobe, + (source, tag, *commPtr, &flag, stat) + ); + return flag; + } + } } diff --git a/Code/net/MpiCommunicator.h b/Code/net/MpiCommunicator.h index a76efa756..462e456a4 100644 --- a/Code/net/MpiCommunicator.h +++ b/Code/net/MpiCommunicator.h @@ -22,6 +22,7 @@ namespace hemelb { class MpiGroup; class MpiRequest; + class MpiStatus; class MpiCommunicator { @@ -92,6 +93,11 @@ namespace hemelb */ MpiCommunicator Duplicate() const; + void Barrier() const; + MpiRequest Ibarrier() const; + + bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const; + template void Broadcast(T& val, const int root) const; template @@ -137,6 +143,13 @@ namespace hemelb template MpiRequest Isend(const T* valPtr, int count, int dest, int tag=0) const; + template + MpiRequest Issend(const T& val, int dest, int tag=0) const; + template + MpiRequest Issend(const std::vector& vals, int dest, int tag=0) const; + template + MpiRequest Issend(const T* valPtr, int count, int dest, int tag=0) const; + template MpiRequest Irecv(T& val, int source, int tag=0) const; template diff --git a/Code/net/MpiCommunicator.hpp b/Code/net/MpiCommunicator.hpp index c2ac69dd5..3e3c1047c 100644 --- a/Code/net/MpiCommunicator.hpp +++ b/Code/net/MpiCommunicator.hpp @@ -199,6 +199,28 @@ namespace hemelb return Isend(&vals[0], vals.size(), dest, tag); } + // Issend implementations + template + MpiRequest MpiCommunicator::Issend(const T* valPtr, int count, int dest, int tag) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Issend, + (const_cast(valPtr), count, MpiDataType(), dest, tag, *this, &req) + ); + return MpiRequest(req); + } + template + MpiRequest MpiCommunicator::Issend(const T& val, int dest, int tag) const + { + return Issend(&val, 1, dest, tag); + } + template + MpiRequest MpiCommunicator::Issend(const std::vector& vals, int dest, int tag) const + { + return Issend(&vals[0], vals.size(), dest, tag); + } + // Irecv implementations template MpiRequest MpiCommunicator::Irecv(T* valPtr, int count, int source, int tag) const diff --git a/Code/net/MpiRequest.cc b/Code/net/MpiRequest.cc index 477346ec4..854a6be25 100644 --- a/Code/net/MpiRequest.cc +++ b/Code/net/MpiRequest.cc @@ -102,5 +102,43 @@ namespace hemelb } } */ + + bool MpiRequest::Test() + { + int flag; + HEMELB_MPI_CALL(MPI_Test, (reqPtr.get(), &flag, MPI_STATUS_IGNORE)); + return flag; + } + + bool MpiRequest::TestAll(ReqVec& reqs) + { + int flag; + size_t n = reqs.size(); + if (n == 0) + return true; + + MPI_Request* rawreqs = new MPI_Request[n]; + try + { + for (size_t i = 0; i < n; ++i) + { + rawreqs[i] = reqs[i]; + } + + HEMELB_MPI_CALL( + MPI_Testall, + (n, rawreqs, &flag, MPI_STATUSES_IGNORE) + ); + } + catch (const std::exception& e) + { + delete[] rawreqs; + throw; + } + + delete[] rawreqs; + return flag; + } + } } diff --git a/Code/net/MpiRequest.h b/Code/net/MpiRequest.h index 3ba7fb807..0b8fee7de 100644 --- a/Code/net/MpiRequest.h +++ b/Code/net/MpiRequest.h @@ -47,6 +47,9 @@ namespace hemelb static void WaitAll(ReqVec& reqs); static void WaitAll(ReqVec& reqs, StatVec& stats); + bool Test(); + static bool TestAll(ReqVec& reqs); + private: boost::shared_ptr reqPtr; diff --git a/Code/net/mixins/InterfaceDelegationNet.h b/Code/net/mixins/InterfaceDelegationNet.h index 0b12ac0bd..f6ff93182 100644 --- a/Code/net/mixins/InterfaceDelegationNet.h +++ b/Code/net/mixins/InterfaceDelegationNet.h @@ -27,13 +27,13 @@ namespace hemelb } template - void RequestSendV(std::vector &payload, proc_t toRank) + void RequestSendV(const std::vector &payload, proc_t toRank) { RequestSend(&payload[0], payload.size(), toRank); } template - void RequestSendR(T& value, proc_t toRank) + void RequestSendR(const T& value, proc_t toRank) { RequestSend(&value, 1, toRank); } @@ -129,9 +129,9 @@ namespace hemelb } template - void RequestSend(T* pointer, int count, proc_t rank) + void RequestSend(const T* pointer, int count, proc_t rank) { - RequestSendImpl(pointer, count, rank, MpiDataType()); + RequestSendImpl(const_cast(pointer), count, rank, MpiDataType()); } template From 483cd778c62c8d254f7d6ef340f1756cf46ccae0 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 18 Apr 2014 12:25:52 +0100 Subject: [PATCH 03/99] Factor the NeighbouringDataManager to use the MapAllToAll and fix a bug in that --- .../neighbouring/NeighbouringDataManager.cc | 119 ++++++++++-------- .../neighbouring/NeighbouringDataManager.h | 7 +- Code/net/MapAllToAll.h | 14 ++- .../NeighbouringDataManagerTests.h | 81 +++--------- 4 files changed, 101 insertions(+), 120 deletions(-) diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.cc b/Code/geometry/neighbouring/NeighbouringDataManager.cc index e3cc9f06b..108d2318a 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.cc +++ b/Code/geometry/neighbouring/NeighbouringDataManager.cc @@ -11,8 +11,9 @@ #include "geometry/neighbouring/NeighbouringDataManager.h" #include "geometry/LatticeData.h" - #include "log/Logger.h" +#include "net/MapAllToAll.h" + namespace hemelb { namespace geometry @@ -24,8 +25,7 @@ namespace hemelb const LatticeData & localLatticeData, NeighbouringLatticeData & neighbouringLatticeData, net::InterfaceDelegationNet & net) : localLatticeData(localLatticeData), neighbouringLatticeData(neighbouringLatticeData), - net(net), needsEachProcHasFromMe(net.Size()), - needsHaveBeenShared(false) + net(net), needsHaveBeenShared(false) { } void NeighbouringDataManager::RegisterNeededSite(site_t globalId, @@ -53,7 +53,7 @@ namespace hemelb // on the sending and receiving procs. // But, the needsEachProcHasFromMe is always ordered, // by the same order, as the neededSites, so this should be OK. - for (std::vector::iterator localNeed = neededSites.begin(); + for (IdVec::iterator localNeed = neededSites.begin(); localNeed != neededSites.end(); localNeed++) { proc_t source = ProcForSite(*localNeed); @@ -68,22 +68,27 @@ namespace hemelb source); net.RequestReceiveR(site.GetWallNormal(), source); } - for (proc_t other = 0; other < net.Size(); other++) + + for (IdsMap::const_iterator iter = needsEachProcHasFromMe.cbegin(); + iter != needsEachProcHasFromMe.cend(); + ++iter) { - for (std::vector::iterator needOnProcFromMe = - needsEachProcHasFromMe[other].begin(); - needOnProcFromMe != needsEachProcHasFromMe[other].end(); needOnProcFromMe++) + proc_t other = iter->first; + const IdVec& neededIds = iter->second; + for (IdVec::const_iterator needOnProcFromMe = neededIds.cbegin(); + needOnProcFromMe != neededIds.cend(); + ++needOnProcFromMe) { site_t localContiguousId = - localLatticeData.GetLocalContiguousIdFromGlobalNoncontiguousId(*needOnProcFromMe); + localLatticeData.GetLocalContiguousIdFromGlobalNoncontiguousId(*needOnProcFromMe); - Site site = - const_cast(localLatticeData).GetSite(localContiguousId); - // have to cast away the const, because no respect for const-ness for sends in MPI - net.RequestSendR(site.GetSiteData().GetWallIntersectionData(), other); - net.RequestSendR(site.GetSiteData().GetIoletIntersectionData(), other); - net.RequestSendR(site.GetSiteData().GetIoletId(), other); - net.RequestSendR(site.GetSiteData().GetSiteType(), other); + const Site site = localLatticeData.GetSite(localContiguousId); + const SiteData& sd = site.GetSiteData(); + + net.RequestSendR(sd.GetWallIntersectionData(), other); + net.RequestSendR(sd.GetIoletIntersectionData(), other); + net.RequestSendR(sd.GetIoletId(), other); + net.RequestSendR(sd.GetSiteType(), other); net.RequestSend(site.GetWallDistances(), localLatticeData.GetLatticeInfo().GetNumVectors() - 1, other); @@ -111,7 +116,7 @@ namespace hemelb // on the sending and receiving procs. // But, the needsEachProcHasFromMe is always ordered, // by the same order, as the neededSites, so this should be OK. - for (std::vector::iterator localNeed = neededSites.begin(); + for (IdVec::iterator localNeed = neededSites.begin(); localNeed != neededSites.end(); localNeed++) { proc_t source = ProcForSite(*localNeed); @@ -121,21 +126,24 @@ namespace hemelb source); } - for (proc_t other = 0; other < net.Size(); other++) + + const unsigned Q = localLatticeData.GetLatticeInfo().GetNumVectors(); + for (IdsMap::const_iterator iter = needsEachProcHasFromMe.cbegin(); + iter != needsEachProcHasFromMe.cend(); + ++iter) { - for (std::vector::iterator needOnProcFromMe = - needsEachProcHasFromMe[other].begin(); - needOnProcFromMe != needsEachProcHasFromMe[other].end(); needOnProcFromMe++) + proc_t other = iter->first; + const IdVec& neededIds = iter->second; + for (IdVec::const_iterator needOnProcFromMe = neededIds.cbegin(); + needOnProcFromMe != neededIds.end(); ++needOnProcFromMe) { site_t localContiguousId = localLatticeData.GetLocalContiguousIdFromGlobalNoncontiguousId(*needOnProcFromMe); - Site site = - const_cast(localLatticeData).GetSite(localContiguousId); + const Site site = localLatticeData.GetSite(localContiguousId); // have to cast away the const, because no respect for const-ness for sends in MPI - net.RequestSend(const_cast(site.GetFOld(localLatticeData.GetLatticeInfo().GetNumVectors())), - localLatticeData.GetLatticeInfo().GetNumVectors(), + net.RequestSend(site.GetFOld(Q), + Q, other); - } } } @@ -146,39 +154,48 @@ namespace hemelb //if (needsHaveBeenShared == true) // return; //TODO: Fix! - // build a table of which procs needs can be achieved from which proc - std::vector > needsIHaveFromEachProc(net.Size()); - std::vector countOfNeedsIHaveFromEachProc(net.Size(), 0); - for (std::vector::iterator localNeed = neededSites.begin(); + // build a table of which sites are needed by this rank, by other rank. + IdsMap needsIHaveFromEachProc; + // This map will count the number per-rank + CountMap countOfNeedsIHaveFromEachProc; + for (IdVec::iterator localNeed = neededSites.begin(); localNeed != neededSites.end(); localNeed++) { - needsIHaveFromEachProc[ProcForSite(*localNeed)].push_back(*localNeed); - countOfNeedsIHaveFromEachProc[ProcForSite(*localNeed)]++; - + site_t needId = *localNeed; + int needRank = ProcForSite(needId); + needsIHaveFromEachProc[needRank].push_back(needId); + countOfNeedsIHaveFromEachProc[needRank]++; } - // every proc must send to all procs, how many it needs from that proc - net.RequestAllToAllSend(countOfNeedsIHaveFromEachProc); - - // every proc must receive from all procs, how many it needs to give that proc - std::vector countOfNeedsOnEachProcFromMe(net.Size(), 0); - net.RequestAllToAllReceive(countOfNeedsOnEachProcFromMe); - net.Dispatch(); - - for (proc_t other = 0; other < net.Size(); other++) + // This will store the numbers of sites other ranks need from this rank + CountMap countOfNeedsOnEachProcFromMe; + // This is collective + const net::MpiCommunicator& comms = net.GetCommunicator(); + net::MapAllToAll(comms, countOfNeedsIHaveFromEachProc, countOfNeedsOnEachProcFromMe, 1234); + + std::vector requestQueue; + // Now, for every rank, which I need something from, send the ids of those + for (CountMap::const_iterator countIt = countOfNeedsIHaveFromEachProc.cbegin(); + countIt != countOfNeedsIHaveFromEachProc.cend(); + ++countIt) { + int other = countIt->first; + requestQueue.push_back(comms.Isend(needsIHaveFromEachProc[other], other)); + } - // now, for every proc, which I need something from,send the ids of those - net.RequestSendV(needsIHaveFromEachProc[other], other); - // and, for every proc, which needs something from me, receive those ids - needsEachProcHasFromMe[other].resize(countOfNeedsOnEachProcFromMe[other]); - net.RequestReceiveV(needsEachProcHasFromMe[other], other); - // In principle, this bit could have been implemented as a separate GatherV onto every proc - // However, in practice, we expect the needs to be basically local - // so using point-to-point will be more efficient. + // And for every rank, which needs something from me, receive those ids + for (CountMap::const_iterator countIt = countOfNeedsOnEachProcFromMe.cbegin(); + countIt != countOfNeedsOnEachProcFromMe.cend(); + ++countIt) + { + int other = countIt->first; + int size = countIt->second; + IdVec& otherNeeds = needsEachProcHasFromMe[other]; + otherNeeds.resize(size); + requestQueue.push_back(comms.Irecv(otherNeeds, other)); } - net.Dispatch(); + net::MpiRequest::WaitAll(requestQueue); needsHaveBeenShared = true; } } diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.h b/Code/geometry/neighbouring/NeighbouringDataManager.h index 69caa50b7..65d685bbf 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.h +++ b/Code/geometry/neighbouring/NeighbouringDataManager.h @@ -58,8 +58,11 @@ namespace hemelb NeighbouringLatticeData & neighbouringLatticeData; net::InterfaceDelegationNet & net; - std::vector neededSites; - std::vector > needsEachProcHasFromMe; + typedef std::vector IdVec; + typedef std::map CountMap; + typedef std::map IdsMap; + IdVec neededSites; + IdsMap needsEachProcHasFromMe; bool needsHaveBeenShared; diff --git a/Code/net/MapAllToAll.h b/Code/net/MapAllToAll.h index 065619165..d95c0b520 100644 --- a/Code/net/MapAllToAll.h +++ b/Code/net/MapAllToAll.h @@ -23,13 +23,22 @@ namespace hemelb std::map& receivedVals, const int tag = 1234) { - std::vector sendReqs(valsToSend.size()); receivedVals.clear(); + + std::vector sendReqs; int localRank = comm.Rank(); + int nSends = valsToSend.size(); + // Is localRank in the map of send destinations? + if (valsToSend.find(localRank) != valsToSend.end()) + // If so, reduce the number of sends by 1. + nSends -= 1; + // Set up a container for all the Status objs + sendReqs.resize(nSends); typename std::map::const_iterator iter = valsToSend.cbegin(), end = valsToSend.cend(); - for (int i = 0; iter != end; ++iter, ++i) { + int i = 0; + for (; iter != end; ++iter) { int rank = iter->first; const T& val = iter->second; if (rank == localRank) @@ -40,6 +49,7 @@ namespace hemelb { // Synchronous to ensure we know when this is matched sendReqs[i] = comm.Issend(val, rank, tag); + i++; } } diff --git a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h index e1cfeda43..254def5f8 100644 --- a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h +++ b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h @@ -50,6 +50,17 @@ namespace hemelb manager = new NeighbouringDataManager(*latDat, *data, *netMock); } + void UseRealCommunicator() + { + delete manager; + delete netMock; + delete communicatorMock; + + communicatorMock = new net::MpiCommunicator(net::MpiCommunicator::World()); + netMock = new net::NetMock(*communicatorMock); + manager = new NeighbouringDataManager(*latDat, *data, *netMock); + } + void tearDown() { delete manager; @@ -93,26 +104,10 @@ namespace hemelb void TestShareNeedsOneProc() { + UseRealCommunicator(); manager->RegisterNeededSite(43); - // We should receive a signal that we need one from ourselves - std::vector countOfNeedsToZeroFromZero; - countOfNeedsToZeroFromZero.push_back(1); // expectation - std::vector countOfNeedsFromZeroToZero; - countOfNeedsFromZeroToZero.push_back(1); //fixture - netMock->RequireSend(&countOfNeedsToZeroFromZero.front(), 1, 0, "CountToSelf"); - netMock->RequireReceive(&countOfNeedsFromZeroToZero.front(), 1, 0, "CountFromSelf"); - - // Once we've received the signal that we need one from ourselves, we should receive that one. - std::vector needsShouldBeSentToSelf; - std::vector needsShouldBeReceivedFromSelf; - needsShouldBeSentToSelf.push_back(43); //expectation - needsShouldBeReceivedFromSelf.push_back(43); //fixture - netMock->RequireSend(&needsShouldBeSentToSelf.front(), 1, 0, "NeedToSelf"); - netMock->RequireReceive(&needsShouldBeSentToSelf.front(), 1, 0, "NeedFromSelf"); - manager->ShareNeeds(); - netMock->ExpectationsAllCompleted(); CPPUNIT_ASSERT_EQUAL(manager->GetNeedsForProc(0).size(), static_cast::size_type> (1)); @@ -121,24 +116,9 @@ namespace hemelb void TestShareConstantDataOneProc() { - // As for ShareNeeds test, set up the site as needed from itself. - std::vector countOfNeedsToZeroFromZero; - countOfNeedsToZeroFromZero.push_back(1); // expectation - std::vector countOfNeedsFromZeroToZero; - countOfNeedsFromZeroToZero.push_back(1); //fixture - netMock->RequireSend(&countOfNeedsToZeroFromZero.front(), 1, 0, "CountToSelf"); - netMock->RequireReceive(&countOfNeedsFromZeroToZero.front(), 1, 0, "CountFromSelf"); - - std::vector needsShouldBeSentToSelf; - std::vector needsShouldBeReceivedFromSelf; - needsShouldBeSentToSelf.push_back(43); //expectation - needsShouldBeReceivedFromSelf.push_back(43); //fixture - netMock->RequireSend(&needsShouldBeSentToSelf.front(), 1, 0, "NeedToSelf"); - netMock->RequireReceive(&needsShouldBeSentToSelf.front(), 1, 0, "NeedFromSelf"); - + UseRealCommunicator(); manager->RegisterNeededSite(43); manager->ShareNeeds(); - netMock->ExpectationsAllCompleted(); // Now, transfer the data about that site. Site < LatticeData > exampleSite @@ -194,6 +174,8 @@ namespace hemelb void TestShareFieldDataOneProc() { + UseRealCommunicator(); + site_t targetGlobalOneDIdx = 43; LatticeVector targetGlobalThreeDIdx = latDat->GetSiteCoordsFromSiteId(targetGlobalOneDIdx); site_t targetLocalIdx = latDat->GetLocalContiguousIdFromGlobalNoncontiguousId(targetGlobalOneDIdx); @@ -201,24 +183,8 @@ namespace hemelb for (unsigned int direction = 0; direction < 3; direction++) CPPUNIT_ASSERT_EQUAL(site_t(1), targetGlobalThreeDIdx[direction]); - // begin by setting up mocks for the required site - std::vector countOfNeedsToZeroFromZero; - countOfNeedsToZeroFromZero.push_back(1); // expectation - std::vector countOfNeedsFromZeroToZero; - countOfNeedsFromZeroToZero.push_back(1); //fixture - netMock->RequireSend(&countOfNeedsToZeroFromZero.front(), 1, 0, "CountToSelf"); - netMock->RequireReceive(&countOfNeedsFromZeroToZero.front(), 1, 0, "CountFromSelf"); - - std::vector needsShouldBeSentToSelf; - std::vector needsShouldBeReceivedFromSelf; - needsShouldBeSentToSelf.push_back(targetGlobalOneDIdx); //expectation - needsShouldBeReceivedFromSelf.push_back(targetGlobalOneDIdx); //fixture - netMock->RequireSend(&needsShouldBeSentToSelf.front(), 1, 0, "NeedToSelf"); - netMock->RequireReceive(&needsShouldBeSentToSelf.front(), 1, 0, "NeedFromSelf"); - manager->RegisterNeededSite(targetGlobalOneDIdx); manager->ShareNeeds(); - netMock->ExpectationsAllCompleted(); // Now, transfer the data about that site. Site < LatticeData > exampleSite = latDat->GetSite(targetLocalIdx); @@ -249,27 +215,12 @@ namespace hemelb void TestShareFieldDataOneProcViaIterableAction() { + UseRealCommunicator(); site_t targetGlobalOneDIdx = 43; site_t targetLocalIdx = latDat->GetLocalContiguousIdFromGlobalNoncontiguousId(targetGlobalOneDIdx); - // begin by setting up mocks for the required site - std::vector countOfNeedsToZeroFromZero; - countOfNeedsToZeroFromZero.push_back(1); // expectation - std::vector countOfNeedsFromZeroToZero; - countOfNeedsFromZeroToZero.push_back(1); //fixture - netMock->RequireSend(&countOfNeedsToZeroFromZero.front(), 1, 0, "CountToSelf"); - netMock->RequireReceive(&countOfNeedsFromZeroToZero.front(), 1, 0, "CountFromSelf"); - - std::vector needsShouldBeSentToSelf; - std::vector needsShouldBeReceivedFromSelf; - needsShouldBeSentToSelf.push_back(targetGlobalOneDIdx); //expectation - needsShouldBeReceivedFromSelf.push_back(targetGlobalOneDIdx); //fixture - netMock->RequireSend(&needsShouldBeSentToSelf.front(), 1, 0, "NeedToSelf"); - netMock->RequireReceive(&needsShouldBeSentToSelf.front(), 1, 0, "NeedFromSelf"); - manager->RegisterNeededSite(targetGlobalOneDIdx); manager->ShareNeeds(); - netMock->ExpectationsAllCompleted(); // Now, transfer the data about that site. Site < LatticeData > exampleSite = latDat->GetSite(targetLocalIdx); From cb881bbfa9c7e9d92485d106b5c2de9ccae20c98 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 18 Apr 2014 12:26:00 +0100 Subject: [PATCH 04/99] Remove AllToAll from the net class hierarchy (as it's unused) --- Code/CMakeLists.txt | 2 -- Code/net/BaseNet.cc | 3 --- Code/net/BaseNet.h | 7 ------ Code/net/BuildInfo.h.in | 3 +-- Code/net/CMakeLists.txt | 2 -- Code/net/ProcComms.h | 6 ----- Code/net/mixins/InterfaceDelegationNet.h | 30 ------------------------ Code/net/mixins/StoringNet.cc | 9 ------- Code/net/mixins/StoringNet.h | 7 ------ Code/net/mixins/mixins.h | 2 -- Code/net/net.h | 3 +-- Code/reporting/BuildInfo.h.in | 4 +--- Code/resources/report.txt.ctp | 1 - Code/resources/report.xml.ctp | 3 +-- Code/unittests/net/NetMock.h | 9 +++---- 15 files changed, 7 insertions(+), 84 deletions(-) diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 4aea565e9..ec9a52521 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -56,8 +56,6 @@ set(HEMELB_POINTPOINT_IMPLEMENTATION Coalesce CACHE STRING "Point to point comms implementation, choose 'Coalesce', 'Separated', or 'Immediate'" ) set(HEMELB_GATHERS_IMPLEMENTATION Separated CACHE STRING "Gather comms implementation, choose 'Separated', or 'ViaPointPoint'" ) -set(HEMELB_ALLTOALL_IMPLEMENTATION Separated - CACHE STRING "Alltoall comms implementation, choose 'Separated', or 'ViaPointPoint'" ) option(HEMELB_SEPARATE_CONCERNS "Communicate for each concern separately" OFF) # Add warnings flags to development build types diff --git a/Code/net/BaseNet.cc b/Code/net/BaseNet.cc index e9481508d..5bc44286b 100644 --- a/Code/net/BaseNet.cc +++ b/Code/net/BaseNet.cc @@ -42,7 +42,6 @@ namespace hemelb { ReceiveGathers(); ReceiveGatherVs(); - ReceiveAllToAll(); // Ensure collectives are called before point-to-point, as some implementing mixins implement collectives via point-to-point ReceivePointToPoint(); } @@ -52,7 +51,6 @@ namespace hemelb SendGathers(); SendGatherVs(); - SendAllToAll(); // Ensure collectives are called before point-to-point, as some implementing mixins implement collectives via point-to-point SendPointToPoint(); } @@ -64,7 +62,6 @@ namespace hemelb WaitGathers(); WaitGatherVs(); WaitPointToPoint(); - WaitAllToAll(); displacementsBuffer.clear(); countsBuffer.clear(); diff --git a/Code/net/BaseNet.h b/Code/net/BaseNet.h index a00850c7a..b6406b581 100644 --- a/Code/net/BaseNet.h +++ b/Code/net/BaseNet.h @@ -64,17 +64,14 @@ namespace hemelb virtual void SendPointToPoint()=0; virtual void SendGathers()=0; virtual void SendGatherVs()=0; - virtual void SendAllToAll()=0; virtual void ReceiveGathers()=0; virtual void ReceiveGatherVs()=0; virtual void ReceivePointToPoint()=0; - virtual void ReceiveAllToAll()=0; virtual void WaitPointToPoint()=0; virtual void WaitGathers()=0; virtual void WaitGatherVs()=0; - virtual void WaitAllToAll()=0; // Interfaces exposing MPI_Datatype, not intended for client class use virtual void RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type)=0; @@ -91,10 +88,6 @@ namespace hemelb virtual void RequestGatherSendImpl(void* buffer, proc_t toRank, MPI_Datatype type)=0; virtual void RequestGatherVReceiveImpl(void* buffer, int * displacements, int *counts, MPI_Datatype type)=0; - - virtual void RequestAllToAllReceiveImpl(void * buffer,int count,MPI_Datatype type)=0; - virtual void RequestAllToAllSendImpl(void * buffer,int count,MPI_Datatype type)=0; - std::vector & GetDisplacementsBuffer(); std::vector & GetCountsBuffer(); diff --git a/Code/net/BuildInfo.h.in b/Code/net/BuildInfo.h.in index 336a6bc8c..bc4f2d72f 100644 --- a/Code/net/BuildInfo.h.in +++ b/Code/net/BuildInfo.h.in @@ -15,7 +15,6 @@ namespace hemelb { typedef @HEMELB_POINTPOINT_IMPLEMENTATION@PointPoint PointPointImpl ; typedef @HEMELB_GATHERS_IMPLEMENTATION@Gathers GathersImpl ; - typedef @HEMELB_ALLTOALL_IMPLEMENTATION@AllToAll AllToAllImpl ; #cmakedefine HEMELB_SEPARATE_CONCERNS #ifdef HEMELB_SEPARATE_CONCERNS static const bool separate_communications = true; @@ -24,4 +23,4 @@ namespace hemelb #endif } } -#endif // HEMELB_NET_BUILDINFO_H_IN \ No newline at end of file +#endif // HEMELB_NET_BUILDINFO_H_IN diff --git a/Code/net/CMakeLists.txt b/Code/net/CMakeLists.txt index f0844905f..531fb06a2 100644 --- a/Code/net/CMakeLists.txt +++ b/Code/net/CMakeLists.txt @@ -9,8 +9,6 @@ mixins/pointpoint/SeparatedPointPoint.cc mixins/pointpoint/ImmediatePointPoint.cc mixins/gathers/SeparatedGathers.cc mixins/gathers/ViaPointPointGathers.cc -mixins/alltoall/SeparatedAllToAll.cc -mixins/alltoall/ViaPointPointAllToAll.cc mixins/StoringNet.cc ProcComms.cc phased/StepManager.cc) configure_file ( diff --git a/Code/net/ProcComms.h b/Code/net/ProcComms.h index 8be053a0a..970b0690b 100644 --- a/Code/net/ProcComms.h +++ b/Code/net/ProcComms.h @@ -37,12 +37,6 @@ namespace hemelb }; - class AllToAllProcComms : public BaseProcComms - // Rank of request is not used - is all-to-all - { - - }; - class GatherVReceiveProcComms : public BaseProcComms { diff --git a/Code/net/mixins/InterfaceDelegationNet.h b/Code/net/mixins/InterfaceDelegationNet.h index f6ff93182..670676a30 100644 --- a/Code/net/mixins/InterfaceDelegationNet.h +++ b/Code/net/mixins/InterfaceDelegationNet.h @@ -109,25 +109,6 @@ namespace hemelb RequestGatherVSend(&payload.front(), payload.size(), toRank); } - /*** - * This is for a scalar all to all - * @param buffer vector with length same as communicator size - */ - template - void RequestAllToAllSend(std::vector &buffer) - { - RequestAllToAllSend(&buffer.front(), 1); - } - /*** - * This is for a scalar all to all - * @param buffer vector with length same as communicator size - */ - template - void RequestAllToAllReceive(std::vector &buffer) - { - RequestAllToAllReceive(&buffer.front(), 1); - } - template void RequestSend(const T* pointer, int count, proc_t rank) { @@ -170,17 +151,6 @@ namespace hemelb RequestGatherVReceiveImpl(buffer, displacements, counts, MpiDataType()); } - template - void RequestAllToAllSend(T* buffer, int count) - { - RequestAllToAllSendImpl(buffer, count, MpiDataType()); - } - template - void RequestAllToAllReceive(T* buffer, int count) - { - RequestAllToAllReceiveImpl(buffer, count, MpiDataType()); - } - } ; } diff --git a/Code/net/mixins/StoringNet.cc b/Code/net/mixins/StoringNet.cc index f5f0f9fd6..7c5e50bae 100644 --- a/Code/net/mixins/StoringNet.cc +++ b/Code/net/mixins/StoringNet.cc @@ -66,14 +66,5 @@ namespace hemelb counts, type)); } - - void StoringNet::RequestAllToAllReceiveImpl(void * buffer, int count, MPI_Datatype type) - { - allToAllReceiveProcComms.push_back(SimpleRequest(buffer, count, type, 0)); - } - void StoringNet::RequestAllToAllSendImpl(void * buffer, int count, MPI_Datatype type) - { - allToAllSendProcComms.push_back(SimpleRequest(buffer, count, type, 0)); - } } } diff --git a/Code/net/mixins/StoringNet.h b/Code/net/mixins/StoringNet.h index f1693e00f..f5b2506a2 100644 --- a/Code/net/mixins/StoringNet.h +++ b/Code/net/mixins/StoringNet.h @@ -30,9 +30,6 @@ namespace hemelb void RequestGatherSendImpl(void* buffer, proc_t toRank, MPI_Datatype type); void RequestGatherVReceiveImpl(void* buffer, int * displacements, int *counts, MPI_Datatype type); - virtual void RequestAllToAllReceiveImpl(void * buffer, int count, MPI_Datatype type); - virtual void RequestAllToAllSendImpl(void * buffer, int count, MPI_Datatype type); - protected: /** * Struct representing all that's needed to successfully communicate with another processor. @@ -46,10 +43,6 @@ namespace hemelb std::map gatherSendProcessorComms; GatherProcComms gatherReceiveProcessorComms; - - AllToAllProcComms allToAllReceiveProcComms; - AllToAllProcComms allToAllSendProcComms; - }; } } diff --git a/Code/net/mixins/mixins.h b/Code/net/mixins/mixins.h index 28dc07ed8..ca6c1d84b 100644 --- a/Code/net/mixins/mixins.h +++ b/Code/net/mixins/mixins.h @@ -17,6 +17,4 @@ #include "net/mixins/gathers/SeparatedGathers.h" #include "net/mixins/InterfaceDelegationNet.h" #include "net/mixins/gathers/ViaPointPointGathers.h" -#include "net/mixins/alltoall/SeparatedAllToAll.h" -#include "net/mixins/alltoall/ViaPointPointAllToAll.h" #endif diff --git a/Code/net/net.h b/Code/net/net.h index e9a4e3600..e3569df53 100644 --- a/Code/net/net.h +++ b/Code/net/net.h @@ -19,13 +19,12 @@ namespace hemelb { class Net : public PointPointImpl, public InterfaceDelegationNet, - public AllToAllImpl, public GathersImpl { public: Net(const MpiCommunicator &communicator) : BaseNet(communicator), StoringNet(communicator), PointPointImpl(communicator), - InterfaceDelegationNet(communicator), AllToAllImpl(communicator), + InterfaceDelegationNet(communicator), GathersImpl(communicator) { } diff --git a/Code/reporting/BuildInfo.h.in b/Code/reporting/BuildInfo.h.in index bed699e4e..f463bf61a 100644 --- a/Code/reporting/BuildInfo.h.in +++ b/Code/reporting/BuildInfo.h.in @@ -31,7 +31,6 @@ namespace hemelb static const std::string wall_outlet_boundary_condition="@HEMELB_WALL_OUTLET_BOUNDARY@"; static const std::string point_point_impl="@HEMELB_POINTPOINT_IMPLEMENTATION@"; static const std::string gathers_impl="@HEMELB_GATHERS_IMPLEMENTATION@"; - static const std::string alltoall_impl="@HEMELB_ALLTOALL_IMPLEMENTATION@"; static const std::string separate_concerns="@HEMELB_SEPARATE_CONCERNS@"; @@ -54,11 +53,10 @@ namespace hemelb build->SetValue("WALL_INLET_BOUNDARY_CONDITION", wall_inlet_boundary_condition); build->SetValue("WALL_OUTLET_BOUNDARY_CONDITION", wall_outlet_boundary_condition); build->SetValue("SEPARATE_CONCERNS",separate_concerns); - build->SetValue("ALLTOALL_IMPLEMENTATION",alltoall_impl); build->SetValue("GATHERS_IMPLEMENTATION",gathers_impl); build->SetValue("POINTPOINT_IMPLEMENTATION",point_point_impl); } }; } } -#endif // HEMELB_REPORTING_BUILDINFO_H_IN \ No newline at end of file +#endif // HEMELB_REPORTING_BUILDINFO_H_IN diff --git a/Code/resources/report.txt.ctp b/Code/resources/report.txt.ctp index 8390d6ea0..7c73cfa0f 100644 --- a/Code/resources/report.txt.ctp +++ b/Code/resources/report.txt.ctp @@ -42,7 +42,6 @@ Wall/iolet boundary condition: {{WALL_IOLET_BOUNDARY_CONDITION}} Communications options: Point to point implementation: {{POINTPOINT_IMPLEMENTATION}} -All to all implementation: {{ALLTOALL_IMPLEMENTATION}} Gathers implementation: {{GATHERS_IMPLEMENTATION}} Separated concerns: {{SEPARATE_CONCERNS}} {{/BUILD}} \ No newline at end of file diff --git a/Code/resources/report.xml.ctp b/Code/resources/report.xml.ctp index 17cc7e052..51a4e6e64 100644 --- a/Code/resources/report.xml.ctp +++ b/Code/resources/report.xml.ctp @@ -27,7 +27,6 @@ {{POINTPOINT_IMPLEMENTATION}} - {{ALLTOALL_IMPLEMENTATION}} {{GATHERS_IMPLEMENTATION}} {{SEPARATE_CONCERNS}} @@ -79,4 +78,4 @@ {{/TIMER}} - \ No newline at end of file + diff --git a/Code/unittests/net/NetMock.h b/Code/unittests/net/NetMock.h index 369664b86..ace7109f2 100644 --- a/Code/unittests/net/NetMock.h +++ b/Code/unittests/net/NetMock.h @@ -27,16 +27,13 @@ namespace hemelb { using namespace hemelb::net; - class NetMock : public InterfaceDelegationNet, - public RecordingNet, - public ViaPointPointGathers, - public ViaPointPointAllToAll + class NetMock : public InterfaceDelegationNet,public RecordingNet, + public ViaPointPointGathers { public: NetMock(net::MpiCommunicator & communicator) : BaseNet(communicator), StoringNet(communicator), InterfaceDelegationNet(communicator), - RecordingNet(communicator), ViaPointPointGathers(communicator), - ViaPointPointAllToAll(communicator) + RecordingNet(communicator), ViaPointPointGathers(communicator) { } From 0240e8c34db08004071b4ac55e04e534c4bdabb2 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 18 Apr 2014 12:26:04 +0100 Subject: [PATCH 05/99] Remove a load of random extra parentheses in the code --- .../decomposition/OptimisedDecomposition.cc | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Code/geometry/decomposition/OptimisedDecomposition.cc b/Code/geometry/decomposition/OptimisedDecomposition.cc index 0b3c2ad17..fc48ec521 100644 --- a/Code/geometry/decomposition/OptimisedDecomposition.cc +++ b/Code/geometry/decomposition/OptimisedDecomposition.cc @@ -589,7 +589,7 @@ namespace hemelb timers[hemelb::reporting::Timers::moveForcingData].Start(); // Now get all the blocks being forced upon me. std::map > blocksForcedOnMeByEachProc; - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) { if (blocksForcedOnMe[otherProc] > 0) { @@ -609,7 +609,7 @@ namespace hemelb log::Logger::Log("Moving forcing block ids"); netForMoveSending.Dispatch(); // Now go through every block forced upon me and add it to the list of ones I want. - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) { if (blocksForcedOnMe[otherProc] > 0) { @@ -683,7 +683,7 @@ namespace hemelb log::Logger::Log("Calculating block requirements"); timers[hemelb::reporting::Timers::blockRequirements].Start(); // Populate numberOfBlocksRequiredFrom - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) { numberOfBlocksRequiredFrom[otherProc] = blockIdsIRequireFromX.count(otherProc) == 0 ? 0 : @@ -699,7 +699,7 @@ namespace hemelb // Awesome. Now we need to get a list of all the blocks wanted from each core by each other // core. net::Net netForMoveSending(comms); - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) { blockIdsXRequiresFromMe[otherProc] = std::vector(numberOfBlocksXRequiresFromMe[otherProc]); @@ -732,10 +732,10 @@ namespace hemelb } } - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) { for (site_t blockNum = 0; - blockNum < (site_t) ( ( ( ( (blockIdsXRequiresFromMe[otherProc].size()))))); + blockNum < site_t(blockIdsXRequiresFromMe[otherProc].size()); ++blockNum) { site_t blockId = blockIdsXRequiresFromMe[otherProc][blockNum]; @@ -751,7 +751,7 @@ namespace hemelb } - for (site_t moveNumber = 0; moveNumber < (site_t) ( ( ( ( (moveData.size()))))); + for (site_t moveNumber = 0; moveNumber < site_t(moveData.size()); moveNumber += 3) { site_t blockId = moveData[moveNumber]; @@ -766,7 +766,7 @@ namespace hemelb } net::Net netForMoveSending(comms); - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) { for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); it != blockIdsIRequireFromX[otherProc].end(); ++it) @@ -812,7 +812,7 @@ namespace hemelb net::Net netForMoveSending(comms); - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) { allMoves[otherProc] = 0; for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); @@ -1147,7 +1147,7 @@ namespace hemelb log::Logger::Log("Validating neighbour data"); // Now spread and compare the adjacency information. Larger ranks send data to smaller // ranks which receive the data and compare it. - for (proc_t neigh = 0; neigh < (proc_t) ( ( ( ( (comms.Size()))))); ++neigh) + for (proc_t neigh = 0; neigh < proc_t(comms.Size()); ++neigh) { SendAdjacencyDataToLowerRankedProc(neigh, counts[neigh], From f0ef956becae096da9901fa68402225dec237fae Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 21 Apr 2014 18:06:00 +0100 Subject: [PATCH 06/99] Reimplement the StabilityTester with a non-blocking collective --- Code/SimulationMaster.cc | 4 +- Code/lb/StabilityTester.h | 323 ++++++++++++++------------------------ 2 files changed, 118 insertions(+), 209 deletions(-) diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index 172c9aea2..8c5242fb7 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -216,7 +216,8 @@ void SimulationMaster::Initialise() &communicationNet, simulationState, timings, - monitoringConfig); + monitoringConfig->doConvergenceCheck, + monitoringConfig->convergenceRelativeTolerance); entropyTester = NULL; if (monitoringConfig->doIncompressibilityCheck) @@ -323,6 +324,7 @@ void SimulationMaster::Initialise() stepManager->RegisterIteratedActorSteps(*outletValues, 1); stepManager->RegisterIteratedActorSteps(*steeringCpt, 1); stepManager->RegisterIteratedActorSteps(*stabilityTester, 1); + stepManager->RegisterCommsSteps(*stabilityTester, 1); if (entropyTester != NULL) { stepManager->RegisterIteratedActorSteps(*entropyTester, 1); diff --git a/Code/lb/StabilityTester.h b/Code/lb/StabilityTester.h index 325b8863b..255d909ab 100644 --- a/Code/lb/StabilityTester.h +++ b/Code/lb/StabilityTester.h @@ -10,7 +10,8 @@ #ifndef HEMELB_LB_STABILITYTESTER_H #define HEMELB_LB_STABILITYTESTER_H -#include "net/PhasedBroadcastRegular.h" +#include "net/phased/Concern.h" +#include "net/phased/steps.h" #include "geometry/LatticeData.h" namespace hemelb @@ -18,258 +19,163 @@ namespace hemelb namespace lb { /** - * Class to repeatedly assess the stability of the simulation, using the PhasedBroadcast - * interface. - * - * PhasedBroadcastRegular is used because we know in advance which iterations we will - * want to communicate on. The default parameters suffice: no initial action is necessary - * because we can assess the stability just before communicating (it doesn't have to happen - * at the same time on all nodes), only one communication is needed between depths, which - * can't overlap. We go down the tree to pass the overall stability to all nodes, and we go up - * the tree to compose the local stability for all nodes to discover whether the simulation as - * a whole is stable. + * Class to repeatedly assess the stability of the simulation, using non-blocking collective */ template - class StabilityTester : public net::PhasedBroadcastRegular<> + class StabilityTester : public net::phased::Concern { public: StabilityTester(const geometry::LatticeData * iLatDat, net::Net* net, SimulationState* simState, reporting::Timers& timings, - const hemelb::configuration::SimConfig::MonitoringConfig* testerConfig) : - net::PhasedBroadcastRegular<>(net, simState, SPREADFACTOR), mLatDat(iLatDat), - mSimState(simState), timings(timings), testerConfig(testerConfig) + bool checkForConvergence, double relativeTolerance) : + mLatDat(iLatDat), + mSimState(simState), timings(timings), checkForConvergence(checkForConvergence), + relativeTolerance(relativeTolerance), collectiveComm(net->GetCommunicator().Duplicate()) { Reset(); } + /** * Override the reset method in the base class, to reset the stability variables. */ void Reset() { - mUpwardsStability = UndefinedStability; - mDownwardsStability = UndefinedStability; - + localStability = UndefinedStability; + globalStability = UndefinedStability; mSimState->SetStability(UndefinedStability); + } - for (unsigned int ii = 0; ii < SPREADFACTOR; ii++) + bool CallAction(int action) + { + switch (static_cast(action)) { - mChildrensStability[ii] = UndefinedStability; + case net::phased::steps::PreSend: + PreSend(); + return true; + case net::phased::steps::Send: + Send(); + return true; + case net::phased::steps::Wait: + Wait(); + return true; + case net::phased::steps::EndPhase: + Effect(); + return true; + default: + return false; } } protected: /** - * Override the methods from the base class to propagate data from the root, and - * to send data about this node and its childrens' stabilities up towards the root. - */ - void ProgressFromChildren(unsigned long splayNumber) - { - ReceiveFromChildren(mChildrensStability, 1); - } - - void ProgressFromParent(unsigned long splayNumber) - { - ReceiveFromParent(&mDownwardsStability, 1); - } - - void ProgressToChildren(unsigned long splayNumber) - { - SendToChildren(&mDownwardsStability, 1); - } - - void ProgressToParent(unsigned long splayNumber) - { - SendToParent(&mUpwardsStability, 1); - } - - /** - * The algorithm that checks distribution function convergence must be run in this - * method rather than in ProgressToParent to make sure that the current timestep has - * finished streaming. - * - * @param splayNumber + * Compute the local stability/convergence state. */ - void PostSendToParent(unsigned long splayNumber) + void PreSend(void) { timings[hemelb::reporting::Timers::monitoring].Start(); + bool unconvergedSitePresent = false; + bool checkConvThisTimeStep = checkForConvergence; + localStability = Stable; - // No need to bother testing out local lattice points if we're going to be - // sending up a 'Unstable' value anyway. - if (mUpwardsStability != Unstable) + for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) { - bool unconvergedSitePresent = false; - - for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) + for (unsigned int l = 0; l < LatticeType::NUMVECTORS; l++) { - for (unsigned int l = 0; l < LatticeType::NUMVECTORS; l++) - { - distribn_t value = *mLatDat->GetFNew(i * LatticeType::NUMVECTORS + l); + distribn_t value = *mLatDat->GetFNew(i * LatticeType::NUMVECTORS + l); - // Note that by testing for value > 0.0, we also catch stray NaNs. - if (! (value > 0.0)) - { - mUpwardsStability = Unstable; - break; - } - } - ///@todo: If we refactor the previous loop out, we can get away with a single break statement - if (mUpwardsStability == Unstable) + // Note that by testing for value > 0.0, we also catch stray NaNs. + if (! (value > 0.0)) { + localStability = Unstable; break; } + } + ///@todo: If we refactor the previous loop out, we can get away with a single break statement + if (localStability == Unstable) + break; - if (testerConfig->doConvergenceCheck) - { - distribn_t relativeDifference = - ComputeRelativeDifference(mLatDat->GetFNew(i * LatticeType::NUMVECTORS), - mLatDat->GetSite(i).GetFOld()); + if (checkConvThisTimeStep) + { + distribn_t relativeDifference = + ComputeRelativeDifference(mLatDat->GetFNew(i * LatticeType::NUMVECTORS), + mLatDat->GetSite(i).GetFOld()); - if (relativeDifference > testerConfig->convergenceRelativeTolerance) - { - // The simulation is stable but hasn't converged in the whole domain yet. - unconvergedSitePresent = true; - } + if (relativeDifference > relativeTolerance) + { + // The simulation is stable but hasn't converged in the whole domain yet. + unconvergedSitePresent = true; + // Skip further tests for this timestep + checkConvThisTimeStep = false; } } + } - switch (mUpwardsStability) + if (localStability == Stable) + { + if (checkForConvergence && !unconvergedSitePresent) { - case UndefinedStability: - case Stable: - case StableAndConverged: - mUpwardsStability = (testerConfig->doConvergenceCheck && !unconvergedSitePresent) ? - StableAndConverged : - Stable; - break; - case Unstable: - break; + localStability = StableAndConverged; } } - timings[hemelb::reporting::Timers::monitoring].Stop(); } /** - * Computes the relative difference between the densities at the beginning and end of a - * timestep, i.e. |(rho_new - rho_old) / (rho_old - rho_0)|. - * - * @param fNew Distribution function after stream and collide, i.e. solution of the current timestep. - * @param fOld Distribution function at the end of the previous timestep. - * @return relative difference between the densities computed from fNew and fOld. + * Initiate the collective. */ - inline double ComputeRelativeDifference(const distribn_t* fNew, - const distribn_t* fOld) const + void Send(void) { - distribn_t newDensity; - distribn_t newMomentumX; - distribn_t newMomentumY; - distribn_t newMomentumZ; - LatticeType::CalculateDensityAndMomentum(fNew, - newDensity, - newMomentumX, - newMomentumY, - newMomentumZ); - - distribn_t oldDensity; - distribn_t oldMomentumX; - distribn_t oldMomentumY; - distribn_t oldMomentumZ; - LatticeType::CalculateDensityAndMomentum(fOld, - oldDensity, - oldMomentumX, - oldMomentumY, - oldMomentumZ); - - distribn_t absoluteError; - distribn_t referenceValue; - - switch (testerConfig->convergenceVariable) - { - case extraction::OutputField::Velocity: - { - distribn_t diff_vel_x = newMomentumX / newDensity - oldMomentumX / oldDensity; - distribn_t diff_vel_y = newMomentumY / newDensity - oldMomentumY / oldDensity; - distribn_t diff_vel_z = newMomentumZ / newDensity - oldMomentumZ / oldDensity; - - absoluteError = sqrt(diff_vel_x * diff_vel_x + diff_vel_y * diff_vel_y - + diff_vel_z * diff_vel_z); - referenceValue = testerConfig->convergenceReferenceValue; - break; - } - default: - // Never reached - throw Exception() - << "Convergence check based on requested variable currently not available"; - } - - return absoluteError / referenceValue; + // Begin collective. + HEMELB_MPI_CALL( + MPI_Iallreduce, + (static_cast(&localStability), static_cast(&globalStability), 1, + net::MpiDataType(localStability), MPI_MIN, collectiveComm, &collectiveReq) + ); } /** - * Take the combined stability information (an int, with a value of hemelb::lb::Unstable - * if any child node is unstable) and start passing it back down the tree. + * Wait on the collectives to finish. */ - void TopNodeAction() + void Wait(void) { - mDownwardsStability = mUpwardsStability; + timings[hemelb::reporting::Timers::mpiWait].Start(); + HEMELB_MPI_CALL( + MPI_Wait, + (&collectiveReq, MPI_STATUS_IGNORE) + ); + timings[hemelb::reporting::Timers::mpiWait].Stop(); } /** - * Override the method from the base class to use the data from child nodes. + * Computes the relative difference between the densities at the beginning and end of a + * timestep, i.e. |(rho_new - rho_old) / (rho_old - rho_0)|. + * + * @param fNew Distribution function after stream and collide, i.e. solution of the current timestep. + * @param fOld Distribution function at the end of the previous timestep. + * @return relative difference between the densities computed from fNew and fOld. */ - void PostReceiveFromChildren(unsigned long splayNumber) + inline double ComputeRelativeDifference(const distribn_t* fNew, + const distribn_t* fOld) const { - timings[hemelb::reporting::Timers::monitoring].Start(); - - // No need to test children's stability if this node is already unstable. - if (mUpwardsStability != Unstable) + distribn_t new_density = 0.; + distribn_t old_density = 0.; + for (unsigned int l = 0; l < LatticeType::NUMVECTORS; l++) { - for (int ii = 0; ii < (int) SPREADFACTOR; ii++) - { - if (mChildrensStability[ii] == Unstable) - { - mUpwardsStability = Unstable; - break; - } - } - - // If the simulation wasn't found to be unstable and we need to check for convergence, do it now. - if ( (mUpwardsStability != Unstable) && testerConfig->doConvergenceCheck) - { - bool anyStableNotConverged = false; - bool anyConverged = false; - - // mChildrensStability will contain UndefinedStability for non-existent children - for (int ii = 0; ii < (int) SPREADFACTOR; ii++) - { - if (mChildrensStability[ii] == StableAndConverged) - { - anyConverged = true; - } - - if (mChildrensStability[ii] == Stable) - { - anyStableNotConverged = true; - } - } + new_density += fNew[l]; + old_density += fOld[l]; + } - // With the current configuration the root node of the tree won't own any fluid sites. Its - // state only depends on children nodes not on local state. - if (anyConverged - && (mUpwardsStability == StableAndConverged || GetParent() == NOPARENT)) - { - mUpwardsStability = StableAndConverged; - } + // This is equivalent to REFERENCE_PRESSURE_mmHg in lattice units + distribn_t ref_density = 1.0; - if (anyStableNotConverged) - { - mUpwardsStability = Stable; - } - } + if (old_density == ref_density) + { + // We want to avoid returning inf if the site is at pressure = REFERENCE_PRESSURE_mmHg + return 0.0; } - timings[hemelb::reporting::Timers::monitoring].Stop(); + return fabs( (new_density - old_density) / (old_density - ref_density)); } /** @@ -277,29 +183,19 @@ namespace hemelb */ void Effect() { - mSimState->SetStability((Stability) mDownwardsStability); + mSimState->SetStability((Stability) globalStability); } private: - /** - * Slightly arbitrary spread factor for the tree. - */ - static const unsigned int SPREADFACTOR = 10; const geometry::LatticeData * mLatDat; /** - * Stability value of this node and its children to propagate upwards. - */ - int mUpwardsStability; - /** - * Stability value as understood by the root node, to pass downwards. + * Local and global stability. */ - int mDownwardsStability; - /** - * Array for storing the passed-up stability values from child nodes. - */ - int mChildrensStability[SPREADFACTOR]; + int localStability; + int globalStability; + /** * Pointer to the simulation state used in the rest of the simulation. */ @@ -308,8 +204,19 @@ namespace hemelb /** Timing object. */ reporting::Timers& timings; - /** Object containing the user-provided configuration for this class */ - const hemelb::configuration::SimConfig::MonitoringConfig* testerConfig; + /** Whether to check for steady flow simulation convergence */ + bool checkForConvergence; + + /** Relative error tolerance in convergence check */ + double relativeTolerance; + /** + * Private communicator for non-blocking collectives. + */ + net::MpiCommunicator collectiveComm; + /** + * Request object for the collective + */ + MPI_Request collectiveReq; }; } } From d057876eb5aeed6bc418a364cc30e2a5048cfeda Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 21 Apr 2014 18:06:02 +0100 Subject: [PATCH 07/99] Move the stability tester implementation into an .hpp file --- Code/lb/StabilityTester.h | 138 +++----------------------- Code/lb/StabilityTester.hpp | 186 ++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+), 127 deletions(-) create mode 100644 Code/lb/StabilityTester.hpp diff --git a/Code/lb/StabilityTester.h b/Code/lb/StabilityTester.h index 255d909ab..7cb41e55a 100644 --- a/Code/lb/StabilityTester.h +++ b/Code/lb/StabilityTester.h @@ -27,125 +27,31 @@ namespace hemelb public: StabilityTester(const geometry::LatticeData * iLatDat, net::Net* net, SimulationState* simState, reporting::Timers& timings, - bool checkForConvergence, double relativeTolerance) : - mLatDat(iLatDat), - mSimState(simState), timings(timings), checkForConvergence(checkForConvergence), - relativeTolerance(relativeTolerance), collectiveComm(net->GetCommunicator().Duplicate()) - { - Reset(); - } + bool checkForConvergence, double relativeTolerance); /** * Override the reset method in the base class, to reset the stability variables. */ - void Reset() - { - localStability = UndefinedStability; - globalStability = UndefinedStability; - mSimState->SetStability(UndefinedStability); - } - - bool CallAction(int action) - { - switch (static_cast(action)) - { - case net::phased::steps::PreSend: - PreSend(); - return true; - case net::phased::steps::Send: - Send(); - return true; - case net::phased::steps::Wait: - Wait(); - return true; - case net::phased::steps::EndPhase: - Effect(); - return true; - default: - return false; - } - } + void Reset(); + + bool CallAction(int action); protected: /** * Compute the local stability/convergence state. */ - void PreSend(void) - { - timings[hemelb::reporting::Timers::monitoring].Start(); - bool unconvergedSitePresent = false; - bool checkConvThisTimeStep = checkForConvergence; - localStability = Stable; - - for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) - { - for (unsigned int l = 0; l < LatticeType::NUMVECTORS; l++) - { - distribn_t value = *mLatDat->GetFNew(i * LatticeType::NUMVECTORS + l); - - // Note that by testing for value > 0.0, we also catch stray NaNs. - if (! (value > 0.0)) - { - localStability = Unstable; - break; - } - } - ///@todo: If we refactor the previous loop out, we can get away with a single break statement - if (localStability == Unstable) - break; - - if (checkConvThisTimeStep) - { - distribn_t relativeDifference = - ComputeRelativeDifference(mLatDat->GetFNew(i * LatticeType::NUMVECTORS), - mLatDat->GetSite(i).GetFOld()); - - if (relativeDifference > relativeTolerance) - { - // The simulation is stable but hasn't converged in the whole domain yet. - unconvergedSitePresent = true; - // Skip further tests for this timestep - checkConvThisTimeStep = false; - } - } - } - - if (localStability == Stable) - { - if (checkForConvergence && !unconvergedSitePresent) - { - localStability = StableAndConverged; - } - } - timings[hemelb::reporting::Timers::monitoring].Stop(); - } + void PreSend(void); /** * Initiate the collective. */ - void Send(void) - { - // Begin collective. - HEMELB_MPI_CALL( - MPI_Iallreduce, - (static_cast(&localStability), static_cast(&globalStability), 1, - net::MpiDataType(localStability), MPI_MIN, collectiveComm, &collectiveReq) - ); - } + void Send(void); /** * Wait on the collectives to finish. */ - void Wait(void) - { - timings[hemelb::reporting::Timers::mpiWait].Start(); - HEMELB_MPI_CALL( - MPI_Wait, - (&collectiveReq, MPI_STATUS_IGNORE) - ); - timings[hemelb::reporting::Timers::mpiWait].Stop(); - } + void Wait(void); /** * Computes the relative difference between the densities at the beginning and end of a @@ -155,36 +61,13 @@ namespace hemelb * @param fOld Distribution function at the end of the previous timestep. * @return relative difference between the densities computed from fNew and fOld. */ - inline double ComputeRelativeDifference(const distribn_t* fNew, - const distribn_t* fOld) const - { - distribn_t new_density = 0.; - distribn_t old_density = 0.; - for (unsigned int l = 0; l < LatticeType::NUMVECTORS; l++) - { - new_density += fNew[l]; - old_density += fOld[l]; - } - - // This is equivalent to REFERENCE_PRESSURE_mmHg in lattice units - distribn_t ref_density = 1.0; - - if (old_density == ref_density) - { - // We want to avoid returning inf if the site is at pressure = REFERENCE_PRESSURE_mmHg - return 0.0; - } - - return fabs( (new_density - old_density) / (old_density - ref_density)); - } + double ComputeRelativeDifference(const distribn_t* fNew, + const distribn_t* fOld) const; /** * Apply the stability value sent by the root node to the simulation logic. */ - void Effect() - { - mSimState->SetStability((Stability) globalStability); - } + void Effect(); private: @@ -221,4 +104,5 @@ namespace hemelb } } +#include "lb/StabilityTester.hpp" #endif /* HEMELB_LB_STABILITYTESTER_H */ diff --git a/Code/lb/StabilityTester.hpp b/Code/lb/StabilityTester.hpp new file mode 100644 index 000000000..30e4bd87a --- /dev/null +++ b/Code/lb/StabilityTester.hpp @@ -0,0 +1,186 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_LB_STABILITYTESTER_HPP +#define HEMELB_LB_STABILITYTESTER_HPP + +namespace hemelb +{ + namespace lb + { + /** + * Class to repeatedly assess the stability of the simulation, using non-blocking collective + */ + template + StabilityTester::StabilityTester(const geometry::LatticeData * iLatDat, + net::Net* net, SimulationState* simState, + reporting::Timers& timings, + bool checkForConvergence, + double relativeTolerance) : + mLatDat(iLatDat), mSimState(simState), timings(timings), + checkForConvergence(checkForConvergence), relativeTolerance(relativeTolerance), + collectiveComm(net->GetCommunicator().Duplicate()) + { + Reset(); + } + + /** + * Override the reset method in the base class, to reset the stability variables. + */ + template + void StabilityTester::Reset() + { + localStability = UndefinedStability; + globalStability = UndefinedStability; + mSimState->SetStability(UndefinedStability); + } + + template + bool StabilityTester::CallAction(int action) + { + switch (static_cast(action)) + { + case net::phased::steps::PreSend: + PreSend(); + return true; + case net::phased::steps::Send: + Send(); + return true; + case net::phased::steps::Wait: + Wait(); + return true; + case net::phased::steps::EndPhase: + Effect(); + return true; + default: + return false; + } + } + + /** + * Compute the local stability/convergence state. + */ + template + void StabilityTester::PreSend(void) + { + timings[hemelb::reporting::Timers::monitoring].Start(); + bool unconvergedSitePresent = false; + bool checkConvThisTimeStep = checkForConvergence; + localStability = Stable; + + for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) + { + for (unsigned int l = 0; l < LatticeType::NUMVECTORS; l++) + { + distribn_t value = *mLatDat->GetFNew(i * LatticeType::NUMVECTORS + l); + + // Note that by testing for value > 0.0, we also catch stray NaNs. + if (! (value > 0.0)) + { + localStability = Unstable; + break; + } + } + ///@todo: If we refactor the previous loop out, we can get away with a single break statement + if (localStability == Unstable) + break; + + if (checkConvThisTimeStep) + { + distribn_t relativeDifference = ComputeRelativeDifference(mLatDat->GetFNew(i + * LatticeType::NUMVECTORS), + mLatDat->GetSite(i).GetFOld< + LatticeType>()); + + if (relativeDifference > relativeTolerance) + { + // The simulation is stable but hasn't converged in the whole domain yet. + unconvergedSitePresent = true; + // Skip further tests for this timestep + checkConvThisTimeStep = false; + } + } + } + + if (localStability == Stable) + { + if (checkForConvergence && !unconvergedSitePresent) + { + localStability = StableAndConverged; + } + } + timings[hemelb::reporting::Timers::monitoring].Stop(); + } + + /** + * Initiate the collective. + */ + template + void StabilityTester::Send(void) + { + // Begin collective. + HEMELB_MPI_CALL(MPI_Iallreduce, + (static_cast(&localStability), static_cast(&globalStability), 1, net::MpiDataType(localStability), MPI_MIN, collectiveComm, &collectiveReq)); + } + + /** + * Wait on the collectives to finish. + */ + template + void StabilityTester::Wait(void) + { + timings[hemelb::reporting::Timers::mpiWait].Start(); + HEMELB_MPI_CALL(MPI_Wait, (&collectiveReq, MPI_STATUS_IGNORE)); + timings[hemelb::reporting::Timers::mpiWait].Stop(); + } + + /** + * Computes the relative difference between the densities at the beginning and end of a + * timestep, i.e. |(rho_new - rho_old) / (rho_old - rho_0)|. + * + * @param fNew Distribution function after stream and collide, i.e. solution of the current timestep. + * @param fOld Distribution function at the end of the previous timestep. + * @return relative difference between the densities computed from fNew and fOld. + */ + template + double StabilityTester::ComputeRelativeDifference(const distribn_t* fNew, + const distribn_t* fOld) const + { + distribn_t new_density = 0.; + distribn_t old_density = 0.; + for (unsigned int l = 0; l < LatticeType::NUMVECTORS; l++) + { + new_density += fNew[l]; + old_density += fOld[l]; + } + + // This is equivalent to REFERENCE_PRESSURE_mmHg in lattice units + distribn_t ref_density = 1.0; + + if (old_density == ref_density) + { + // We want to avoid returning inf if the site is at pressure = REFERENCE_PRESSURE_mmHg + return 0.0; + } + + return fabs( (new_density - old_density) / (old_density - ref_density)); + } + + /** + * Apply the stability value sent by the root node to the simulation logic. + */ + template + void StabilityTester::Effect() + { + mSimState->SetStability((Stability) globalStability); + } + } +} + +#endif /* HEMELB_LB_STABILITYTESTER_HPP */ From 6446e555dee0d8b2769f62532762a02e36a564bf Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 21 Apr 2014 18:07:25 +0100 Subject: [PATCH 08/99] MAke the StabilityTester use the wrapped MPI async interface --- Code/lb/StabilityTester.h | 2 +- Code/lb/StabilityTester.hpp | 5 ++--- Code/net/MpiCommunicator.h | 3 +++ Code/net/MpiCommunicator.hpp | 11 +++++++++++ 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/Code/lb/StabilityTester.h b/Code/lb/StabilityTester.h index 7cb41e55a..6d713cd24 100644 --- a/Code/lb/StabilityTester.h +++ b/Code/lb/StabilityTester.h @@ -99,7 +99,7 @@ namespace hemelb /** * Request object for the collective */ - MPI_Request collectiveReq; + net::MpiRequest collectiveReq; }; } } diff --git a/Code/lb/StabilityTester.hpp b/Code/lb/StabilityTester.hpp index 30e4bd87a..8f0da1b07 100644 --- a/Code/lb/StabilityTester.hpp +++ b/Code/lb/StabilityTester.hpp @@ -125,8 +125,7 @@ namespace hemelb void StabilityTester::Send(void) { // Begin collective. - HEMELB_MPI_CALL(MPI_Iallreduce, - (static_cast(&localStability), static_cast(&globalStability), 1, net::MpiDataType(localStability), MPI_MIN, collectiveComm, &collectiveReq)); + collectiveReq = collectiveComm.Iallreduce(localStability, MPI_MIN, globalStability); } /** @@ -136,7 +135,7 @@ namespace hemelb void StabilityTester::Wait(void) { timings[hemelb::reporting::Timers::mpiWait].Start(); - HEMELB_MPI_CALL(MPI_Wait, (&collectiveReq, MPI_STATUS_IGNORE)); + collectiveReq.Wait(); timings[hemelb::reporting::Timers::mpiWait].Stop(); } diff --git a/Code/net/MpiCommunicator.h b/Code/net/MpiCommunicator.h index 462e456a4..efc010d2d 100644 --- a/Code/net/MpiCommunicator.h +++ b/Code/net/MpiCommunicator.h @@ -108,6 +108,9 @@ namespace hemelb template std::vector AllReduce(const std::vector& vals, const MPI_Op& op) const; + template + MpiRequest Iallreduce(const T& val, const MPI_Op& op, T& out) const; + template T Reduce(const T& val, const MPI_Op& op, const int root) const; template diff --git a/Code/net/MpiCommunicator.hpp b/Code/net/MpiCommunicator.hpp index 3e3c1047c..4218f1ffa 100644 --- a/Code/net/MpiCommunicator.hpp +++ b/Code/net/MpiCommunicator.hpp @@ -56,6 +56,17 @@ namespace hemelb return ans; } + template + MpiRequest MpiCommunicator::Iallreduce(const T& val, const MPI_Op& op, T& out) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Iallreduce, + (&val, &out, 1, MpiDataType(), op, *commPtr, &req) + ); + return MpiRequest(req); + } + template T MpiCommunicator::Reduce(const T& val, const MPI_Op& op, const int root) const { From a3b63dbce3e14031bb190360e7ff2d8a79fb6c9a Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 21 Apr 2014 18:07:58 +0100 Subject: [PATCH 09/99] Remove the unphysical Mazzeo collisions used in his BCs --- Code/lb/collisions/Collisions.h | 3 - .../NonZeroVelocityEquilibriumFixedDensity.h | 71 ------- Code/lb/collisions/ZeroVelocityEquilibrium.h | 70 ------- .../ZeroVelocityEquilibriumFixedDensity.h | 66 ------ Code/unittests/lbtests/CollisionTests.h | 191 ------------------ 5 files changed, 401 deletions(-) delete mode 100644 Code/lb/collisions/NonZeroVelocityEquilibriumFixedDensity.h delete mode 100644 Code/lb/collisions/ZeroVelocityEquilibrium.h delete mode 100644 Code/lb/collisions/ZeroVelocityEquilibriumFixedDensity.h diff --git a/Code/lb/collisions/Collisions.h b/Code/lb/collisions/Collisions.h index 257339abd..0c9c0540a 100644 --- a/Code/lb/collisions/Collisions.h +++ b/Code/lb/collisions/Collisions.h @@ -10,9 +10,6 @@ #ifndef HEMELB_LB_COLLISIONS_COLLISIONS_H #define HEMELB_LB_COLLISIONS_COLLISIONS_H -#include "lb/collisions/NonZeroVelocityEquilibriumFixedDensity.h" #include "lb/collisions/Normal.h" -#include "lb/collisions/ZeroVelocityEquilibrium.h" -#include "lb/collisions/ZeroVelocityEquilibriumFixedDensity.h" #endif /* HEMELB_LB_COLLISIONS_COLLISIONS_H */ diff --git a/Code/lb/collisions/NonZeroVelocityEquilibriumFixedDensity.h b/Code/lb/collisions/NonZeroVelocityEquilibriumFixedDensity.h deleted file mode 100644 index 3032d6ae0..000000000 --- a/Code/lb/collisions/NonZeroVelocityEquilibriumFixedDensity.h +++ /dev/null @@ -1,71 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_LB_COLLISIONS_NONZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H -#define HEMELB_LB_COLLISIONS_NONZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H - -#include "lb/collisions/BaseCollision.h" -#include "lb/kernels/BaseKernel.h" - -namespace hemelb -{ - namespace lb - { - namespace collisions - { - template - class NonZeroVelocityEquilibriumFixedDensity : public BaseCollision< - NonZeroVelocityEquilibriumFixedDensity, KernelType> - { - public: - typedef KernelType CKernel; - - NonZeroVelocityEquilibriumFixedDensity(kernels::InitParams& initParams) : - kernel(initParams), boundaryObject(initParams.boundaryObject) - { - - } - - inline void DoCalculatePreCollision(kernels::HydroVars& hydroVars, - const geometry::Site& site) - { - CKernel::LatticeType::CalculateDensityAndMomentum(hydroVars.f, - hydroVars.density, - hydroVars.momentum.x, - hydroVars.momentum.y, - hydroVars.momentum.z); - - // Externally impose a density. Keep a record of the old one so we can scale the - // momentum vector. - distribn_t previousDensity = hydroVars.density; - hydroVars.density = boundaryObject->GetBoundaryDensity(site.GetIoletId()); - - hydroVars.momentum *= (hydroVars.density / previousDensity); - - kernel.CalculateFeq(hydroVars, site.GetIndex()); - } - - inline void DoCollide(const LbmParameters* lbmParams, kernels::HydroVars& iHydroVars) - { - for (Direction direction = 0; direction < CKernel::LatticeType::NUMVECTORS; ++direction) - { - iHydroVars.SetFPostCollision(direction, iHydroVars.GetFEq()[direction]); - } - } - - private: - KernelType kernel; - iolets::BoundaryValues* boundaryObject; - }; - - } - } -} - -#endif /* HEMELB_LB_COLLISIONS_NONZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H */ diff --git a/Code/lb/collisions/ZeroVelocityEquilibrium.h b/Code/lb/collisions/ZeroVelocityEquilibrium.h deleted file mode 100644 index 48353d82b..000000000 --- a/Code/lb/collisions/ZeroVelocityEquilibrium.h +++ /dev/null @@ -1,70 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUM_H -#define HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUM_H - -#include "lb/collisions/BaseCollision.h" -#include "lb/kernels/BaseKernel.h" - -namespace hemelb -{ - namespace lb - { - namespace collisions - { - /** - * Collision operator that maintains the density, set velocity to 0 and fully relaxes - * to equilibrium. - */ - template - class ZeroVelocityEquilibrium : public BaseCollision, KernelType> - { - public: - typedef KernelType CKernel; - - ZeroVelocityEquilibrium(kernels::InitParams& initParams) : - BaseCollision, KernelType>(), kernel(initParams) - { - - } - - inline void DoCalculatePreCollision(kernels::HydroVars& hydroVars, - const geometry::Site& site) - { - hydroVars.density = 0.0; - - for (unsigned int ii = 0; ii < CKernel::LatticeType::NUMVECTORS; ++ii) - { - hydroVars.density += hydroVars.f[ii]; - } - - hydroVars.momentum = util::Vector3D::Zero(); - - kernel.CalculateFeq(hydroVars, site.GetIndex()); - } - - inline void DoCollide(const LbmParameters* lbmParams, kernels::HydroVars& iHydroVars) - { - for (Direction direction = 0; direction < CKernel::LatticeType::NUMVECTORS; ++direction) - { - iHydroVars.SetFPostCollision(direction, iHydroVars.GetFEq()[direction]); - } - } - - private: - KernelType kernel; - - }; - - } - } -} - -#endif /* HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUM_H */ diff --git a/Code/lb/collisions/ZeroVelocityEquilibriumFixedDensity.h b/Code/lb/collisions/ZeroVelocityEquilibriumFixedDensity.h deleted file mode 100644 index 57b48431b..000000000 --- a/Code/lb/collisions/ZeroVelocityEquilibriumFixedDensity.h +++ /dev/null @@ -1,66 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H -#define HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H - -#include "lb/collisions/BaseCollision.h" -#include "lb/kernels/BaseKernel.h" - -namespace hemelb -{ - namespace lb - { - namespace collisions - { - /** - * Collision operator that uses an externally imposed density, zero velocity and relaxes - * fully to equilibrium. - */ - template - class ZeroVelocityEquilibriumFixedDensity : public BaseCollision, - KernelType> - { - public: - typedef KernelType CKernel; - - ZeroVelocityEquilibriumFixedDensity(kernels::InitParams& initParams) : - BaseCollision, KernelType>(), kernel(initParams), boundaryObject(initParams.boundaryObject) - { - } - - inline void DoCalculatePreCollision(kernels::HydroVars& hydroVars, - const geometry::Site& site) - { - hydroVars.density = boundaryObject->GetBoundaryDensity(site.GetIoletId()); - - hydroVars.momentum = util::Vector3D::Zero(); - - kernel.CalculateFeq(hydroVars, site.GetIndex()); - } - - inline void DoCollide(const LbmParameters* lbmParams, kernels::HydroVars& iHydroVars) - { - for (Direction direction = 0; direction < CKernel::LatticeType::NUMVECTORS; ++direction) - { - iHydroVars.SetFPostCollision(direction, iHydroVars.GetFEq()[direction]); - } - } - - private: - KernelType kernel; - iolets::BoundaryValues* boundaryObject; - - }; - - } - } -} - -#endif /* HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H */ diff --git a/Code/unittests/lbtests/CollisionTests.h b/Code/unittests/lbtests/CollisionTests.h index edf2e15a3..c403f88bb 100644 --- a/Code/unittests/lbtests/CollisionTests.h +++ b/Code/unittests/lbtests/CollisionTests.h @@ -33,199 +33,8 @@ namespace hemelb class CollisionTests : public helpers::FourCubeBasedTestFixture { CPPUNIT_TEST_SUITE( CollisionTests); - CPPUNIT_TEST( TestNonZeroVelocityEquilibriumFixedDensity); - CPPUNIT_TEST( TestZeroVelocityEquilibriumFixedDensity); - CPPUNIT_TEST( TestZeroVelocityEquilibrium); CPPUNIT_TEST( TestNormal);CPPUNIT_TEST_SUITE_END(); public: - void TestNonZeroVelocityEquilibriumFixedDensity() - { - lb::iolets::BoundaryValues inletBoundary(geometry::INLET_TYPE, - latDat, - simConfig->GetInlets(), - simState, - Comms(), - *unitConverter); - - initParams.boundaryObject = &inletBoundary; - - lb::collisions::NonZeroVelocityEquilibriumFixedDensity > - nonZeroVFixedDensityILet(initParams); - - distribn_t allowedError = 1e-10; - - // Initialise the fOld and the hydro vars. - distribn_t fOld[lb::lattices::D3Q15::NUMVECTORS]; - - LbTestsHelper::InitialiseAnisotropicTestData(0, fOld); - - lb::kernels::HydroVars > hydroVars(fOld); - - // Test the pre-collision step, which should calculate the correct - // post-collisional density, velocity and equilibrium distribution. - geometry::Site dummySite(0, *latDat); - latDat->SetIoletId(0, 0); - - nonZeroVFixedDensityILet.CalculatePreCollision(hydroVars, dummySite); - - // Calculate the expected density, velocity and f_eq. - distribn_t expectedRho = inletBoundary.GetBoundaryDensity(0); - distribn_t expectedMomentum[3]; - - distribn_t originalRho; - LbTestsHelper::CalculateRhoMomentum(fOld, originalRho, expectedMomentum); - - // Now need to scale the momentum, expectedV, to account for the difference between - // original and enforced densities. - for (unsigned axis = 0; axis < 3; ++axis) - expectedMomentum[axis] *= (expectedRho / originalRho); - - distribn_t expectedFeq[lb::lattices::D3Q15::NUMVECTORS]; - LbTestsHelper::CalculateLBGKEqmF(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq); - - // Compare. - LbTestsHelper::CompareHydros(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq, - "Non-0 velocity eqm fixed density, calculate pre collision", - hydroVars, - allowedError); - - // Next, compare the collision function itself. The result should be the equilibrium - // distribution. - nonZeroVFixedDensityILet.Collide(lbmParams, hydroVars); - - for (unsigned int ii = 0; ii < lb::lattices::D3Q15::NUMVECTORS; ++ii) - { - CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Non-0 velocity eqm fixed density, collide", - expectedFeq[ii], - hydroVars.GetFPostCollision()[ii], - allowedError); - } - } - - void TestZeroVelocityEquilibriumFixedDensity() - { - lb::iolets::BoundaryValues outletBoundary(geometry::OUTLET_TYPE, - latDat, - simConfig->GetOutlets(), - simState, - Comms(), - *unitConverter); - initParams.boundaryObject = &outletBoundary; - - lb::collisions::ZeroVelocityEquilibriumFixedDensity > - zeroVFixedDensityOLet(initParams); - - distribn_t allowedError = 1e-10; - - // Initialise the fOld and the hydro vars. - distribn_t fOld[lb::lattices::D3Q15::NUMVECTORS]; - - LbTestsHelper::InitialiseAnisotropicTestData(0, fOld); - - lb::kernels::HydroVars > hydroVars(fOld); - - // Test the pre-collision step, which should calculate the correct - // post-collisional density, velocity and equilibrium distribution. - latDat->SetIoletId(0, 0); - zeroVFixedDensityOLet.CalculatePreCollision(hydroVars, latDat->GetSite(0)); - - // Calculate the expected density, velocity and f_eq. - distribn_t expectedRho = outletBoundary.GetBoundaryDensity(0); - distribn_t expectedMomentum[3] = { 0., 0., 0. }; - - distribn_t expectedFeq[lb::lattices::D3Q15::NUMVECTORS]; - LbTestsHelper::CalculateLBGKEqmF(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq); - - // Compare. - LbTestsHelper::CompareHydros(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq, - "0 velocity eqm fixed density, calculate pre collision", - hydroVars, - allowedError); - - // Next, compare the collision function itself. The result should be the equilibrium - // distribution. - zeroVFixedDensityOLet.Collide(lbmParams, hydroVars); - - for (unsigned int ii = 0; ii < lb::lattices::D3Q15::NUMVECTORS; ++ii) - { - CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("0 velocity eqm fixed density, collide", - expectedFeq[ii], - hydroVars.GetFPostCollision()[ii], - allowedError); - } - } - - void TestZeroVelocityEquilibrium() - { - lb::collisions::ZeroVelocityEquilibrium > zeroVEqm(initParams); - - distribn_t allowedError = 1e-10; - - // Initialise the fOld and the hydro vars. - distribn_t fOld[lb::lattices::D3Q15::NUMVECTORS]; - - LbTestsHelper::InitialiseAnisotropicTestData(0, fOld); - - lb::kernels::HydroVars > hydroVars(fOld); - - // Test the pre-collision step, which should calculate the correct - // post-collisional density, velocity and equilibrium distribution. - zeroVEqm.CalculatePreCollision(hydroVars, latDat->GetSite(0)); - - // Calculate the expected density, velocity and f_eq. - distribn_t expectedRho = 0.0; - distribn_t expectedMomentum[3] = { 0., 0., 0. }; - - for (unsigned int ii = 0; ii < lb::lattices::D3Q15::NUMVECTORS; ++ii) - { - expectedRho += fOld[ii]; - } - - distribn_t expectedFeq[lb::lattices::D3Q15::NUMVECTORS]; - LbTestsHelper::CalculateLBGKEqmF(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq); - - // Compare. - LbTestsHelper::CompareHydros(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq, - "0 velocity eqm, calculate pre collision", - hydroVars, - allowedError); - - // Next, compare the collision function itself. The result should be the equilibrium - // distribution. - zeroVEqm.Collide(lbmParams, hydroVars); - - for (unsigned int ii = 0; ii < lb::lattices::D3Q15::NUMVECTORS; ++ii) - { - CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("0 velocity eqm, collide", - expectedFeq[ii], - hydroVars.GetFPostCollision()[ii], - allowedError); - } - } void TestNormal() { From 9b963a90e2920a26fde9912dcb95682bc873135a Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 21 Apr 2014 18:08:05 +0100 Subject: [PATCH 10/99] Remove some unneeded specialisations of PhasedBroadcast from the vis::Control class --- Code/vis/Control.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/Code/vis/Control.h b/Code/vis/Control.h index e715a748b..b54705c97 100644 --- a/Code/vis/Control.h +++ b/Code/vis/Control.h @@ -114,8 +114,6 @@ namespace hemelb void InstantBroadcast(unsigned long startIteration); private: - typedef net::PhasedBroadcastIrregular base; - typedef net::PhasedBroadcast deepbase; typedef std::map mapType; typedef std::multimap multimapType; From b07157c0f21828f45dce860b32f202ab75106711 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 21 Apr 2014 18:08:14 +0100 Subject: [PATCH 11/99] remove the DensityTracker class from IncompressibilityChecker --- Code/lb/IncompressibilityChecker.cc | 75 ++++++++++++ Code/lb/IncompressibilityChecker.h | 165 +++++++++++++-------------- Code/lb/IncompressibilityChecker.hpp | 82 ------------- 3 files changed, 157 insertions(+), 165 deletions(-) diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index c7dac833d..0190e1a45 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -14,6 +14,81 @@ namespace hemelb { namespace lb { + DensityTracker::DensityTracker() : + allocatedHere(true) + { + densitiesArray = new distribn_t[DENSITY_TRACKER_SIZE]; + + //! @todo #23 do we have a policy on floating point constants? + densitiesArray[MIN_DENSITY] = DBL_MAX; + densitiesArray[MAX_DENSITY] = -DBL_MAX; + densitiesArray[MAX_VELOCITY_MAGNITUDE] = 0.0; + } + + DensityTracker::DensityTracker(distribn_t* const densityValues) : + densitiesArray(densityValues), allocatedHere(false) + { + } + + DensityTracker::~DensityTracker() + { + if (allocatedHere) + { + delete[] densitiesArray; + } + } + + void DensityTracker::operator=(const DensityTracker& newValues) + { + for (unsigned trackerEntry = 0; trackerEntry < DENSITY_TRACKER_SIZE; trackerEntry++) + { + densitiesArray[trackerEntry] = newValues.GetDensitiesArray()[trackerEntry]; + } + } + + distribn_t& DensityTracker::operator[](DensityTrackerIndices densityIndex) const + { + return densitiesArray[densityIndex]; + } + + distribn_t* DensityTracker::GetDensitiesArray() const + { + return densitiesArray; + } + + void DensityTracker::UpdateDensityTracker(const DensityTracker& newValues) + { + if (newValues[MIN_DENSITY] < densitiesArray[MIN_DENSITY]) + { + densitiesArray[MIN_DENSITY] = newValues[MIN_DENSITY]; + } + if (newValues[MAX_DENSITY] > densitiesArray[MAX_DENSITY]) + { + densitiesArray[MAX_DENSITY] = newValues[MAX_DENSITY]; + } + if (newValues[MAX_VELOCITY_MAGNITUDE] > densitiesArray[MAX_VELOCITY_MAGNITUDE]) + { + densitiesArray[MAX_VELOCITY_MAGNITUDE] = newValues[MAX_VELOCITY_MAGNITUDE]; + } + } + + void DensityTracker::UpdateDensityTracker(distribn_t newDensity, + distribn_t newVelocityMagnitude) + { + if (newDensity < densitiesArray[MIN_DENSITY]) + { + densitiesArray[MIN_DENSITY] = newDensity; + } + if (newDensity > densitiesArray[MAX_DENSITY]) + { + densitiesArray[MAX_DENSITY] = newDensity; + } + if (newVelocityMagnitude > densitiesArray[MAX_VELOCITY_MAGNITUDE]) + { + densitiesArray[MAX_VELOCITY_MAGNITUDE] = newVelocityMagnitude; + } + } + // Explicit instantiation template class IncompressibilityChecker > ; } diff --git a/Code/lb/IncompressibilityChecker.h b/Code/lb/IncompressibilityChecker.h index f1801d0e8..9efb55359 100644 --- a/Code/lb/IncompressibilityChecker.h +++ b/Code/lb/IncompressibilityChecker.h @@ -27,6 +27,87 @@ namespace hemelb */ static const distribn_t REFERENCE_DENSITY = 1.0; + class DensityTracker + { + /** + * This is convenience class that encapsulates fluid magnitudes of interest being tracked across the domain. At the + * moment it tracks maximum and minimum density, it can be extended to accommodate other magnitudes. + */ + public: + /** Number of densities being tracked. Needs to be exposed for send/receive. */ + static const unsigned DENSITY_TRACKER_SIZE = 3; + + /** Identifiers of the densities being tracked. Cardinality must be kept consistent with DENSITY_TRACKER_SIZE */ + typedef enum + { + MIN_DENSITY = 0u, + MAX_DENSITY, + MAX_VELOCITY_MAGNITUDE + } DensityTrackerIndices; + + /** + * Default constructor, initialises the tracker with some large/small min/max densities. + */ + DensityTracker(); + + /** + * Constructor, initialises the tracker with density values provided. + * + * @param densityValues density values used for initialisation + */ + DensityTracker(distribn_t* const densityValues); + + /** + * Destructor + */ + ~DensityTracker(); + + /** + * Updates densities based on the values of another DensityTracker object. + * + * @param newValues density tracker object to copy from + */ + void operator=(const DensityTracker& newValues); + + /** + * Access individually each of the densities tracked. + * + * @param index index of the density of interest (see DensityTrackerIndices enum for a list) + * @return density value + */ + distribn_t& operator[](DensityTrackerIndices densityIndex) const; + + /** + * Returns a pointer to the internal array used to store densities. + * Only to be used for sending/receiving. Use [] operator, instead. + * + * @return pointer to the internal array used to store densities + */ + distribn_t* GetDensitiesArray() const; + + /** + * Updates min/max densities with values in newValue object if they are smaller/larger + * + * @param newValues new values to be considered for an update + */ + void UpdateDensityTracker(const DensityTracker& newValues); + + /** + * Updates min/max densities with value newValue if it is smaller/larger + * + * @param newDensity new density value to be considered for an update + * @param newVelocityMagnitude new velocity magnitude to be considered for an update + */ + void UpdateDensityTracker(distribn_t newDensity, distribn_t newVelocityMagnitude); + + private: + /** Array storing all the densities being tracked */ + distribn_t* densitiesArray; + + /** Keeps track on whether densitiesArray was allocated locally */ + bool allocatedHere; + }; + template class IncompressibilityChecker : public BroadcastPolicy, public reporting::Reportable @@ -36,87 +117,6 @@ namespace hemelb */ public: - class DensityTracker - { - /** - * This is convenience class that encapsulates fluid magnitudes of interest being tracked across the domain. At the - * moment it tracks maximum and minimum density, it can be extended to accommodate other magnitudes. - */ - public: - /** Number of densities being tracked. Needs to be exposed for send/receive. */ - static const unsigned DENSITY_TRACKER_SIZE = 3; - - /** Identifiers of the densities being tracked. Cardinality must be kept consistent with DENSITY_TRACKER_SIZE */ - typedef enum - { - MIN_DENSITY = 0u, - MAX_DENSITY, - MAX_VELOCITY_MAGNITUDE - } DensityTrackerIndices; - - /** - * Default constructor, initialises the tracker with some large/small min/max densities. - */ - DensityTracker(); - - /** - * Constructor, initialises the tracker with density values provided. - * - * @param densityValues density values used for initialisation - */ - DensityTracker(distribn_t* const densityValues); - - /** - * Destructor - */ - ~DensityTracker(); - - /** - * Updates densities based on the values of another DensityTracker object. - * - * @param newValues density tracker object to copy from - */ - void operator=(const DensityTracker& newValues); - - /** - * Access individually each of the densities tracked. - * - * @param index index of the density of interest (see DensityTrackerIndices enum for a list) - * @return density value - */ - distribn_t& operator[](DensityTrackerIndices densityIndex) const; - - /** - * Returns a pointer to the internal array used to store densities. - * Only to be used for sending/receiving. Use [] operator, instead. - * - * @return pointer to the internal array used to store densities - */ - distribn_t* GetDensitiesArray() const; - - /** - * Updates min/max densities with values in newValue object if they are smaller/larger - * - * @param newValues new values to be considered for an update - */ - void UpdateDensityTracker(const DensityTracker& newValues); - - /** - * Updates min/max densities with value newValue if it is smaller/larger - * - * @param newDensity new density value to be considered for an update - * @param newVelocityMagnitude new velocity magnitude to be considered for an update - */ - void UpdateDensityTracker(distribn_t newDensity, distribn_t newVelocityMagnitude); - - private: - /** Array storing all the densities being tracked */ - distribn_t* densitiesArray; - - /** Keeps track on whether densitiesArray was allocated locally */ - bool allocatedHere; - }; - /** * Constructor * @@ -125,8 +125,7 @@ namespace hemelb * @param simState simulation state * @param maximumRelativeDensityDifferenceAllowed maximum density difference allowed in the domain (relative to reference density, default 5%) */ - IncompressibilityChecker(const geometry::LatticeData * latticeData, - net::Net* net, + IncompressibilityChecker(const geometry::LatticeData * latticeData, net::Net* net, SimulationState* simState, lb::MacroscopicPropertyCache& propertyCache, reporting::Timers& timings, diff --git a/Code/lb/IncompressibilityChecker.hpp b/Code/lb/IncompressibilityChecker.hpp index a21e4e3dc..0b2db8271 100644 --- a/Code/lb/IncompressibilityChecker.hpp +++ b/Code/lb/IncompressibilityChecker.hpp @@ -16,88 +16,6 @@ namespace hemelb { namespace lb { - template - IncompressibilityChecker::DensityTracker::DensityTracker() : - allocatedHere(true) - { - densitiesArray = new distribn_t[DENSITY_TRACKER_SIZE]; - - //! @todo #23 do we have a policy on floating point constants? - densitiesArray[MIN_DENSITY] = DBL_MAX; - densitiesArray[MAX_DENSITY] = -DBL_MAX; - densitiesArray[MAX_VELOCITY_MAGNITUDE] = 0.0; - } - - template - IncompressibilityChecker::DensityTracker::DensityTracker(distribn_t* const densityValues) : - densitiesArray(densityValues), allocatedHere(false) - { - } - - template - IncompressibilityChecker::DensityTracker::~DensityTracker() - { - if (allocatedHere) - { - delete[] densitiesArray; - } - } - - template - void IncompressibilityChecker::DensityTracker::operator=(const DensityTracker& newValues) - { - for (unsigned trackerEntry = 0; trackerEntry < DENSITY_TRACKER_SIZE; trackerEntry++) - { - densitiesArray[trackerEntry] = newValues.GetDensitiesArray()[trackerEntry]; - } - } - - template - distribn_t& IncompressibilityChecker::DensityTracker::operator[](DensityTrackerIndices densityIndex) const - { - return densitiesArray[densityIndex]; - } - - template - distribn_t* IncompressibilityChecker::DensityTracker::GetDensitiesArray() const - { - return densitiesArray; - } - - template - void IncompressibilityChecker::DensityTracker::UpdateDensityTracker(const DensityTracker& newValues) - { - if (newValues[MIN_DENSITY] < densitiesArray[MIN_DENSITY]) - { - densitiesArray[MIN_DENSITY] = newValues[MIN_DENSITY]; - } - if (newValues[MAX_DENSITY] > densitiesArray[MAX_DENSITY]) - { - densitiesArray[MAX_DENSITY] = newValues[MAX_DENSITY]; - } - if (newValues[MAX_VELOCITY_MAGNITUDE] > densitiesArray[MAX_VELOCITY_MAGNITUDE]) - { - densitiesArray[MAX_VELOCITY_MAGNITUDE] = newValues[MAX_VELOCITY_MAGNITUDE]; - } - } - - template - void IncompressibilityChecker::DensityTracker::UpdateDensityTracker(distribn_t newDensity, - distribn_t newVelocityMagnitude) - { - if (newDensity < densitiesArray[MIN_DENSITY]) - { - densitiesArray[MIN_DENSITY] = newDensity; - } - if (newDensity > densitiesArray[MAX_DENSITY]) - { - densitiesArray[MAX_DENSITY] = newDensity; - } - if (newVelocityMagnitude > densitiesArray[MAX_VELOCITY_MAGNITUDE]) - { - densitiesArray[MAX_VELOCITY_MAGNITUDE] = newVelocityMagnitude; - } - } template IncompressibilityChecker::IncompressibilityChecker(const geometry::LatticeData * latticeData, From 9ea2cbd843bf0f35ce77e8f9c496139b6a3a081a Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 21 Apr 2014 18:09:34 +0100 Subject: [PATCH 12/99] Create a CollectiveAction base class and make the IncompressibiltyChecker use it --- Code/SimulationMaster.cc | 6 +- Code/SimulationMaster.h | 2 +- Code/lb/IncompressibilityChecker.cc | 153 +++++++++++++++------- Code/lb/IncompressibilityChecker.h | 155 ++++------------------ Code/lb/IncompressibilityChecker.hpp | 187 --------------------------- Code/net/CMakeLists.txt | 1 + Code/net/CollectiveAction.cc | 52 ++++++++ Code/net/CollectiveAction.h | 55 ++++++++ 8 files changed, 236 insertions(+), 375 deletions(-) delete mode 100644 Code/lb/IncompressibilityChecker.hpp create mode 100644 Code/net/CollectiveAction.cc create mode 100644 Code/net/CollectiveAction.h diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index 8c5242fb7..c74479c7f 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -222,8 +222,7 @@ void SimulationMaster::Initialise() if (monitoringConfig->doIncompressibilityCheck) { - incompressibilityChecker = new hemelb::lb::IncompressibilityChecker< - hemelb::net::PhasedBroadcastRegular<> >(latticeData, + incompressibilityChecker = new hemelb::lb::IncompressibilityChecker(latticeData, &communicationNet, simulationState, latticeBoltzmannModel->GetPropertyCache(), @@ -634,8 +633,7 @@ void SimulationMaster::Abort() void SimulationMaster::LogStabilityReport() { - if (monitoringConfig->doIncompressibilityCheck - && incompressibilityChecker->AreDensitiesAvailable()) + if (monitoringConfig->doIncompressibilityCheck) { hemelb::log::Logger::Log("time step %i, tau %.6f, max_relative_press_diff %.3f, Ma %.3f, max_vel_phys %e", simulationState->GetTimeStep(), diff --git a/Code/SimulationMaster.h b/Code/SimulationMaster.h index c799ceeee..214d31fe8 100644 --- a/Code/SimulationMaster.h +++ b/Code/SimulationMaster.h @@ -101,7 +101,7 @@ class SimulationMaster hemelb::lb::StabilityTester* stabilityTester; hemelb::lb::EntropyTester* entropyTester; /** Actor in charge of checking the maximum density difference across the domain */ - hemelb::lb::IncompressibilityChecker >* incompressibilityChecker; + hemelb::lb::IncompressibilityChecker* incompressibilityChecker; hemelb::colloids::ColloidController* colloidController; hemelb::net::Net communicationNet; diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index 0190e1a45..5fc50dea8 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -6,90 +6,143 @@ // file, or any part thereof, other than as allowed by any agreement // specifically made by you with University College London. // - -#include "lb/lattices/D3Q15.h" -#include "lb/IncompressibilityChecker.hpp" +#include "lb/IncompressibilityChecker.h" namespace hemelb { - namespace lb + namespace net { - DensityTracker::DensityTracker() : - allocatedHere(true) + template<> + MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() { - densitiesArray = new distribn_t[DENSITY_TRACKER_SIZE]; + MPI_Datatype dType = lb::IncompressibilityChecker::GetDensityMpiDatatype(); + HEMELB_MPI_CALL(MPI_Type_commit, (&dType)); + return dType; + } + } - //! @todo #23 do we have a policy on floating point constants? - densitiesArray[MIN_DENSITY] = DBL_MAX; - densitiesArray[MAX_DENSITY] = -DBL_MAX; - densitiesArray[MAX_VELOCITY_MAGNITUDE] = 0.0; + namespace lb + { + /** + * Create the MPI Datatype for sending DensityEtc structs. + */ + MPI_Datatype IncompressibilityChecker::GetDensityMpiDatatype() + { + HEMELB_MPI_TYPE_BEGIN(dType, ::hemelb::lb::DensityEtc, 3); + HEMELB_MPI_TYPE_ADD_MEMBER(min); + HEMELB_MPI_TYPE_ADD_MEMBER(max); + HEMELB_MPI_TYPE_ADD_MEMBER(maxVel); + HEMELB_MPI_TYPE_END(dType, ::hemelb::lb::DensityEtc); + return dType; } - DensityTracker::DensityTracker(distribn_t* const densityValues) : - densitiesArray(densityValues), allocatedHere(false) + void IncompressibilityChecker::UpdateDensityEtc(const DensityEtc& in, DensityEtc& inout) { + inout.min = std::min(in.min, inout.min); + inout.max = std::max(in.max, inout.max); + inout.maxVel = std::max(in.maxVel, inout.maxVel); } - DensityTracker::~DensityTracker() + void IncompressibilityChecker::MpiOpUpdateFunc(void* invec, void* inoutvec, int *len, + MPI_Datatype *datatype) { - if (allocatedHere) + const DensityEtc* iv = static_cast(invec); + DensityEtc* iov = static_cast(inoutvec); + for (int i = 0; i < *len; ++i) { - delete[] densitiesArray; + UpdateDensityEtc(iv[i], iov[i]); } } - void DensityTracker::operator=(const DensityTracker& newValues) + IncompressibilityChecker::IncompressibilityChecker( + const geometry::LatticeData * latticeData, net::Net* net, SimulationState* simState, + lb::MacroscopicPropertyCache& propertyCache, reporting::Timers& timings, + distribn_t maximumRelativeDensityDifferenceAllowed) : net::CollectiveAction(net->GetCommunicator(), timings), + mLatDat(latticeData), propertyCache(propertyCache), mSimState(simState), + maximumRelativeDensityDifferenceAllowed(maximumRelativeDensityDifferenceAllowed) { - for (unsigned trackerEntry = 0; trackerEntry < DENSITY_TRACKER_SIZE; trackerEntry++) - { - densitiesArray[trackerEntry] = newValues.GetDensitiesArray()[trackerEntry]; - } + HEMELB_MPI_CALL(MPI_Op_create, (&IncompressibilityChecker::MpiOpUpdateFunc, 1, &reduction)); + localDensity.min = std::numeric_limits::max(); + localDensity.max = std::numeric_limits::min(); + localDensity.maxVel = std::numeric_limits::min(); + globalDensity = localDensity; } - distribn_t& DensityTracker::operator[](DensityTrackerIndices densityIndex) const + IncompressibilityChecker::~IncompressibilityChecker() { - return densitiesArray[densityIndex]; + HEMELB_MPI_CALL(MPI_Op_free, (&reduction)); } - distribn_t* DensityTracker::GetDensitiesArray() const + distribn_t IncompressibilityChecker::GetGlobalSmallestDensity() const { - return densitiesArray; + return globalDensity.min; } - void DensityTracker::UpdateDensityTracker(const DensityTracker& newValues) + distribn_t IncompressibilityChecker::GetGlobalLargestDensity() const { - if (newValues[MIN_DENSITY] < densitiesArray[MIN_DENSITY]) - { - densitiesArray[MIN_DENSITY] = newValues[MIN_DENSITY]; - } - if (newValues[MAX_DENSITY] > densitiesArray[MAX_DENSITY]) - { - densitiesArray[MAX_DENSITY] = newValues[MAX_DENSITY]; - } - if (newValues[MAX_VELOCITY_MAGNITUDE] > densitiesArray[MAX_VELOCITY_MAGNITUDE]) - { - densitiesArray[MAX_VELOCITY_MAGNITUDE] = newValues[MAX_VELOCITY_MAGNITUDE]; - } + return globalDensity.max; } - void DensityTracker::UpdateDensityTracker(distribn_t newDensity, - distribn_t newVelocityMagnitude) + double IncompressibilityChecker::GetMaxRelativeDensityDifference() const { - if (newDensity < densitiesArray[MIN_DENSITY]) - { - densitiesArray[MIN_DENSITY] = newDensity; - } - if (newDensity > densitiesArray[MAX_DENSITY]) + distribn_t maxDensityDiff = GetGlobalLargestDensity() - GetGlobalSmallestDensity(); + assert(maxDensityDiff >= 0.0); + return maxDensityDiff / REFERENCE_DENSITY; + } + + double IncompressibilityChecker::GetMaxRelativeDensityDifferenceAllowed() const + { + return maximumRelativeDensityDifferenceAllowed; + } + + void IncompressibilityChecker::PreSend() + { + for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) { - densitiesArray[MAX_DENSITY] = newDensity; + DensityEtc etc; + etc.min = propertyCache.densityCache.Get(i); + etc.max = propertyCache.densityCache.Get(i); + etc.maxVel = propertyCache.velocityCache.Get(i).GetMagnitude(); + UpdateDensityEtc(etc, localDensity); } - if (newVelocityMagnitude > densitiesArray[MAX_VELOCITY_MAGNITUDE]) + + } + /** + * Initiate the collective. + */ + void IncompressibilityChecker::Send(void) + { + // Begin collective. + collectiveReq = collectiveComm.Iallreduce(localDensity, reduction, globalDensity); + } + + +// void IncompressibilityChecker::Effect(void) +// { +// // No-op. +// } +// + bool IncompressibilityChecker::IsDensityDiffWithinRange() const + { + return (GetMaxRelativeDensityDifference() < maximumRelativeDensityDifferenceAllowed); + } + + void IncompressibilityChecker::Report(ctemplate::TemplateDictionary& dictionary) + { + if (!IsDensityDiffWithinRange()) { - densitiesArray[MAX_VELOCITY_MAGNITUDE] = newVelocityMagnitude; + ctemplate::TemplateDictionary *incomp = dictionary.AddSectionDictionary("DENSITIES"); + incomp->SetFormattedValue("ALLOWED", + "%.1f%%", + GetMaxRelativeDensityDifferenceAllowed() * 100); + incomp->SetFormattedValue("ACTUAL", "%.1f%%", GetMaxRelativeDensityDifference() * 100); } } - // Explicit instantiation - template class IncompressibilityChecker > ; + double IncompressibilityChecker::GetGlobalLargestVelocityMagnitude() const + { + return globalDensity.maxVel; + } + } } diff --git a/Code/lb/IncompressibilityChecker.h b/Code/lb/IncompressibilityChecker.h index 9efb55359..339ee6fab 100644 --- a/Code/lb/IncompressibilityChecker.h +++ b/Code/lb/IncompressibilityChecker.h @@ -10,11 +10,10 @@ #ifndef HEMELB_LB_INCOMPRESSIBILITYCHECKER_H #define HEMELB_LB_INCOMPRESSIBILITYCHECKER_H +#include "net/CollectiveAction.h" #include "geometry/LatticeData.h" #include "lb/MacroscopicPropertyCache.h" -#include "net/PhasedBroadcastRegular.h" #include "reporting/Reportable.h" -#include namespace hemelb { @@ -27,95 +26,15 @@ namespace hemelb */ static const distribn_t REFERENCE_DENSITY = 1.0; - class DensityTracker - { - /** - * This is convenience class that encapsulates fluid magnitudes of interest being tracked across the domain. At the - * moment it tracks maximum and minimum density, it can be extended to accommodate other magnitudes. - */ - public: - /** Number of densities being tracked. Needs to be exposed for send/receive. */ - static const unsigned DENSITY_TRACKER_SIZE = 3; - - /** Identifiers of the densities being tracked. Cardinality must be kept consistent with DENSITY_TRACKER_SIZE */ - typedef enum - { - MIN_DENSITY = 0u, - MAX_DENSITY, - MAX_VELOCITY_MAGNITUDE - } DensityTrackerIndices; - - /** - * Default constructor, initialises the tracker with some large/small min/max densities. - */ - DensityTracker(); - - /** - * Constructor, initialises the tracker with density values provided. - * - * @param densityValues density values used for initialisation - */ - DensityTracker(distribn_t* const densityValues); - - /** - * Destructor - */ - ~DensityTracker(); - - /** - * Updates densities based on the values of another DensityTracker object. - * - * @param newValues density tracker object to copy from - */ - void operator=(const DensityTracker& newValues); - - /** - * Access individually each of the densities tracked. - * - * @param index index of the density of interest (see DensityTrackerIndices enum for a list) - * @return density value - */ - distribn_t& operator[](DensityTrackerIndices densityIndex) const; - - /** - * Returns a pointer to the internal array used to store densities. - * Only to be used for sending/receiving. Use [] operator, instead. - * - * @return pointer to the internal array used to store densities - */ - distribn_t* GetDensitiesArray() const; - - /** - * Updates min/max densities with values in newValue object if they are smaller/larger - * - * @param newValues new values to be considered for an update - */ - void UpdateDensityTracker(const DensityTracker& newValues); - - /** - * Updates min/max densities with value newValue if it is smaller/larger - * - * @param newDensity new density value to be considered for an update - * @param newVelocityMagnitude new velocity magnitude to be considered for an update - */ - void UpdateDensityTracker(distribn_t newDensity, distribn_t newVelocityMagnitude); - - private: - /** Array storing all the densities being tracked */ - distribn_t* densitiesArray; - - /** Keeps track on whether densitiesArray was allocated locally */ - bool allocatedHere; + struct DensityEtc { + double min; + double max; + double maxVel; }; - template - class IncompressibilityChecker : public BroadcastPolicy, + class IncompressibilityChecker : public net::CollectiveAction, public reporting::Reportable { - /** - * This class uses the phased broadcast infrastructure to keep track of the maximum density difference across the domain. - */ - public: /** * Constructor @@ -188,47 +107,23 @@ namespace hemelb */ double GetGlobalLargestVelocityMagnitude() const; - protected: - /** - * Override the methods from the base class to propagate data from the root, and - * to send data about this node and its childrens' density trackers up towards the root. - */ - void ProgressFromChildren(unsigned long splayNumber); - void ProgressFromParent(unsigned long splayNumber); - void ProgressToChildren(unsigned long splayNumber); - void ProgressToParent(unsigned long splayNumber); - - /** - * Take the combined density tracker information and start passing it back down the tree. - */ - void TopNodeAction(); - - /** - * Override the method from the base class to use the data from child nodes. - */ - void PostReceiveFromChildren(unsigned long splayNumber); + static MPI_Datatype GetDensityMpiDatatype(); + protected: /** - * Override the method from the base class for a node in the tree to update its density tracker - * - * @param splayNumber The parameter splayNumber is 0 indexed and less than splay (a template parameter of PhasedBroadcast) - */ - virtual void PostSendToParent(unsigned long splayNumber); + * Compute the local state. + */ + void PreSend(void); - /** - * Use the density tracker sent by the root node. - */ - void Effect(); + /** + * Initiate the collective. + */ + void Send(void); private: - - /** - * Slightly arbitrary spread factor for the tree. - * - * @todo #23 This is defined in StabilityChecker as well, refactor somewhere else? - */ - static const unsigned int SPREADFACTOR = 10u; - + static void UpdateDensityEtc(const DensityEtc& in, DensityEtc& inout); + static void MpiOpUpdateFunc(void* invec, void* inoutvec, + int *len, MPI_Datatype *datatype); /** Pointer to lattice data object. */ const geometry::LatticeData * mLatDat; @@ -238,23 +133,17 @@ namespace hemelb /** Pointer to the simulation state used in the rest of the simulation. */ lb::SimulationState* mSimState; - /** Timing object. */ - reporting::Timers& timings; - /** Maximum density difference allowed in the domain (relative to reference density) */ distribn_t maximumRelativeDensityDifferenceAllowed; /** Density tracker with the densities agreed on. */ - DensityTracker* globalDensityTracker; + DensityEtc globalDensity; /** Density tracker of this node and its children to propagate upwards. */ - DensityTracker upwardsDensityTracker; - - /** Density tracker as agreed at the root node, to pass downwards. */ - DensityTracker downwardsDensityTracker; + DensityEtc localDensity; - /** Array for storing the passed-up densities from child nodes. */ - distribn_t childrenDensitiesSerialised[SPREADFACTOR * DensityTracker::DENSITY_TRACKER_SIZE]; + /** Custom operator for reduction/ */ + MPI_Op reduction; }; } diff --git a/Code/lb/IncompressibilityChecker.hpp b/Code/lb/IncompressibilityChecker.hpp deleted file mode 100644 index 0b2db8271..000000000 --- a/Code/lb/IncompressibilityChecker.hpp +++ /dev/null @@ -1,187 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_LB_INCOMPRESSIBILITYCHECKER_HPP -#define HEMELB_LB_INCOMPRESSIBILITYCHECKER_HPP - -#include "lb/IncompressibilityChecker.h" - -namespace hemelb -{ - namespace lb - { - - template - IncompressibilityChecker::IncompressibilityChecker(const geometry::LatticeData * latticeData, - net::Net* net, - SimulationState* simState, - lb::MacroscopicPropertyCache& propertyCache, - reporting::Timers& timings, - distribn_t maximumRelativeDensityDifferenceAllowed) : - BroadcastPolicy(net, simState, SPREADFACTOR), mLatDat(latticeData), propertyCache(propertyCache), mSimState(simState), timings(timings), maximumRelativeDensityDifferenceAllowed(maximumRelativeDensityDifferenceAllowed), globalDensityTracker(NULL) - { - /* - * childrenDensitiesSerialised must be initialised to something sensible since ReceiveFromChildren won't - * fill it in completely unless the logarithm base SPREADFACTOR of the number of processes is an integer. - */ - for (unsigned leaf_index = 0; leaf_index < SPREADFACTOR; leaf_index++) - { - unsigned offset = leaf_index * DensityTracker::DENSITY_TRACKER_SIZE; - for (unsigned tracker_index = 0; tracker_index < DensityTracker::DENSITY_TRACKER_SIZE; tracker_index++) - { - switch (tracker_index) - { - case DensityTracker::MIN_DENSITY: - case DensityTracker::MAX_DENSITY: - childrenDensitiesSerialised[offset + tracker_index] = REFERENCE_DENSITY; - break; - case DensityTracker::MAX_VELOCITY_MAGNITUDE: - childrenDensitiesSerialised[offset + tracker_index] = 0.0; - break; - default: - // This should never trip. It only occurs when a new entry is added to the density tracker and - // no suitable initialisation is provided. - assert(false); - } - } - } - - } - - template - IncompressibilityChecker::~IncompressibilityChecker() - { - } - - template - distribn_t IncompressibilityChecker::GetGlobalSmallestDensity() const - { - assert(AreDensitiesAvailable()); - return (*globalDensityTracker)[DensityTracker::MIN_DENSITY]; - } - - template - distribn_t IncompressibilityChecker::GetGlobalLargestDensity() const - { - assert(AreDensitiesAvailable()); - return (*globalDensityTracker)[DensityTracker::MAX_DENSITY]; - } - - template - double IncompressibilityChecker::GetMaxRelativeDensityDifference() const - { - distribn_t maxDensityDiff = GetGlobalLargestDensity() - GetGlobalSmallestDensity(); - assert(maxDensityDiff >= 0.0); - return maxDensityDiff / REFERENCE_DENSITY; - } - - template - double IncompressibilityChecker::GetMaxRelativeDensityDifferenceAllowed() const - { - return maximumRelativeDensityDifferenceAllowed; - } - - template - void IncompressibilityChecker::PostReceiveFromChildren(unsigned long splayNumber) - { - timings[hemelb::reporting::Timers::monitoring].Start(); - - for (int childIndex = 0; childIndex < (int) SPREADFACTOR; childIndex++) - { - DensityTracker childDensities(&childrenDensitiesSerialised[childIndex * DensityTracker::DENSITY_TRACKER_SIZE]); - - upwardsDensityTracker.UpdateDensityTracker(childDensities); - } - - timings[hemelb::reporting::Timers::monitoring].Stop(); - } - - template - void IncompressibilityChecker::PostSendToParent(unsigned long splayNumber) - { - timings[hemelb::reporting::Timers::monitoring].Start(); - - for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) - { - upwardsDensityTracker.UpdateDensityTracker(propertyCache.densityCache.Get(i), - propertyCache.velocityCache.Get(i).GetMagnitude()); - } - - timings[hemelb::reporting::Timers::monitoring].Stop(); - } - - template - void IncompressibilityChecker::ProgressFromChildren(unsigned long splayNumber) - { - this->ReceiveFromChildren(childrenDensitiesSerialised, DensityTracker::DENSITY_TRACKER_SIZE); - } - - template - void IncompressibilityChecker::ProgressFromParent(unsigned long splayNumber) - { - this->ReceiveFromParent(downwardsDensityTracker.GetDensitiesArray(), DensityTracker::DENSITY_TRACKER_SIZE); - } - - template - void IncompressibilityChecker::ProgressToChildren(unsigned long splayNumber) - { - this->SendToChildren(downwardsDensityTracker.GetDensitiesArray(), DensityTracker::DENSITY_TRACKER_SIZE); - } - - template - void IncompressibilityChecker::ProgressToParent(unsigned long splayNumber) - { - this->SendToParent(upwardsDensityTracker.GetDensitiesArray(), DensityTracker::DENSITY_TRACKER_SIZE); - } - - template - void IncompressibilityChecker::TopNodeAction() - { - downwardsDensityTracker = upwardsDensityTracker; - } - - template - void IncompressibilityChecker::Effect() - { - globalDensityTracker = &downwardsDensityTracker; - } - - template - bool IncompressibilityChecker::AreDensitiesAvailable() const - { - return (globalDensityTracker != NULL); - } - - template - bool IncompressibilityChecker::IsDensityDiffWithinRange() const - { - return (GetMaxRelativeDensityDifference() < maximumRelativeDensityDifferenceAllowed); - } - - template - void IncompressibilityChecker::Report(ctemplate::TemplateDictionary& dictionary) - { - if (AreDensitiesAvailable() && !IsDensityDiffWithinRange()) - { - ctemplate::TemplateDictionary *incomp = dictionary.AddSectionDictionary("DENSITIES"); - incomp->SetFormattedValue("ALLOWED", "%.1f%%", GetMaxRelativeDensityDifferenceAllowed() * 100); - incomp->SetFormattedValue("ACTUAL", "%.1f%%", GetMaxRelativeDensityDifference() * 100); - } - } - - template - double IncompressibilityChecker::GetGlobalLargestVelocityMagnitude() const - { - assert(AreDensitiesAvailable()); - return (*globalDensityTracker)[DensityTracker::MAX_VELOCITY_MAGNITUDE]; - } - } -} - -#endif /* HEMELB_LB_INCOMPRESSIBILITYCHECKER_HPP */ diff --git a/Code/net/CMakeLists.txt b/Code/net/CMakeLists.txt index 531fb06a2..8088d79f1 100644 --- a/Code/net/CMakeLists.txt +++ b/Code/net/CMakeLists.txt @@ -3,6 +3,7 @@ add_library(hemelb_net MpiCommunicator.cc MpiGroup.cc MpiFile.cc MpiRequest.cc MpiStatus.cc IteratedAction.cc BaseNet.cc + CollectiveAction.cc IOCommunicator.cc mixins/pointpoint/CoalescePointPoint.cc mixins/pointpoint/SeparatedPointPoint.cc diff --git a/Code/net/CollectiveAction.cc b/Code/net/CollectiveAction.cc new file mode 100644 index 000000000..728dd624b --- /dev/null +++ b/Code/net/CollectiveAction.cc @@ -0,0 +1,52 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#include "net/CollectiveAction.h" +#include "net/phased/steps.h" + +namespace hemelb +{ + namespace net + { + CollectiveAction::CollectiveAction(const MpiCommunicator& comm, reporting::Timers& timings_) + : collectiveComm(comm.Duplicate()), timings(timings_), collectiveReq() + { + } + bool CollectiveAction::CallAction(int action) + { + switch (static_cast(action)) + { + case phased::steps::PreSend: + PreSend(); + return true; + case phased::steps::Send: + Send(); + return true; + case phased::steps::Wait: + Wait(); + return true; + case phased::steps::EndPhase: + PostReceive(); + return true; + default: + return false; + } + } + /** + * Wait on the collectives to finish. + */ + void CollectiveAction::Wait(void) + { + timings[hemelb::reporting::Timers::mpiWait].Start(); + collectiveReq.Wait(); + timings[hemelb::reporting::Timers::mpiWait].Stop(); + } + + } +} diff --git a/Code/net/CollectiveAction.h b/Code/net/CollectiveAction.h new file mode 100644 index 000000000..f35949ba2 --- /dev/null +++ b/Code/net/CollectiveAction.h @@ -0,0 +1,55 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_NET_COLLECTIVEACTION_H +#define HEMELB_NET_COLLECTIVEACTION_H + +#include "net/IteratedAction.h" +#include "reporting/Timers.h" +#include "net/mpi.h" + +namespace hemelb +{ + namespace net + { + class CollectiveAction : public IteratedAction + { + public: + bool CallAction(int action); + + /** + * Initiate the collective. + */ + virtual void Send(void) = 0; + + /** + * Wait on the collectives to finish. + */ + virtual void Wait(void); + + protected: + CollectiveAction(const MpiCommunicator& comm, reporting::Timers& timings); + + /** + * Private communicator for non-blocking collectives. + */ + MpiCommunicator collectiveComm; + /** + * Timings for the wait etc. + */ + reporting::Timers& timings; + /** + * Request object for the collective + */ + MpiRequest collectiveReq; + }; + } +} + +#endif /* HEMELB_NET_COLLECTIVEACTION_H */ From 17f369fd36eccb41da317811fb634be35e59598a Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 21 Apr 2014 18:10:06 +0100 Subject: [PATCH 13/99] Make the IncompressibiltyCHecker unittests pass. --- Code/SimulationMaster.h | 2 +- Code/lb/IncompressibilityChecker.cc | 8 +- Code/net/MpiRequest.cc | 8 ++ Code/net/MpiRequest.h | 2 + .../lbtests/IncompressibilityCheckerTests.h | 115 +++++++++--------- Code/unittests/reporting/ReporterTests.h | 2 +- 6 files changed, 76 insertions(+), 61 deletions(-) diff --git a/Code/SimulationMaster.h b/Code/SimulationMaster.h index 214d31fe8..b8df57df6 100644 --- a/Code/SimulationMaster.h +++ b/Code/SimulationMaster.h @@ -24,7 +24,7 @@ #include "reporting/Reporter.h" #include "reporting/Timers.h" #include "reporting/BuildInfo.h" -#include "lb/IncompressibilityChecker.hpp" +#include "lb/IncompressibilityChecker.h" #include "colloids/ColloidController.h" #include "net/phased/StepManager.h" #include "net/phased/NetConcern.h" diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index 5fc50dea8..569fe3e4e 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -63,8 +63,8 @@ namespace hemelb { HEMELB_MPI_CALL(MPI_Op_create, (&IncompressibilityChecker::MpiOpUpdateFunc, 1, &reduction)); localDensity.min = std::numeric_limits::max(); - localDensity.max = std::numeric_limits::min(); - localDensity.maxVel = std::numeric_limits::min(); + localDensity.max = std::numeric_limits::lowest(); + localDensity.maxVel = 0; globalDensity = localDensity; } @@ -86,7 +86,9 @@ namespace hemelb double IncompressibilityChecker::GetMaxRelativeDensityDifference() const { distribn_t maxDensityDiff = GetGlobalLargestDensity() - GetGlobalSmallestDensity(); - assert(maxDensityDiff >= 0.0); + if (maxDensityDiff < 0.0) + return std::numeric_limits::quiet_NaN(); + return maxDensityDiff / REFERENCE_DENSITY; } diff --git a/Code/net/MpiRequest.cc b/Code/net/MpiRequest.cc index 854a6be25..67451fa5c 100644 --- a/Code/net/MpiRequest.cc +++ b/Code/net/MpiRequest.cc @@ -24,6 +24,14 @@ namespace hemelb reqPtr.reset(new MPI_Request(req)); } + MpiRequest::operator bool() const + { + if ((bool)reqPtr) + return *reqPtr != MPI_REQUEST_NULL; + + return false; + } + void MpiRequest::Wait() { HEMELB_MPI_CALL(MPI_Wait, (reqPtr.get(), MPI_STATUS_IGNORE)); diff --git a/Code/net/MpiRequest.h b/Code/net/MpiRequest.h index 0b8fee7de..f7098cecd 100644 --- a/Code/net/MpiRequest.h +++ b/Code/net/MpiRequest.h @@ -41,6 +41,8 @@ namespace hemelb return *reqPtr; } + operator bool() const; + void Wait(); void Wait(MpiStatus& stat); diff --git a/Code/unittests/lbtests/IncompressibilityCheckerTests.h b/Code/unittests/lbtests/IncompressibilityCheckerTests.h index 63f9c0486..a5f0d0bab 100644 --- a/Code/unittests/lbtests/IncompressibilityCheckerTests.h +++ b/Code/unittests/lbtests/IncompressibilityCheckerTests.h @@ -12,12 +12,15 @@ #include -#include "lb/IncompressibilityChecker.hpp" +#include "lb/IncompressibilityChecker.h" +#include "net/phased/steps.h" + #include "unittests/FourCubeLatticeData.h" #include "unittests/lbtests/BroadcastMocks.h" #include "unittests/reporting/Mocks.h" #include "unittests/helpers/FourCubeBasedTestFixture.h" +#include "debug/Debugger.h" namespace hemelb { @@ -29,8 +32,7 @@ namespace hemelb { CPPUNIT_TEST_SUITE (IncompressibilityCheckerTests); CPPUNIT_TEST (TestIncompressibilityCheckerRootNode); - CPPUNIT_TEST (TestIncompressibilityCheckerLeafNode); - CPPUNIT_TEST_SUITE_END(); + CPPUNIT_TEST (TestIncompressibilityCheckerLeafNode);CPPUNIT_TEST_SUITE_END(); public: void setUp() @@ -41,7 +43,9 @@ namespace hemelb cache->densityCache.SetRefreshFlag(); cache->velocityCache.SetRefreshFlag(); - lbtests::LbTestsHelper::UpdatePropertyCache(*latDat, *cache, *simState); + lbtests::LbTestsHelper::UpdatePropertyCache(*latDat, + *cache, + *simState); // These are the smallest and largest density values in FourCubeLatticeData by default //! @23 The lattice class below must be consistent with the one used in FourCubeLatticeData. Consider templating FourCubeLatticeData over lattice class, so both can be controlled from the test. @@ -66,31 +70,30 @@ namespace hemelb void AdvanceActorOneTimeStep(net::IteratedAction& actor) { - cache->densityCache.SetRefreshFlag(); - LbTestsHelper::UpdatePropertyCache(*latDat, *cache, *simState); - - actor.RequestComms(); - actor.PreSend(); - actor.PreReceive(); - actor.PostReceive(); - actor.EndIteration(); + for (int phase = net::phased::steps::BeginPhase; phase <= net::phased::steps::EndPhase; ++phase) + { + actor.CallAction(phase); + } } void TestIncompressibilityCheckerRootNode() { - hemelb::lb::IncompressibilityChecker incompChecker(latDat, - net, - simState, - *cache, - *timings, - 10.0); // Will accept a max/min of (21.45, 12) but not (100,1) + hemelb::lb::IncompressibilityChecker incompChecker(latDat, + net, + simState, + *cache, + *timings, + 10.0); // Will accept a max/min of (21.45, 12) but not (100,1) // Not available until the first broadcast has finished - CPPUNIT_ASSERT(!incompChecker.AreDensitiesAvailable()); AdvanceActorOneTimeStep(incompChecker); - CPPUNIT_ASSERT_DOUBLES_EQUAL(smallestDefaultDensity, incompChecker.GetGlobalSmallestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(largestDefaultDensity, incompChecker.GetGlobalLargestDensity(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(smallestDefaultDensity, + incompChecker.GetGlobalSmallestDensity(), + eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(largestDefaultDensity, + incompChecker.GetGlobalLargestDensity(), + eps); CPPUNIT_ASSERT_DOUBLES_EQUAL( (largestDefaultDensity - smallestDefaultDensity) / hemelb::lb::REFERENCE_DENSITY, incompChecker.GetMaxRelativeDensityDifference(), @@ -100,48 +103,60 @@ namespace hemelb incompChecker.GetGlobalLargestVelocityMagnitude(), eps); - // The broadcast mock injects some smaller and larger densities (1,100) coming from one of the children - CPPUNIT_ASSERT(incompChecker.AreDensitiesAvailable()); + // Insert values at 1 & 100 + debug::Debugger::Get()->BreakHere(); + cache->densityCache.Put(0, 1.0); + cache->densityCache.Put(1, 100.0); + cache->velocityCache.Put(2, LatticeVelocity(10.0, 0., 0.0)); + AdvanceActorOneTimeStep(incompChecker); CPPUNIT_ASSERT_DOUBLES_EQUAL(1.0, incompChecker.GetGlobalSmallestDensity(), eps); CPPUNIT_ASSERT_DOUBLES_EQUAL(100.0, incompChecker.GetGlobalLargestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(99.0, incompChecker.GetMaxRelativeDensityDifference(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(99.0, + incompChecker.GetMaxRelativeDensityDifference(), + eps); CPPUNIT_ASSERT(!incompChecker.IsDensityDiffWithinRange()); - CPPUNIT_ASSERT_DOUBLES_EQUAL(10.0, incompChecker.GetGlobalLargestVelocityMagnitude(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(10.0, + incompChecker.GetGlobalLargestVelocityMagnitude(), + eps); - // The previous values are not reported by any children anymore. Testing that the checker remembers them - CPPUNIT_ASSERT(incompChecker.AreDensitiesAvailable()); + // Reset the previous values. Testing that the checker remembers the extrema + cache->densityCache.SetRefreshFlag(); + lbtests::LbTestsHelper::UpdatePropertyCache(*latDat, + *cache, + *simState); AdvanceActorOneTimeStep(incompChecker); CPPUNIT_ASSERT_DOUBLES_EQUAL(1.0, incompChecker.GetGlobalSmallestDensity(), eps); CPPUNIT_ASSERT_DOUBLES_EQUAL(100.0, incompChecker.GetGlobalLargestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(99.0, incompChecker.GetMaxRelativeDensityDifference(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(99.0, + incompChecker.GetMaxRelativeDensityDifference(), + eps); CPPUNIT_ASSERT(!incompChecker.IsDensityDiffWithinRange()); - CPPUNIT_ASSERT_DOUBLES_EQUAL(10.0, incompChecker.GetGlobalLargestVelocityMagnitude(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(10.0, + incompChecker.GetGlobalLargestVelocityMagnitude(), + eps); } void TestIncompressibilityCheckerLeafNode() { - hemelb::lb::IncompressibilityChecker incompChecker(latDat, - net, - simState, - *cache, - *timings, - 10.0); // Will accept a max/min of (21.45, 12) but not (100,1) - - // First pass down the tree (uninitialised values being broadcasted) - AdvanceActorOneTimeStep(incompChecker); - - // First pass up the tree (root node gets min/max values reported by the leaf node being simulated) - AdvanceActorOneTimeStep(incompChecker); + hemelb::lb::IncompressibilityChecker incompChecker(latDat, + net, + simState, + *cache, + *timings, + 10.0); // Will accept a max/min of (21.45, 12) but not (100,1) - // Second pass down the tree (all nodes get the min/max values reported by the leaf node being simulated) AdvanceActorOneTimeStep(incompChecker); // This time the current leaf node reported global minimum and maximum values - CPPUNIT_ASSERT_DOUBLES_EQUAL(smallestDefaultDensity, incompChecker.GetGlobalSmallestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(largestDefaultDensity, incompChecker.GetGlobalLargestDensity(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(smallestDefaultDensity, + incompChecker.GetGlobalSmallestDensity(), + eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(largestDefaultDensity, + incompChecker.GetGlobalLargestDensity(), + eps); CPPUNIT_ASSERT_DOUBLES_EQUAL( (largestDefaultDensity - smallestDefaultDensity) / hemelb::lb::REFERENCE_DENSITY, incompChecker.GetMaxRelativeDensityDifference(), @@ -150,18 +165,6 @@ namespace hemelb CPPUNIT_ASSERT_DOUBLES_EQUAL(largestDefaultVelocityMagnitude, incompChecker.GetGlobalLargestVelocityMagnitude(), eps); - - // Second pass up. The broadcast mock injects some smaller and larger densities (1,100) coming from another hypothetical leaf node. - AdvanceActorOneTimeStep(incompChecker); - - // Third pass down. (1,100) arrives to all the leaf nodes. - AdvanceActorOneTimeStep(incompChecker); - - CPPUNIT_ASSERT_DOUBLES_EQUAL(1.0, incompChecker.GetGlobalSmallestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(100.0, incompChecker.GetGlobalLargestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(99.0, incompChecker.GetMaxRelativeDensityDifference(), eps); - CPPUNIT_ASSERT(!incompChecker.IsDensityDiffWithinRange()); - CPPUNIT_ASSERT_DOUBLES_EQUAL(10.0, incompChecker.GetGlobalLargestVelocityMagnitude(), eps); } private: diff --git a/Code/unittests/reporting/ReporterTests.h b/Code/unittests/reporting/ReporterTests.h index 0bef80012..fc36954ff 100644 --- a/Code/unittests/reporting/ReporterTests.h +++ b/Code/unittests/reporting/ReporterTests.h @@ -28,7 +28,7 @@ namespace hemelb using namespace hemelb::reporting; typedef TimersBase TimersMock; - typedef lb::IncompressibilityChecker IncompressibilityCheckerMock; + typedef lb::IncompressibilityChecker IncompressibilityCheckerMock; class ReporterTests : public helpers::HasCommsTestFixture { From f7cb9cc7242b6664ea8c265526f5472ee8ec3aee Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 28 Apr 2014 17:32:44 +0100 Subject: [PATCH 14/99] Actually register the IncompressibilityChecker to do it's communication --- Code/SimulationMaster.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index c74479c7f..8e72d450c 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -332,6 +332,7 @@ void SimulationMaster::Initialise() if (monitoringConfig->doIncompressibilityCheck) { stepManager->RegisterIteratedActorSteps(*incompressibilityChecker, 1); + stepManager->RegisterCommsSteps(*incompressibilityChecker, 1); } stepManager->RegisterIteratedActorSteps(*visualisationControl, 1); if (propertyExtractor != NULL) From b558efacc9ff483acb7593c615a0b0bda7ecab48 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 28 Apr 2014 17:33:16 +0100 Subject: [PATCH 15/99] Make the StabilityTester a CollectiveAction --- Code/lb/StabilityTester.h | 27 ++++--------------------- Code/lb/StabilityTester.hpp | 40 +++---------------------------------- 2 files changed, 7 insertions(+), 60 deletions(-) diff --git a/Code/lb/StabilityTester.h b/Code/lb/StabilityTester.h index 6d713cd24..9b26a9c33 100644 --- a/Code/lb/StabilityTester.h +++ b/Code/lb/StabilityTester.h @@ -10,8 +10,7 @@ #ifndef HEMELB_LB_STABILITYTESTER_H #define HEMELB_LB_STABILITYTESTER_H -#include "net/phased/Concern.h" -#include "net/phased/steps.h" +#include "net/CollectiveAction.h" #include "geometry/LatticeData.h" namespace hemelb @@ -22,7 +21,7 @@ namespace hemelb * Class to repeatedly assess the stability of the simulation, using non-blocking collective */ template - class StabilityTester : public net::phased::Concern + class StabilityTester : public net::CollectiveAction { public: StabilityTester(const geometry::LatticeData * iLatDat, net::Net* net, @@ -35,8 +34,6 @@ namespace hemelb */ void Reset(); - bool CallAction(int action); - protected: /** * Compute the local stability/convergence state. @@ -48,11 +45,6 @@ namespace hemelb */ void Send(void); - /** - * Wait on the collectives to finish. - */ - void Wait(void); - /** * Computes the relative difference between the densities at the beginning and end of a * timestep, i.e. |(rho_new - rho_old) / (rho_old - rho_0)|. @@ -67,11 +59,11 @@ namespace hemelb /** * Apply the stability value sent by the root node to the simulation logic. */ - void Effect(); + void PostReceive(void); private: - const geometry::LatticeData * mLatDat; + const geometry::LatticeData* mLatDat; /** * Local and global stability. @@ -84,22 +76,11 @@ namespace hemelb */ lb::SimulationState* mSimState; - /** Timing object. */ - reporting::Timers& timings; - /** Whether to check for steady flow simulation convergence */ bool checkForConvergence; /** Relative error tolerance in convergence check */ double relativeTolerance; - /** - * Private communicator for non-blocking collectives. - */ - net::MpiCommunicator collectiveComm; - /** - * Request object for the collective - */ - net::MpiRequest collectiveReq; }; } } diff --git a/Code/lb/StabilityTester.hpp b/Code/lb/StabilityTester.hpp index 8f0da1b07..99e53c5ef 100644 --- a/Code/lb/StabilityTester.hpp +++ b/Code/lb/StabilityTester.hpp @@ -23,9 +23,8 @@ namespace hemelb reporting::Timers& timings, bool checkForConvergence, double relativeTolerance) : - mLatDat(iLatDat), mSimState(simState), timings(timings), - checkForConvergence(checkForConvergence), relativeTolerance(relativeTolerance), - collectiveComm(net->GetCommunicator().Duplicate()) + CollectiveAction(net->GetCommunicator(), timings), mLatDat(iLatDat), mSimState(simState), + checkForConvergence(checkForConvergence), relativeTolerance(relativeTolerance) { Reset(); } @@ -41,28 +40,6 @@ namespace hemelb mSimState->SetStability(UndefinedStability); } - template - bool StabilityTester::CallAction(int action) - { - switch (static_cast(action)) - { - case net::phased::steps::PreSend: - PreSend(); - return true; - case net::phased::steps::Send: - Send(); - return true; - case net::phased::steps::Wait: - Wait(); - return true; - case net::phased::steps::EndPhase: - Effect(); - return true; - default: - return false; - } - } - /** * Compute the local stability/convergence state. */ @@ -128,17 +105,6 @@ namespace hemelb collectiveReq = collectiveComm.Iallreduce(localStability, MPI_MIN, globalStability); } - /** - * Wait on the collectives to finish. - */ - template - void StabilityTester::Wait(void) - { - timings[hemelb::reporting::Timers::mpiWait].Start(); - collectiveReq.Wait(); - timings[hemelb::reporting::Timers::mpiWait].Stop(); - } - /** * Computes the relative difference between the densities at the beginning and end of a * timestep, i.e. |(rho_new - rho_old) / (rho_old - rho_0)|. @@ -175,7 +141,7 @@ namespace hemelb * Apply the stability value sent by the root node to the simulation logic. */ template - void StabilityTester::Effect() + void StabilityTester::PostReceive() { mSimState->SetStability((Stability) globalStability); } From 679dbc8f7faeea4c5a971fd100079e4f9be134a3 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 28 Apr 2014 17:33:58 +0100 Subject: [PATCH 16/99] Make the EntropyTester a CollectiveAction instead of a PhasedBroadcast --- Code/lb/EntropyTester.h | 169 +++++++++++++---------------------- Code/net/CollectiveAction.cc | 9 ++ Code/net/CollectiveAction.h | 18 ++-- Code/net/MpiCommunicator.h | 3 + Code/net/MpiCommunicator.hpp | 11 +++ 5 files changed, 93 insertions(+), 117 deletions(-) diff --git a/Code/lb/EntropyTester.h b/Code/lb/EntropyTester.h index 0b2f1fd53..4e39a18a7 100644 --- a/Code/lb/EntropyTester.h +++ b/Code/lb/EntropyTester.h @@ -10,7 +10,7 @@ #ifndef HEMELB_LB_ENTROPYTESTER_H #define HEMELB_LB_ENTROPYTESTER_H -#include "net/PhasedBroadcastRegular.h" +#include "net/CollectiveAction.h" #include "geometry/LatticeData.h" #include "lb/HFunction.h" #include "log/Logger.h" @@ -20,15 +20,14 @@ namespace hemelb namespace lb { template - class EntropyTester : public net::PhasedBroadcastRegular + class EntropyTester : public net::CollectiveAction { public: - EntropyTester(int* collisionTypes, - unsigned int typesTested, - const geometry::LatticeData * iLatDat, - net::Net* net, - SimulationState* simState) : - net::PhasedBroadcastRegular(net, simState, SPREADFACTOR), mLatDat(iLatDat) + EntropyTester(int* collisionTypes, unsigned int typesTested, + const geometry::LatticeData * iLatDat, net::Net* net, + SimulationState* simState, reporting::Timers& timings) : + net::CollectiveAction(net->GetCommunicator(), timings), mLatDat(iLatDat), + mHPreCollision(mLatDat->GetLocalFluidSiteCount()) { for (unsigned int i = 0; i < COLLISION_TYPES; i++) { @@ -39,37 +38,24 @@ namespace hemelb mCollisionTypesTested[collisionTypes[i]] = true; } - mHPreCollision = new double[mLatDat->GetLocalFluidSiteCount()]; - Reset(); } - ~EntropyTester() - { - delete[] mHPreCollision; - } - - void PreReceive() + // Run at BeginPhase + void RequestComms(void) { + // Store pre-collision values. site_t offset = 0; - double dHMax = 0.0; - - // The order of arguments in max is important - // If distributions go negative HFunc.eval() will return a NaN. Due to the - // nature of NaN and the structure of max, dH will be assigned as NaN if this is the case - // This is what we want, because the EntropyTester will fail otherwise and abort when - // it is simply sufficient to wait until StabilityTester restarts. for (unsigned int collision_type = 0; collision_type < COLLISION_TYPES; collision_type++) { if (mCollisionTypesTested[collision_type]) { - for (site_t i = offset; i < offset + mLatDat->GetMidDomainCollisionCount(collision_type); i++) + for (site_t i = offset; + i < offset + mLatDat->GetMidDomainCollisionCount(collision_type); i++) { - const geometry::Site site = mLatDat->GetSite(i); - - HFunction HFunc(site.GetFOld(), NULL); - dHMax = util::NumericalFunctions::max(dHMax, HFunc.eval() - mHPreCollision[i]); + HFunction HFuncOldOld(mLatDat->GetFNew(LatticeType::NUMVECTORS*i), NULL); + mHPreCollision[i] = HFuncOldOld.eval(); } } @@ -80,68 +66,39 @@ namespace hemelb { if (mCollisionTypesTested[collision_type]) { - for (site_t i = offset; i < offset + mLatDat->GetDomainEdgeCollisionCount(collision_type); i++) + for (site_t i = offset; + i < offset + mLatDat->GetDomainEdgeCollisionCount(collision_type); i++) { - const geometry::Site site = mLatDat->GetSite(i); - - HFunction HFunc(site.GetFOld(), NULL); - dHMax = util::NumericalFunctions::max(dHMax, HFunc.eval() - mHPreCollision[i]); + HFunction HFuncOldOld(mLatDat->GetFNew(LatticeType::NUMVECTORS*i), NULL); + mHPreCollision[i] = HFuncOldOld.eval(); } } offset += mLatDat->GetDomainEdgeCollisionCount(collision_type); } - - /* - * Ideally dH should never be greater than zero. However, because H is positive definite accuracy as well as - * rounding and truncation errors can make dH greater than zero in certain cases. The tolerance - * is limited by the accuracy of the simulation (including the accuracy to which alpha is calculated) - * The tolerance has to be at least as big as the accuracy to which alpha is calculated - */ - if (dHMax > 1.0E-6) - { - mUpwardsValue = DISOBEYED; - } - } - - /** - * Override the reset method in the base class, to reset the stability variables. - */ - void Reset() - { - // Re-initialise all values to indicate that the H-theorem is obeyed. - mUpwardsValue = OBEYED; - - for (unsigned int ii = 0; ii < SPREADFACTOR; ii++) - { - mChildrensValues[ii] = OBEYED; - } } - - protected: - /** - * Override the methods from the base class to propagate data from the root, and - * to send data about this node and its childrens' stabilities up towards the root. - */ - void ProgressFromChildren(unsigned long splayNumber) + void PreSend() { - ReceiveFromChildren(mChildrensValues, 1); - } - - void ProgressToParent(unsigned long splayNumber) - { - // Store pre-collision values. site_t offset = 0; + double dHMax = 0.0; + + // The order of arguments in max is important + // If distributions go negative HFunc.eval() will return a NaN. Due to the + // nature of NaN and the structure of max, dH will be assigned as NaN if this is the case + // This is what we want, because the EntropyTester will fail otherwise and abort when + // it is simply sufficient to wait until StabilityTester restarts. for (unsigned int collision_type = 0; collision_type < COLLISION_TYPES; collision_type++) { if (mCollisionTypesTested[collision_type]) { - for (site_t i = offset; i < offset + mLatDat->GetMidDomainCollisionCount(collision_type); i++) + for (site_t i = offset; + i < offset + mLatDat->GetMidDomainCollisionCount(collision_type); i++) { const geometry::Site site = mLatDat->GetSite(i); + HFunction HFunc(site.GetFOld(), NULL); - mHPreCollision[i] = HFunc.eval(); + dHMax = util::NumericalFunctions::max(dHMax, HFunc.eval() - mHPreCollision[i]); } } @@ -152,76 +109,72 @@ namespace hemelb { if (mCollisionTypesTested[collision_type]) { - for (site_t i = offset; i < offset + mLatDat->GetDomainEdgeCollisionCount(collision_type); i++) + for (site_t i = offset; + i < offset + mLatDat->GetDomainEdgeCollisionCount(collision_type); i++) { const geometry::Site site = mLatDat->GetSite(i); + HFunction HFunc(site.GetFOld(), NULL); - mHPreCollision[i] = HFunc.eval(); + dHMax = util::NumericalFunctions::max(dHMax, HFunc.eval() - mHPreCollision[i]); } } offset += mLatDat->GetDomainEdgeCollisionCount(collision_type); } - SendToParent(&mUpwardsValue, 1); + /* + * Ideally dH should never be greater than zero. However, because H is positive definite accuracy as well as + * rounding and truncation errors can make dH greater than zero in certain cases. The tolerance + * is limited by the accuracy of the simulation (including the accuracy to which alpha is calculated) + * The tolerance has to be at least as big as the accuracy to which alpha is calculated + */ + if (dHMax > 1.0E-6) + { + localHTheorem = DISOBEYED; + } + } + void Send(void) + { + collectiveReq = collectiveComm.Ireduce(localHTheorem, MPI_MAX, RootRank, globalHTheorem); } - /** * Take the combined stability information (an int, with a value of hemelb::lb::Unstable * if any child node is unstable) and start passing it back down the tree. */ - void TopNodeAction() + void PostReceive() { - if (mUpwardsValue == DISOBEYED) + if (collectiveComm.Rank() == RootRank && globalHTheorem == DISOBEYED) { log::Logger::Log("H Theorem violated."); } } - /** - * Override the method from the base class to use the data from child nodes. + * Override the reset method in the base class, to reset the stability variables. */ - void PostReceiveFromChildren(unsigned long splayNumber) + void Reset() { - // No need to test children's entropy direction if this node already disobeys H-theorem. - if (mUpwardsValue == OBEYED) - { - for (int ii = 0; ii < (int) SPREADFACTOR; ii++) - { - if (mChildrensValues[ii] == DISOBEYED) - { - mUpwardsValue = DISOBEYED; - break; - } - } - } + // Re-initialise all values to indicate that the H-theorem is obeyed. + localHTheorem = OBEYED; + globalHTheorem = OBEYED; } - private: enum HTHEOREM { - OBEYED, - DISOBEYED + OBEYED = 0, + DISOBEYED = 1 }; + static const int RootRank = 0; /** * Slightly arbitrary spread factor for the tree. */ - static const unsigned int SPREADFACTOR = 10; - const geometry::LatticeData * mLatDat; - /** - * Stability value of this node and its children to propagate upwards. - */ - int mUpwardsValue; - /** - * Array for storing the passed-up stability values from child nodes. - */ - int mChildrensValues[SPREADFACTOR]; + int localHTheorem; + int globalHTheorem; bool mCollisionTypesTested[COLLISION_TYPES]; - double* mHPreCollision; + std::vector mHPreCollision; }; } diff --git a/Code/net/CollectiveAction.cc b/Code/net/CollectiveAction.cc index 728dd624b..6227f7c56 100644 --- a/Code/net/CollectiveAction.cc +++ b/Code/net/CollectiveAction.cc @@ -22,18 +22,27 @@ namespace hemelb { switch (static_cast(action)) { + case phased::steps::BeginPhase: + RequestComms(); + return true; case phased::steps::PreSend: PreSend(); return true; case phased::steps::Send: Send(); return true; + case phased::steps::PreWait: + PreReceive(); + return true; case phased::steps::Wait: Wait(); return true; case phased::steps::EndPhase: PostReceive(); return true; + case phased::steps::EndAll: + EndIteration(); + return true; default: return false; } diff --git a/Code/net/CollectiveAction.h b/Code/net/CollectiveAction.h index f35949ba2..43ce7cce6 100644 --- a/Code/net/CollectiveAction.h +++ b/Code/net/CollectiveAction.h @@ -23,18 +23,18 @@ namespace hemelb public: bool CallAction(int action); - /** - * Initiate the collective. - */ - virtual void Send(void) = 0; + /** + * Initiate the collective. + */ + virtual void Send(void) = 0; - /** - * Wait on the collectives to finish. - */ - virtual void Wait(void); + /** + * Wait on the collectives to finish. + */ + virtual void Wait(void); protected: - CollectiveAction(const MpiCommunicator& comm, reporting::Timers& timings); + CollectiveAction(const MpiCommunicator& comm, reporting::Timers& timings); /** * Private communicator for non-blocking collectives. diff --git a/Code/net/MpiCommunicator.h b/Code/net/MpiCommunicator.h index efc010d2d..a7d56e8e4 100644 --- a/Code/net/MpiCommunicator.h +++ b/Code/net/MpiCommunicator.h @@ -111,6 +111,9 @@ namespace hemelb template MpiRequest Iallreduce(const T& val, const MPI_Op& op, T& out) const; + template + MpiRequest Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const; + template T Reduce(const T& val, const MPI_Op& op, const int root) const; template diff --git a/Code/net/MpiCommunicator.hpp b/Code/net/MpiCommunicator.hpp index 4218f1ffa..350cd8f37 100644 --- a/Code/net/MpiCommunicator.hpp +++ b/Code/net/MpiCommunicator.hpp @@ -67,6 +67,17 @@ namespace hemelb return MpiRequest(req); } + template + MpiRequest MpiCommunicator::Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Ireduce, + (&val, &out, 1, MpiDataType(), op, root, *commPtr, &req) + ); + return MpiRequest(req); + } + template T MpiCommunicator::Reduce(const T& val, const MPI_Op& op, const int root) const { From 9c0a7b88581fa6ce5d6d6e54d378cbd644f1c8ad Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 28 Apr 2014 17:34:55 +0100 Subject: [PATCH 17/99] Make the steering data distribution object (SteeringComponent) a CollectiveAction --- Code/SimulationMaster.cc | 3 +- Code/net/MpiCommunicator.h | 5 ++++ Code/net/MpiCommunicator.hpp | 21 +++++++++++++ Code/steering/SteeringComponent.h | 20 ++++++------- Code/steering/basic/SteeringComponentB.cc | 36 ++++++++++------------- Code/steering/none/SteeringComponentN.cc | 28 +++++++----------- 6 files changed, 63 insertions(+), 50 deletions(-) diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index 8e72d450c..3468f1b95 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -280,7 +280,8 @@ void SimulationMaster::Initialise() &communicationNet, simulationState, simConfig, - unitConverter); + unitConverter, + timings); // Read in the visualisation parameters. latticeBoltzmannModel->ReadVisParameters(); diff --git a/Code/net/MpiCommunicator.h b/Code/net/MpiCommunicator.h index a7d56e8e4..28d59bdf7 100644 --- a/Code/net/MpiCommunicator.h +++ b/Code/net/MpiCommunicator.h @@ -103,6 +103,11 @@ namespace hemelb template void Broadcast(std::vector& vals, const int root) const; + template + MpiRequest Ibcast(T& val, const int root) const; + template + MpiRequest Ibcast(std::vector& vals, const int root) const; + template T AllReduce(const T& val, const MPI_Op& op) const; template diff --git a/Code/net/MpiCommunicator.hpp b/Code/net/MpiCommunicator.hpp index 350cd8f37..515e7256c 100644 --- a/Code/net/MpiCommunicator.hpp +++ b/Code/net/MpiCommunicator.hpp @@ -34,6 +34,27 @@ namespace hemelb ); } + template + MpiRequest MpiCommunicator::Ibcast(T& val, const int root) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Ibcast, + (&val, 1, MpiDataType(), root, *commPtr, &req) + ); + return MpiRequest(req); + } + template + MpiRequest MpiCommunicator::Ibcast(std::vector& vals, const int root) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Ibcast, + (&vals[0], vals.size(), MpiDataType(), root, *commPtr, &req) + ); + return MpiRequest(req); + } + template T MpiCommunicator::AllReduce(const T& val, const MPI_Op& op) const { diff --git a/Code/steering/SteeringComponent.h b/Code/steering/SteeringComponent.h index a3da16883..2a5b6e6c0 100644 --- a/Code/steering/SteeringComponent.h +++ b/Code/steering/SteeringComponent.h @@ -10,7 +10,7 @@ #ifndef HEMELB_STEERING_STEERINGCOMPONENT_H #define HEMELB_STEERING_STEERINGCOMPONENT_H -#include "net/PhasedBroadcastRegular.h" +#include "net/CollectiveAction.h" #include "lb/SimulationState.h" #include "configuration/SimConfig.h" #include "steering/Network.h" @@ -55,7 +55,7 @@ namespace hemelb * we only need to pass from the top-most node (which handles network communication) downwards, * on one iteration between each pair of consecutive depths. */ - class SteeringComponent : public net::PhasedBroadcastRegular + class SteeringComponent : public net::CollectiveAction { public: SteeringComponent(Network* iNetwork, @@ -64,7 +64,8 @@ namespace hemelb net::Net * iNet, lb::SimulationState * iSimState, configuration::SimConfig* iSimConfig, - const util::UnitConverter* iUnits); + const util::UnitConverter* iUnits, + reporting::Timers& timings); static bool RequiresSeparateSteeringCore(); @@ -78,18 +79,15 @@ namespace hemelb bool readyForNextImage; bool updatedMouseCoords; - protected: - void ProgressFromParent(unsigned long splayNumber); - void ProgressToChildren(unsigned long splayNumber); - - void TopNodeAction(); - void Effect(); + void PreSend(); + void Send(); + void PostReceive(); private: void AssignValues(); const static int STEERABLE_PARAMETERS = 21; - const static unsigned int SPREADFACTOR = 10; + const static int RootRank = 0; bool isConnected; @@ -97,7 +95,7 @@ namespace hemelb lb::SimulationState* mSimState; vis::Control* mVisControl; steering::ImageSendComponent* imageSendComponent; - float privateSteeringParams[STEERABLE_PARAMETERS + 1]; + std::vector privateSteeringParams; const util::UnitConverter* mUnits; configuration::SimConfig* simConfig; }; diff --git a/Code/steering/basic/SteeringComponentB.cc b/Code/steering/basic/SteeringComponentB.cc index 1cc715352..d776bf241 100644 --- a/Code/steering/basic/SteeringComponentB.cc +++ b/Code/steering/basic/SteeringComponentB.cc @@ -18,38 +18,30 @@ namespace hemelb { namespace steering { - SteeringComponent::SteeringComponent(Network* iNetwork, - vis::Control* iVisControl, + SteeringComponent::SteeringComponent(Network* iNetwork, vis::Control* iVisControl, steering::ImageSendComponent* imageSendComponent, - net::Net * iNet, - lb::SimulationState * iSimState, + net::Net * iNet, lb::SimulationState * iSimState, configuration::SimConfig* iSimConfig, - const util::UnitConverter* iUnits) : - net::PhasedBroadcastRegular(iNet, iSimState, SPREADFACTOR), - mNetwork(iNetwork), mSimState(iSimState), mVisControl(iVisControl), imageSendComponent(imageSendComponent), - mUnits(iUnits),simConfig(iSimConfig) + const util::UnitConverter* iUnits, + reporting::Timers& timings) : + net::CollectiveAction(iNet->GetCommunicator(), timings), mNetwork(iNetwork), + mSimState(iSimState), mVisControl(iVisControl), imageSendComponent(imageSendComponent), + privateSteeringParams(STEERABLE_PARAMETERS + 1), mUnits(iUnits), simConfig(iSimConfig) { ClearValues(); AssignValues(); } - void SteeringComponent::ProgressFromParent(unsigned long splayNumber) - { - ReceiveFromParent(privateSteeringParams, STEERABLE_PARAMETERS + 1); - } - - void SteeringComponent::ProgressToChildren(unsigned long splayNumber) - { - SendToChildren(privateSteeringParams, STEERABLE_PARAMETERS + 1); - } - bool SteeringComponent::RequiresSeparateSteeringCore() { return true; } - void SteeringComponent::TopNodeAction() + void SteeringComponent::PreSend() { + if (collectiveComm.Rank() != RootRank) + return; + /* * The final steering parameter is DoRendering, which is true if we're connected and ready * for the next frame. @@ -101,8 +93,12 @@ namespace hemelb steeringStream.readFloat(privateSteeringParams[i]); } } + void SteeringComponent::Send() + { + collectiveReq = collectiveComm.Ibcast(privateSteeringParams, RootRank); + } - void SteeringComponent::Effect() + void SteeringComponent::PostReceive() { // TODO we need to make sure that doing this doesn't overwrite the values in the config.xml file. // At the moment, it definitely does. diff --git a/Code/steering/none/SteeringComponentN.cc b/Code/steering/none/SteeringComponentN.cc index 266bdbe82..8dba9bbf7 100644 --- a/Code/steering/none/SteeringComponentN.cc +++ b/Code/steering/none/SteeringComponentN.cc @@ -24,15 +24,15 @@ namespace hemelb * @param iSimState * @return */ - SteeringComponent::SteeringComponent(Network* network, - vis::Control* iVisControl, + SteeringComponent::SteeringComponent(Network* network, vis::Control* iVisControl, steering::ImageSendComponent* imageSendComponent, - net::Net * iNet, - lb::SimulationState * iSimState, + net::Net * iNet, lb::SimulationState * iSimState, configuration::SimConfig* iSimConfig, - const util::UnitConverter* iUnits) : - net::PhasedBroadcastRegular(iNet, iSimState, SPREADFACTOR), - mSimState(iSimState), mVisControl(iVisControl), imageSendComponent(imageSendComponent), mUnits(iUnits), simConfig(iSimConfig) + const util::UnitConverter* iUnits, + reporting::Timers& timings) : + net::CollectiveAction(iNet->GetCommunicator(), timings), mSimState(iSimState), + mVisControl(iVisControl), imageSendComponent(imageSendComponent), mUnits(iUnits), + simConfig(iSimConfig), privateSteeringParams(STEERABLE_PARAMETERS + 1) { ClearValues(); AssignValues(); @@ -43,21 +43,13 @@ namespace hemelb return false; } - void SteeringComponent::ProgressFromParent(unsigned long splayNumber) + void SteeringComponent::PreSend() { - } - void SteeringComponent::ProgressToChildren(unsigned long splayNumber) + void SteeringComponent::Send() { - } - - void SteeringComponent::TopNodeAction() - { - - } - - void SteeringComponent::Effect() + void SteeringComponent::PostReceive() { AssignValues(); } From cac78c89155f965830e59ea5343b49f1dd4b9ffb Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 28 Apr 2014 17:35:31 +0100 Subject: [PATCH 18/99] Remove the PhasedBroadcastRegular class template as it's unused. --- Code/net/PhasedBroadcastRegular.h | 296 ----------------- Code/unittests/lbtests/BroadcastMocks.h | 301 ------------------ .../lbtests/IncompressibilityCheckerTests.h | 1 - 3 files changed, 598 deletions(-) delete mode 100644 Code/net/PhasedBroadcastRegular.h delete mode 100644 Code/unittests/lbtests/BroadcastMocks.h diff --git a/Code/net/PhasedBroadcastRegular.h b/Code/net/PhasedBroadcastRegular.h deleted file mode 100644 index 196ed1489..000000000 --- a/Code/net/PhasedBroadcastRegular.h +++ /dev/null @@ -1,296 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_NET_PHASEDBROADCASTREGULAR_H -#define HEMELB_NET_PHASEDBROADCASTREGULAR_H - -#include "net/PhasedBroadcast.h" - -namespace hemelb -{ - namespace net - { - /** - * PhasedBroadcastRegular - a class for performing phased broadcasts starting at regular - * intervals. A longer description is given in PhasedBroadcast.h. - */ - template - class PhasedBroadcastRegular : public PhasedBroadcast - { - public: - /** - * Constructor that calls the base class's constructor. - * - * @param iNet - * @param iSimState - * @param spreadFactor - * @return - */ - PhasedBroadcastRegular(Net * iNet, const lb::SimulationState * iSimState, unsigned int spreadFactor) : - base(iNet, iSimState, spreadFactor) - { - - } - - /** - * Function that requests all the communications from the Net object. - */ - void RequestComms() - { - const unsigned long iCycleNumber = Get0IndexedIterationNumber(); - const unsigned long firstAscent = base::GetFirstAscending(); - const unsigned long firstDescent = base::GetFirstDescending(); - - // Nothing to do for initial action case. - - // Next, deal with the case of a cycle with an initial pass down the tree. - if (goDown) - { - if (iCycleNumber >= firstDescent && iCycleNumber < firstAscent) - { - unsigned long sendOverlap; - unsigned long receiveOverlap; - - if (base::GetSendChildrenOverlap(iCycleNumber - firstDescent, &sendOverlap)) - { - ProgressToChildren(sendOverlap); - } - - if (base::GetReceiveParentOverlap(iCycleNumber - firstDescent, &receiveOverlap)) - { - ProgressFromParent(receiveOverlap); - } - } - } - - // And deal with the case of a cycle with a pass up the tree. - if (goUp) - { - if (iCycleNumber >= firstAscent) - { - unsigned long sendOverlap; - unsigned long receiveOverlap; - - if (base::GetSendParentOverlap(iCycleNumber - firstAscent, &sendOverlap)) - { - ProgressToParent(sendOverlap); - } - - if (base::GetReceiveChildrenOverlap(iCycleNumber - firstAscent, &receiveOverlap)) - { - ProgressFromChildren(receiveOverlap); - } - } - } - } - - /** - * Function called after send begin but before receives are known to be completed. The - * action performed is limited to the InitialAction, if used. - */ - void PreReceive() - { - // The only thing to do while waiting is the initial action. - if (initialAction) - { - if (Get0IndexedIterationNumber() == 0) - { - InitialAction(); - } - } - } - - /** - * Function to be called after the Receives have completed, where the - * data is used. - */ - void PostReceive() - { - const unsigned long iCycleNumber = Get0IndexedIterationNumber(); - const unsigned long firstAscent = - PhasedBroadcast::GetFirstAscending(); - const unsigned long traversalLength = - PhasedBroadcast::GetTraverseTime(); - - // Deal with the case of a cycle with an initial pass down the tree. - if (goDown) - { - const unsigned long firstDescent = - PhasedBroadcast::GetFirstDescending(); - - if (iCycleNumber >= firstDescent && iCycleNumber < firstAscent) - { - unsigned long receiveOverlap; - - if (base::GetReceiveParentOverlap(iCycleNumber - firstDescent, &receiveOverlap)) - { - PostReceiveFromParent(receiveOverlap); - } - - // If we're halfway through the programme, all top-down changes have occurred and - // can be applied on all nodes at once safely. - if ( (iCycleNumber - firstDescent) == (traversalLength - 1)) - { - Effect(); - } - } - } - - // Deal with the case of a cycle with a pass back up the tree. - if (goUp) - { - if (iCycleNumber >= firstAscent) - { - unsigned long receiveOverlap, sendOverlap; - - if (base::GetReceiveChildrenOverlap(iCycleNumber - firstAscent, &receiveOverlap)) - { - PostReceiveFromChildren(receiveOverlap); - } - - if (base::GetSendParentOverlap(iCycleNumber - firstAscent, &sendOverlap)) - { - PostSendToParent(sendOverlap); - } - - } - } - - // If this node is the root of the tree and we've just finished the upwards half, it - // must act. - if (iCycleNumber == (base::GetRoundTripLength() - 1) - && this->mNet->GetCommunicator().Rank() == 0) - { - TopNodeAction(); - } - } - - /** - * Returns the number of the iteration, as an integer between inclusive-0 and - * exclusive-2 * (the tree depth) - */ - unsigned long Get0IndexedIterationNumber() const - { - if (base::GetTreeDepth() > 0) - { - unsigned long stepsPassed = base::mSimState->Get0IndexedTimeStep(); - - return stepsPassed % base::GetRoundTripLength(); - } - else - { - return 0; - } - } - - protected: - - // Typedef for the base class type, for convenience. - typedef PhasedBroadcast base; - - /** - * Overridable function for the initial action performed by a node at the beginning of the - * cycle. Only has an effect if the template paramter initialAction is true. - */ - virtual void InitialAction() - { - - } - - /** - * Overridable function for when a node has to receive from its children in the tree. - * - * Use ReceiveFromChildren to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressFromChildren(unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to receive from its parent in the tree. - * - * Use ReceiveFromParent to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressFromParent(unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to send to its children in the tree. - * - * Use SendToChildren to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressToChildren(unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to send to its parent in the tree. - * - * Use SendToParent to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressToParent(unsigned long splayNumber) - { - - } - - /** - * Overridable function, called by a node after data has been received from its children. - * The parameter splayNumber is 0 indexed and less than splay. - */ - virtual void PostReceiveFromChildren(unsigned long splayNumber) - { - - } - - /** - * Overridable function, called by a node after data has been sent to its parent. - * - * @param splayNumber The parameter splayNumber is 0 indexed and less than splay. - */ - virtual void PostSendToParent(unsigned long splayNumber) - { - } - - /** - * Overridable function, called by a node after data has been received from its parent. The - * parameter splayNumber is 0 indexed and less than splay. - */ - virtual void PostReceiveFromParent(unsigned long splayNumber) - { - - } - - /** - * Action taken when upwards-travelling data reaches the top node. - */ - virtual void TopNodeAction() - { - - } - - /** - * Action taken by all nodes when downwards-travelling data has been sent to every node. - */ - virtual void Effect() - { - - } - }; - } -} - -#endif /* HEMELB_NET_PHASEDBROADCASTREGULAR_H */ diff --git a/Code/unittests/lbtests/BroadcastMocks.h b/Code/unittests/lbtests/BroadcastMocks.h deleted file mode 100644 index 779729bf8..000000000 --- a/Code/unittests/lbtests/BroadcastMocks.h +++ /dev/null @@ -1,301 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_UNITTESTS_LBTESTS_BROADCASTMOCKS_H -#define HEMELB_UNITTESTS_LBTESTS_BROADCASTMOCKS_H - -#include "net/PhasedBroadcastRegular.h" -#include "lb/SimulationState.h" -#include "net/net.h" - -/** - * This file defines two PhasedBroadcastRegular mocks. One that behaves like the root - * node of the broadcast tree and another for a leaf node. - * - * @todo Split file into .h and .cc ? - */ - -namespace hemelb -{ - namespace net - { - /** - * In this mock, we pretend that the current process is the root node of a phased - * broadcast and that a pair of values is going up the tree. The mock simulates - * three cycles of communications (i.e. three messages travelling up the tree and - * reaching the root node): - * 1) All children report (14,15). - * 2) One child reports (1,100) and the rest (14,15). - * 3) All back to (14,15). - */ - class BroadcastMockRootNode : public net::PhasedBroadcastRegular<> - { - - public: - BroadcastMockRootNode(net::Net * net, const lb::SimulationState * simState, unsigned int spreadFactor); - - virtual ~BroadcastMockRootNode(); - - /* - * Overwritten IteraredAction methods that implement the mock. - */ - void RequestComms(); - void PostReceive(); - void EndIteration(); - - protected: - /** - * Receives data from each child. This is a set length per child, and each child's - * data is inserted contiguously into the provided array. - * - * @param dataStart Pointer to the start of the array. - * @param countPerChild Number of elements to receive per child. - */ - template - void ReceiveFromChildren(T* dataStart, int countPerChild); - - private: - /** Number of children nodes in the tree */ - unsigned spreadFactor; - - /** Number of times that the phased broadcast has been completed */ - unsigned callCounter; - }; - - BroadcastMockRootNode::BroadcastMockRootNode(net::Net * net, - const lb::SimulationState * simState, - unsigned int spreadFactor) : - net::PhasedBroadcastRegular<>(net, simState, spreadFactor), spreadFactor(spreadFactor), callCounter(0) - { - } - - BroadcastMockRootNode::~BroadcastMockRootNode() - { - } - - void BroadcastMockRootNode::RequestComms() - { - // Action taken when a node has to receive from its children in the tree. Calls ReceiveFromChildren mock below. - ProgressFromChildren(0); - } - - void BroadcastMockRootNode::PostReceive() - { - // Action taken after data has been received from its children - PostReceiveFromChildren(0); - - // Action taken after data has been sent to its parent - PostSendToParent(0); - } - - void BroadcastMockRootNode::EndIteration() - { - // Action taken when upwards-travelling data reaches the top node - TopNodeAction(); - - // Action taken by all nodes when downwards-travelling data has been sent to every node. - Effect(); - } - - template - void BroadcastMockRootNode::ReceiveFromChildren(T* dataStart, int countPerChild) - { - assert(countPerChild == 3); - - switch (callCounter++) - { - case 0: - for (unsigned childIndex = 0; childIndex < spreadFactor; childIndex++) - { - dataStart[childIndex * countPerChild] = 14.0; - dataStart[childIndex * countPerChild + 1] = 15.0; - dataStart[childIndex * countPerChild + 2] = 0.01; - } - break; - - case 1: - for (unsigned childIndex = 0; childIndex < spreadFactor - 1; childIndex++) - { - dataStart[childIndex * countPerChild] = 14.0; - dataStart[childIndex * countPerChild + 1] = 15.0; - dataStart[childIndex * countPerChild + 2] = 1.0; - } - dataStart[ (spreadFactor - 1) * countPerChild] = 1.0; - dataStart[ (spreadFactor - 1) * countPerChild + 1] = 100.0; - dataStart[ (spreadFactor - 1) * countPerChild + 2] = 10.0; - break; - - case 2: - for (unsigned childIndex = 0; childIndex < spreadFactor; childIndex++) - { - dataStart[childIndex * countPerChild] = 14.0; - dataStart[childIndex * countPerChild + 1] = 15.0; - dataStart[childIndex * countPerChild + 2] = 0.01; - } - break; - - default: - // Sanity check. Control should never reach this branch - assert(false); - } - } - - /** - * In this mock, we pretend that the current process is a leaf node of a phased - * broadcast and that a pair of values is going down and up the tree. The mock - * simulates three complete executions of the phased broadcast algorithm. For all - * three executions, the current leaf node reports (12, 21.45) to its parent (upwards - * tree pass). For the downwards pass, the current leaf receives: - * 1) unitialised data. - * 2) the min and max values reported by the current leaf node itself (12, 21.45). - * No other hypothetical leaf node has reported any smaller min or larger max densities. - * 3) (1,100). Another hypothetical leaf node reported these values during the previous - * upwards pass. - */ - class BroadcastMockLeafNode : public net::PhasedBroadcastRegular<> - { - - public: - BroadcastMockLeafNode(net::Net * net, const lb::SimulationState * simState, unsigned int spreadFactor); - - virtual ~BroadcastMockLeafNode(); - - /* - * Overwritten IteraredAction methods that implement the mock. - */ - void RequestComms(); - void PostReceive(); - void EndIteration(); - - protected: - /** - * Receives data from each child. This is a set length per child, and each child's - * data is inserted contiguously into the provided array. - * - * @param dataStart Pointer to the start of the array. - * @param countPerChild Number of elements to receive per child. - */ - template - void ReceiveFromParent(T* dataStart, int countPerChild); - - /** - * Helper function for sending data to parent nodes. - */ - template - void SendToParent(T* data, int count); - - private: - /** Number of children nodes in the tree */ - // unsigned spreadFactor; - - /** Number of iterations of the phased broadcast algorithm */ - unsigned iterationCounter; - - /** Minimum and maximum values sent up by the leaf node */ - distribn_t minSentUp, maxSentUp, maxVelSentUp; - - /** - * Helper method to find out whether we are in a downward pass - * - * @param iterationCounter iteration number in the phased broadcast algorithm - * @return whether we are in a downward pass - */ - bool DownwardPass(unsigned iterationCounter); - }; - - BroadcastMockLeafNode::BroadcastMockLeafNode(net::Net * net, - const lb::SimulationState * simState, - unsigned int spreadFactor) : - net::PhasedBroadcastRegular<>(net, simState, spreadFactor), /*spreadFactor(spreadFactor),*/ iterationCounter(0) - { - } - - BroadcastMockLeafNode::~BroadcastMockLeafNode() - { - } - - void BroadcastMockLeafNode::RequestComms() - { - if (DownwardPass(iterationCounter)) - { - // Action taken when a node has to receive from its parent in the tree. Calls ReceiveFromParent mock below. - ProgressFromParent(0); - } - else - { - // Action taken when a node has to send to its parent in the tree. Calls SendToParent mock below - ProgressToParent(0); - } - - iterationCounter++; - } - - void BroadcastMockLeafNode::PostReceive() - { - // Action taken after data has been sent to its parent - PostSendToParent(0); - } - - void BroadcastMockLeafNode::EndIteration() - { - // Action taken by all nodes when downwards-travelling data has been sent to every node. - Effect(); - } - - template - void BroadcastMockLeafNode::ReceiveFromParent(T* dataStart, int countPerChild) - { - assert(countPerChild == 3); - - switch (iterationCounter) - { - case 0: - // The first pass down contains rubbish - dataStart[0] = DBL_MAX; - dataStart[1] = -DBL_MAX; - dataStart[1] = 0; - break; - - case 2: - dataStart[0] = minSentUp; - dataStart[1] = maxSentUp; - dataStart[2] = maxVelSentUp; - break; - - case 4: - dataStart[0] = 1.0; - dataStart[1] = 100.0; - dataStart[2] = 10.0; - break; - - default: - // Sanity check. Control should never reach this branch - assert(false); - } - - } - - template - void BroadcastMockLeafNode::SendToParent(T* data, int count) - { - assert(count == 3); - minSentUp = data[0]; - maxSentUp = data[1]; - maxVelSentUp = data[2]; - } - - bool BroadcastMockLeafNode::DownwardPass(unsigned iterationCounter) - { - return (iterationCounter % 2 == 0); - } - - } -} - -#endif /* HEMELB_UNITTESTS_LBTESTS_BROADCASTMOCKS_H */ diff --git a/Code/unittests/lbtests/IncompressibilityCheckerTests.h b/Code/unittests/lbtests/IncompressibilityCheckerTests.h index a5f0d0bab..e33775698 100644 --- a/Code/unittests/lbtests/IncompressibilityCheckerTests.h +++ b/Code/unittests/lbtests/IncompressibilityCheckerTests.h @@ -16,7 +16,6 @@ #include "net/phased/steps.h" #include "unittests/FourCubeLatticeData.h" -#include "unittests/lbtests/BroadcastMocks.h" #include "unittests/reporting/Mocks.h" #include "unittests/helpers/FourCubeBasedTestFixture.h" From 305cc1934f5955409567ee81e33a49af5b8dd9d0 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 28 Apr 2014 17:37:00 +0100 Subject: [PATCH 19/99] Organise the timers used by the CollectiveActions in the monitoring. --- Code/lb/EntropyTester.h | 12 ++++++++---- Code/lb/IncompressibilityChecker.cc | 14 +++++--------- Code/lb/IncompressibilityChecker.h | 3 +++ Code/lb/StabilityTester.h | 2 ++ Code/lb/StabilityTester.hpp | 9 +++++---- Code/net/CollectiveAction.cc | 8 ++++---- Code/net/CollectiveAction.h | 4 ++-- Code/steering/basic/SteeringComponentB.cc | 7 ++++--- Code/steering/none/SteeringComponentN.cc | 6 +++--- 9 files changed, 36 insertions(+), 29 deletions(-) diff --git a/Code/lb/EntropyTester.h b/Code/lb/EntropyTester.h index 4e39a18a7..e95f63d1f 100644 --- a/Code/lb/EntropyTester.h +++ b/Code/lb/EntropyTester.h @@ -26,8 +26,9 @@ namespace hemelb EntropyTester(int* collisionTypes, unsigned int typesTested, const geometry::LatticeData * iLatDat, net::Net* net, SimulationState* simState, reporting::Timers& timings) : - net::CollectiveAction(net->GetCommunicator(), timings), mLatDat(iLatDat), - mHPreCollision(mLatDat->GetLocalFluidSiteCount()) + net::CollectiveAction(net->GetCommunicator(), timings[reporting::Timers::monitoring]), + mLatDat(iLatDat), mHPreCollision(mLatDat->GetLocalFluidSiteCount()), + workTimer(timings[reporting::Timers::monitoring]) { for (unsigned int i = 0; i < COLLISION_TYPES; i++) { @@ -54,7 +55,8 @@ namespace hemelb for (site_t i = offset; i < offset + mLatDat->GetMidDomainCollisionCount(collision_type); i++) { - HFunction HFuncOldOld(mLatDat->GetFNew(LatticeType::NUMVECTORS*i), NULL); + HFunction HFuncOldOld(mLatDat->GetFNew(LatticeType::NUMVECTORS * i), + NULL); mHPreCollision[i] = HFuncOldOld.eval(); } } @@ -69,7 +71,8 @@ namespace hemelb for (site_t i = offset; i < offset + mLatDat->GetDomainEdgeCollisionCount(collision_type); i++) { - HFunction HFuncOldOld(mLatDat->GetFNew(LatticeType::NUMVECTORS*i), NULL); + HFunction HFuncOldOld(mLatDat->GetFNew(LatticeType::NUMVECTORS * i), + NULL); mHPreCollision[i] = HFuncOldOld.eval(); } } @@ -175,6 +178,7 @@ namespace hemelb bool mCollisionTypesTested[COLLISION_TYPES]; std::vector mHPreCollision; + reporting::Timer& workTimer; }; } diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index 569fe3e4e..04be21da5 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -57,9 +57,11 @@ namespace hemelb IncompressibilityChecker::IncompressibilityChecker( const geometry::LatticeData * latticeData, net::Net* net, SimulationState* simState, lb::MacroscopicPropertyCache& propertyCache, reporting::Timers& timings, - distribn_t maximumRelativeDensityDifferenceAllowed) : net::CollectiveAction(net->GetCommunicator(), timings), - mLatDat(latticeData), propertyCache(propertyCache), mSimState(simState), - maximumRelativeDensityDifferenceAllowed(maximumRelativeDensityDifferenceAllowed) + distribn_t maximumRelativeDensityDifferenceAllowed) : + net::CollectiveAction(net->GetCommunicator(), timings[reporting::Timers::monitoring]), + mLatDat(latticeData), propertyCache(propertyCache), mSimState(simState), + maximumRelativeDensityDifferenceAllowed(maximumRelativeDensityDifferenceAllowed), + workTimer(timings[reporting::Timers::monitoring]) { HEMELB_MPI_CALL(MPI_Op_create, (&IncompressibilityChecker::MpiOpUpdateFunc, 1, &reduction)); localDensity.min = std::numeric_limits::max(); @@ -118,12 +120,6 @@ namespace hemelb collectiveReq = collectiveComm.Iallreduce(localDensity, reduction, globalDensity); } - -// void IncompressibilityChecker::Effect(void) -// { -// // No-op. -// } -// bool IncompressibilityChecker::IsDensityDiffWithinRange() const { return (GetMaxRelativeDensityDifference() < maximumRelativeDensityDifferenceAllowed); diff --git a/Code/lb/IncompressibilityChecker.h b/Code/lb/IncompressibilityChecker.h index 339ee6fab..4a6fb8981 100644 --- a/Code/lb/IncompressibilityChecker.h +++ b/Code/lb/IncompressibilityChecker.h @@ -144,6 +144,9 @@ namespace hemelb /** Custom operator for reduction/ */ MPI_Op reduction; + + /** Time spent checking stuff */ + reporting::Timer& workTimer; }; } diff --git a/Code/lb/StabilityTester.h b/Code/lb/StabilityTester.h index 9b26a9c33..3aa791344 100644 --- a/Code/lb/StabilityTester.h +++ b/Code/lb/StabilityTester.h @@ -81,6 +81,8 @@ namespace hemelb /** Relative error tolerance in convergence check */ double relativeTolerance; + + reporting::Timer& workTimer; }; } } diff --git a/Code/lb/StabilityTester.hpp b/Code/lb/StabilityTester.hpp index 99e53c5ef..b1b66cf02 100644 --- a/Code/lb/StabilityTester.hpp +++ b/Code/lb/StabilityTester.hpp @@ -23,8 +23,9 @@ namespace hemelb reporting::Timers& timings, bool checkForConvergence, double relativeTolerance) : - CollectiveAction(net->GetCommunicator(), timings), mLatDat(iLatDat), mSimState(simState), - checkForConvergence(checkForConvergence), relativeTolerance(relativeTolerance) + CollectiveAction(net->GetCommunicator(), timings[reporting::Timers::monitoring]), + mLatDat(iLatDat), mSimState(simState), checkForConvergence(checkForConvergence), + relativeTolerance(relativeTolerance), workTimer(timings[reporting::Timers::monitoring]) { Reset(); } @@ -46,7 +47,7 @@ namespace hemelb template void StabilityTester::PreSend(void) { - timings[hemelb::reporting::Timers::monitoring].Start(); + workTimer.Start(); bool unconvergedSitePresent = false; bool checkConvThisTimeStep = checkForConvergence; localStability = Stable; @@ -92,7 +93,7 @@ namespace hemelb localStability = StableAndConverged; } } - timings[hemelb::reporting::Timers::monitoring].Stop(); + workTimer.Stop(); } /** diff --git a/Code/net/CollectiveAction.cc b/Code/net/CollectiveAction.cc index 6227f7c56..1f609f16f 100644 --- a/Code/net/CollectiveAction.cc +++ b/Code/net/CollectiveAction.cc @@ -14,8 +14,8 @@ namespace hemelb { namespace net { - CollectiveAction::CollectiveAction(const MpiCommunicator& comm, reporting::Timers& timings_) - : collectiveComm(comm.Duplicate()), timings(timings_), collectiveReq() + CollectiveAction::CollectiveAction(const MpiCommunicator& comm, reporting::Timer& wTimer) : + collectiveComm(comm.Duplicate()), waitTimer(wTimer), collectiveReq() { } bool CollectiveAction::CallAction(int action) @@ -52,9 +52,9 @@ namespace hemelb */ void CollectiveAction::Wait(void) { - timings[hemelb::reporting::Timers::mpiWait].Start(); + waitTimer.Start(); collectiveReq.Wait(); - timings[hemelb::reporting::Timers::mpiWait].Stop(); + waitTimer.Stop(); } } diff --git a/Code/net/CollectiveAction.h b/Code/net/CollectiveAction.h index 43ce7cce6..8b5d72529 100644 --- a/Code/net/CollectiveAction.h +++ b/Code/net/CollectiveAction.h @@ -34,7 +34,7 @@ namespace hemelb virtual void Wait(void); protected: - CollectiveAction(const MpiCommunicator& comm, reporting::Timers& timings); + CollectiveAction(const MpiCommunicator& comm, reporting::Timer& waitTimer); /** * Private communicator for non-blocking collectives. @@ -43,7 +43,7 @@ namespace hemelb /** * Timings for the wait etc. */ - reporting::Timers& timings; + reporting::Timer& waitTimer; /** * Request object for the collective */ diff --git a/Code/steering/basic/SteeringComponentB.cc b/Code/steering/basic/SteeringComponentB.cc index d776bf241..a44d2bba9 100644 --- a/Code/steering/basic/SteeringComponentB.cc +++ b/Code/steering/basic/SteeringComponentB.cc @@ -24,9 +24,10 @@ namespace hemelb configuration::SimConfig* iSimConfig, const util::UnitConverter* iUnits, reporting::Timers& timings) : - net::CollectiveAction(iNet->GetCommunicator(), timings), mNetwork(iNetwork), - mSimState(iSimState), mVisControl(iVisControl), imageSendComponent(imageSendComponent), - privateSteeringParams(STEERABLE_PARAMETERS + 1), mUnits(iUnits), simConfig(iSimConfig) + net::CollectiveAction(iNet->GetCommunicator(), timings[reporting::Timers::steeringWait]), + mNetwork(iNetwork), mSimState(iSimState), mVisControl(iVisControl), + imageSendComponent(imageSendComponent), privateSteeringParams(STEERABLE_PARAMETERS + 1), + mUnits(iUnits), simConfig(iSimConfig) { ClearValues(); AssignValues(); diff --git a/Code/steering/none/SteeringComponentN.cc b/Code/steering/none/SteeringComponentN.cc index 8dba9bbf7..0756ea312 100644 --- a/Code/steering/none/SteeringComponentN.cc +++ b/Code/steering/none/SteeringComponentN.cc @@ -30,9 +30,9 @@ namespace hemelb configuration::SimConfig* iSimConfig, const util::UnitConverter* iUnits, reporting::Timers& timings) : - net::CollectiveAction(iNet->GetCommunicator(), timings), mSimState(iSimState), - mVisControl(iVisControl), imageSendComponent(imageSendComponent), mUnits(iUnits), - simConfig(iSimConfig), privateSteeringParams(STEERABLE_PARAMETERS + 1) + net::CollectiveAction(iNet->GetCommunicator(), timings[reporting::Timers::steeringWait]), + mSimState(iSimState), mVisControl(iVisControl), imageSendComponent(imageSendComponent), + mUnits(iUnits), simConfig(iSimConfig), privateSteeringParams(STEERABLE_PARAMETERS + 1) { ClearValues(); AssignValues(); From 0f096791deb6bef3ca729719b1538948c4be3568 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 2 Jul 2014 10:47:47 +0100 Subject: [PATCH 20/99] Avoid the use of C++11 extensions to the standard library vectors (cbegin and cend member functions) --- .../neighbouring/NeighbouringDataManager.cc | 22 +++++++++---------- Code/lb/IncompressibilityChecker.cc | 2 +- Code/net/MapAllToAll.h | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.cc b/Code/geometry/neighbouring/NeighbouringDataManager.cc index 108d2318a..357fafe32 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.cc +++ b/Code/geometry/neighbouring/NeighbouringDataManager.cc @@ -69,14 +69,14 @@ namespace hemelb net.RequestReceiveR(site.GetWallNormal(), source); } - for (IdsMap::const_iterator iter = needsEachProcHasFromMe.cbegin(); - iter != needsEachProcHasFromMe.cend(); + for (IdsMap::const_iterator iter = needsEachProcHasFromMe.begin(); + iter != needsEachProcHasFromMe.end(); ++iter) { proc_t other = iter->first; const IdVec& neededIds = iter->second; - for (IdVec::const_iterator needOnProcFromMe = neededIds.cbegin(); - needOnProcFromMe != neededIds.cend(); + for (IdVec::const_iterator needOnProcFromMe = neededIds.begin(); + needOnProcFromMe != neededIds.end(); ++needOnProcFromMe) { site_t localContiguousId = @@ -128,13 +128,13 @@ namespace hemelb } const unsigned Q = localLatticeData.GetLatticeInfo().GetNumVectors(); - for (IdsMap::const_iterator iter = needsEachProcHasFromMe.cbegin(); - iter != needsEachProcHasFromMe.cend(); + for (IdsMap::const_iterator iter = needsEachProcHasFromMe.begin(); + iter != needsEachProcHasFromMe.end(); ++iter) { proc_t other = iter->first; const IdVec& neededIds = iter->second; - for (IdVec::const_iterator needOnProcFromMe = neededIds.cbegin(); + for (IdVec::const_iterator needOnProcFromMe = neededIds.begin(); needOnProcFromMe != neededIds.end(); ++needOnProcFromMe) { site_t localContiguousId = @@ -175,8 +175,8 @@ namespace hemelb std::vector requestQueue; // Now, for every rank, which I need something from, send the ids of those - for (CountMap::const_iterator countIt = countOfNeedsIHaveFromEachProc.cbegin(); - countIt != countOfNeedsIHaveFromEachProc.cend(); + for (CountMap::const_iterator countIt = countOfNeedsIHaveFromEachProc.begin(); + countIt != countOfNeedsIHaveFromEachProc.end(); ++countIt) { int other = countIt->first; @@ -184,8 +184,8 @@ namespace hemelb } // And for every rank, which needs something from me, receive those ids - for (CountMap::const_iterator countIt = countOfNeedsOnEachProcFromMe.cbegin(); - countIt != countOfNeedsOnEachProcFromMe.cend(); + for (CountMap::const_iterator countIt = countOfNeedsOnEachProcFromMe.begin(); + countIt != countOfNeedsOnEachProcFromMe.end(); ++countIt) { int other = countIt->first; diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index 04be21da5..2c67ee722 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -65,7 +65,7 @@ namespace hemelb { HEMELB_MPI_CALL(MPI_Op_create, (&IncompressibilityChecker::MpiOpUpdateFunc, 1, &reduction)); localDensity.min = std::numeric_limits::max(); - localDensity.max = std::numeric_limits::lowest(); + localDensity.max = -std::numeric_limits::max(); localDensity.maxVel = 0; globalDensity = localDensity; } diff --git a/Code/net/MapAllToAll.h b/Code/net/MapAllToAll.h index d95c0b520..858cdaf9e 100644 --- a/Code/net/MapAllToAll.h +++ b/Code/net/MapAllToAll.h @@ -35,8 +35,8 @@ namespace hemelb // Set up a container for all the Status objs sendReqs.resize(nSends); - typename std::map::const_iterator iter = valsToSend.cbegin(), - end = valsToSend.cend(); + typename std::map::const_iterator iter = valsToSend.begin(), + end = valsToSend.end(); int i = 0; for (; iter != end; ++iter) { int rank = iter->first; From 63d7f8ea24107b5b06c058485627ea194e3c7f7e Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 28 Aug 2014 11:38:36 +0100 Subject: [PATCH 21/99] Record Archer configuration options in fabric --- deploy/machines.yml | 49 ++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/deploy/machines.yml b/deploy/machines.yml index a3bd610bf..8f37cf256 100644 --- a/deploy/machines.yml +++ b/deploy/machines.yml @@ -77,9 +77,14 @@ archer: #the ARCHER supercomputer at EPCC queue: "standard" python_build: "lib64/python2.6" corespernode: 24 -# build_prefix_command: [ "export LDFLAGS=-dynamic", "export XTPE_LINK_TYPE=dynamic" ] + run_prefix_commands: + # Changes here must also be made below in archer_dmapp + - export MPICH_MAX_THREAD_SAFETY=multiple + - export MPICH_NEMESIS_ASYNC_PROGRESS=1 + - export MPICH_GNI_USE_UNASSIGNED_CPUS=enabled cmake_options: - CMAKE_CXX_FLAGS: "-dynamic" + # Changes here must also be made below in archer_dmapp + CMAKE_CXX_FLAGS: '"-dynamic -Wno-unused-local-typedefs"' CMAKE_C_FLAGS: "-dynamic" HEMELB_USE_ALL_WARNINGS_GNU: "OFF" CMAKE_CXX_COMPILER: "/opt/cray/craype/2.1.1/bin/CC" @@ -93,23 +98,31 @@ archer: #the ARCHER supercomputer at EPCC CTEMPLATE_PATCH_ALIGN: "ON" CTEMPLATE_USE_STATIC: "OFF" HEMELB_COMPUTE_ARCHITECTURE: "INTELSANDYBRIDGE" - # CMAKE_VERBOSE_MAKEFILE: "TRUE" -## CMAKE_SKIP_RPATH: "TRUE" -## CMAKE_SKIP_INSTALL_RPATH: "TRUE" -# CMAKE_CXX_FLAGS: "-dynamic" -# CMAKE_C_FLAGS: "-dynamic" -# CMAKE_VERBOSE_MAKEFILE: "TRUE" -## CPPUNIT_USE_STATIC: "ON" -# CPPUNIT_PATCH_LDL: "OFF" -# CPPUNIT_PATCH_DYNAMIC: "OFF" -## CTEMPLATE_USE_STATIC: "ON" -# CTEMPLATE_PATCH_ALIGN: "ON" -# CTEMPLATE_USE_STATIC: "OFF" -## HEMELB_USE_ALL_WARNINGS_GNU: "OFF" -# HEMELB_OPTIMISATION: "-O3" -# HEMELB_DEPENDENCIES_SET_RPATH: "OFF" -# HEMELB_USE_SSE3: "ON" +archer_dmapp: + import: "archer" + cmake_options: + CMAKE_EXE_LINKER_FLAGS: '"-dynamic $CRAY_DMAPP_POST_LINK_OPTS -ldmapp"' + CMAKE_CXX_FLAGS: '"-dynamic -Wno-unused-local-typedefs"' + CMAKE_C_FLAGS: "-dynamic" + HEMELB_USE_ALL_WARNINGS_GNU: "OFF" + CMAKE_CXX_COMPILER: "/opt/cray/craype/2.1.1/bin/CC" + CMAKE_C_COMPILER: "/opt/cray/craype/2.1.1/bin/cc" + CMAKE_CXX_FLAGS_RELEASE: "" + CPPUNIT_PATCH_LDL: OFF + CPPUNIT_PATCH_DYNAMIC: ON + HEMELB_OPTIMISATION: "-O3" + HEMELB_DEPENDENCIES_SET_RPATH: "OFF" + HEMELB_USE_SSE3: "ON" + CTEMPLATE_PATCH_ALIGN: "ON" + CTEMPLATE_USE_STATIC: "OFF" + HEMELB_COMPUTE_ARCHITECTURE: "INTELSANDYBRIDGE" + run_prefix_commands: + - export MPICH_MAX_THREAD_SAFETY=multiple + - export MPICH_NEMESIS_ASYNC_PROGRESS=1 + - export MPICH_GNI_USE_UNASSIGNED_CPUS=enabled + - export MPICH_USE_DMAPP_COLL=1 + legion: job_dispatch: "qsub" run_command: "mpirun -np $cores -machinefile $$TMPDIR/machines" From 075df29b29c19d324e911748df2b4c90d3e28036 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Sat, 6 Sep 2014 10:53:12 +0100 Subject: [PATCH 22/99] Ensure that the MPI_Wait calls of the CollectiveActions are timed by the mpiWait timer --- Code/lb/EntropyTester.h | 2 +- Code/lb/IncompressibilityChecker.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/lb/EntropyTester.h b/Code/lb/EntropyTester.h index e95f63d1f..5a3280675 100644 --- a/Code/lb/EntropyTester.h +++ b/Code/lb/EntropyTester.h @@ -26,7 +26,7 @@ namespace hemelb EntropyTester(int* collisionTypes, unsigned int typesTested, const geometry::LatticeData * iLatDat, net::Net* net, SimulationState* simState, reporting::Timers& timings) : - net::CollectiveAction(net->GetCommunicator(), timings[reporting::Timers::monitoring]), + net::CollectiveAction(net->GetCommunicator(), timings[reporting::Timers::mpiWait]), mLatDat(iLatDat), mHPreCollision(mLatDat->GetLocalFluidSiteCount()), workTimer(timings[reporting::Timers::monitoring]) { diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index 2c67ee722..e58988a12 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -58,7 +58,7 @@ namespace hemelb const geometry::LatticeData * latticeData, net::Net* net, SimulationState* simState, lb::MacroscopicPropertyCache& propertyCache, reporting::Timers& timings, distribn_t maximumRelativeDensityDifferenceAllowed) : - net::CollectiveAction(net->GetCommunicator(), timings[reporting::Timers::monitoring]), + net::CollectiveAction(net->GetCommunicator(), timings[reporting::Timers::mpiWait]), mLatDat(latticeData), propertyCache(propertyCache), mSimState(simState), maximumRelativeDensityDifferenceAllowed(maximumRelativeDensityDifferenceAllowed), workTimer(timings[reporting::Timers::monitoring]) From 384bedbf97343094642ff309bdec33506b5e301f Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 29 Oct 2015 09:43:51 +0000 Subject: [PATCH 23/99] Fix the debugger start up on OSX Yosemite --- Code/debug/OSX/MPIdebug.applescript | 17 ++++++----------- Code/debug/OSX/resume.lldb | 11 +++++++++++ 2 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 Code/debug/OSX/resume.lldb diff --git a/Code/debug/OSX/MPIdebug.applescript b/Code/debug/OSX/MPIdebug.applescript index 83dd59c1b..15a9bca74 100644 --- a/Code/debug/OSX/MPIdebug.applescript +++ b/Code/debug/OSX/MPIdebug.applescript @@ -14,9 +14,9 @@ on run args end repeat set lldbPath to do shell script "which lldb" - if lldbPath is equal to "" + if lldbPath is equal to "" then set gdbPath to do shell script "which gdb" - if gdbPath is equal to "" + if gdbPath is equal to "" then -- Error - no known debugger! return "Cannot find a debugger" else @@ -64,22 +64,17 @@ on CreateTabs(pIds) end CreateTabs on lldbRun(binary, pIds) - set debugger to "lldb" - + set debuggerCommandFile to myDir() & "/resume.lldb" + set debugger to "lldb -s " & debuggerCommandFile + set tabList to CreateTabs(pIds) tell application "Terminal" -- run commands in tabs repeat with i from 1 to count pIds set pId to item i of pIds set curTab to item i of tabList - set cmd to debugger & " -f " & binary & " -p " & pId + set cmd to debugger & " -p " & pId & " " & binary set newTab to do script cmd in curTab - delay 1 - do script ("frame select -r 3") in curTab - do script ("expr amWaiting = 0") in curTab - do script ("breakpoint set -F hemelb::debug::ActiveDebugger::BreakHere()") in curTab - do script ("breakpoint command add -o 'finish' 1") in curTab - do script ("continue") in curTab end repeat tell application "System Events" to tell process "Terminal" to keystroke "}" using command down diff --git a/Code/debug/OSX/resume.lldb b/Code/debug/OSX/resume.lldb new file mode 100644 index 000000000..0bd336b81 --- /dev/null +++ b/Code/debug/OSX/resume.lldb @@ -0,0 +1,11 @@ +# Get out of sleep and disable the waiting flag +frame select -r 3 +expr amWaiting = 0 + +# Set up a breakpoint for our break function and a command to finish +# it automatically +breakpoint set -F hemelb::debug::ActiveDebugger::BreakHere() +breakpoint command add -o 'finish' 1 + +# Carry on +continue From 030a951654d9d3f40afe58fd29d354781db5d1c2 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 3 Dec 2015 11:01:32 +0000 Subject: [PATCH 24/99] Make the collective actions test rather than wait --- Code/net/CollectiveAction.cc | 74 +++++++++++++++++++++++------------- Code/net/CollectiveAction.h | 15 ++++++-- 2 files changed, 59 insertions(+), 30 deletions(-) diff --git a/Code/net/CollectiveAction.cc b/Code/net/CollectiveAction.cc index 1f609f16f..3ae06bf48 100644 --- a/Code/net/CollectiveAction.cc +++ b/Code/net/CollectiveAction.cc @@ -15,47 +15,67 @@ namespace hemelb namespace net { CollectiveAction::CollectiveAction(const MpiCommunicator& comm, reporting::Timer& wTimer) : - collectiveComm(comm.Duplicate()), waitTimer(wTimer), collectiveReq() + isCollectiveRunning(false), mustWait(false), collectiveComm(comm.Duplicate()), waitTimer(wTimer), collectiveReq() { } + bool CollectiveAction::CallAction(int action) { - switch (static_cast(action)) + if (isCollectiveRunning) + { + switch (static_cast(action)) + { + case phased::steps::Wait: + Wait(); + return true; + default: + return false; + } + } + else { - case phased::steps::BeginPhase: - RequestComms(); - return true; - case phased::steps::PreSend: - PreSend(); - return true; - case phased::steps::Send: - Send(); - return true; - case phased::steps::PreWait: - PreReceive(); - return true; - case phased::steps::Wait: - Wait(); - return true; - case phased::steps::EndPhase: - PostReceive(); - return true; - case phased::steps::EndAll: - EndIteration(); - return true; - default: - return false; + switch (static_cast(action)) + { + case phased::steps::BeginPhase: + RequestComms(); + return true; + case phased::steps::PreSend: + PreSend(); + return true; + case phased::steps::Send: + Send(); + isCollectiveRunning = true; + return true; + case phased::steps::EndPhase: + PostReceive(); + return true; + case phased::steps::EndAll: + EndIteration(); + return true; + default: + return false; + } } } + /** * Wait on the collectives to finish. */ void CollectiveAction::Wait(void) { waitTimer.Start(); - collectiveReq.Wait(); + if (mustWait) { + collectiveReq.Wait(); + mustWait = false; + isCollectiveRunning = false; + } + else + { + bool done = collectiveReq.Test(); + if (done) + isCollectiveRunning = false; + } waitTimer.Stop(); } - } } diff --git a/Code/net/CollectiveAction.h b/Code/net/CollectiveAction.h index 8b5d72529..53ef84fae 100644 --- a/Code/net/CollectiveAction.h +++ b/Code/net/CollectiveAction.h @@ -23,18 +23,26 @@ namespace hemelb public: bool CallAction(int action); + inline void MustFinishThisTimeStep() + { + + mustWait = true; + } + + protected: + CollectiveAction(const MpiCommunicator& comm, reporting::Timer& waitTimer); /** * Initiate the collective. */ virtual void Send(void) = 0; /** - * Wait on the collectives to finish. + * Progress the communication */ virtual void Wait(void); - protected: - CollectiveAction(const MpiCommunicator& comm, reporting::Timer& waitTimer); + bool isCollectiveRunning; + bool mustWait; /** * Private communicator for non-blocking collectives. @@ -49,6 +57,7 @@ namespace hemelb */ MpiRequest collectiveReq; }; + } } From 544cd0f8ed453a96d660cba443987fceddb4ef73 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 3 Dec 2015 15:52:58 +0000 Subject: [PATCH 25/99] First pass at removing visualisation - unit tests pass --- Code/CMakeLists.txt | 2 - Code/SimulationMaster.cc | 199 +---- Code/SimulationMaster.h | 10 - Code/configuration/CommandLine.cc | 7 +- Code/configuration/CommandLine.h | 10 +- Code/configuration/SimConfig.cc | 20 - Code/configuration/SimConfig.h | 36 - .../BlockTraverserWithVisitedBlockTracker.cc | 1 - Code/lb/SimulationState.cc | 12 +- Code/lb/SimulationState.h | 3 - Code/lb/lb.h | 33 +- Code/lb/lb.hpp | 52 +- Code/lb/streamers/BaseStreamer.h | 2 +- Code/lb/streamers/JunkYangFactory.h | 2 + Code/reporting/Reporter.cc | 9 +- Code/reporting/Reporter.h | 2 - Code/reporting/Timers.h | 3 +- Code/steering/CMakeLists.txt | 5 - Code/steering/ImageSendComponent.h | 69 -- Code/steering/SteeringComponent.h | 12 +- Code/steering/basic/ImageSendComponent.cc | 121 --- Code/steering/basic/SimulationParameters.cc | 5 - Code/steering/basic/SimulationParameters.h | 4 +- Code/steering/basic/SteeringComponentB.cc | 15 +- Code/steering/common/Singleton.h | 50 -- Code/steering/common/Steerable.h | 77 -- Code/steering/common/Steerer.cc | 27 - Code/steering/common/Steerer.h | 93 --- Code/steering/common/SteeringComponentC.cc | 124 --- Code/steering/common/Tag.h | 128 --- Code/steering/common/TagBase.h | 28 - Code/steering/common/Tags.cc | 46 -- Code/steering/common/Tags.h | 50 -- Code/steering/none/ImageSendComponent.cc | 39 - Code/steering/none/SteeringComponentN.cc | 5 +- Code/unittests/SimulationMasterTests.h | 10 +- .../configuration/CommandLineTests.h | 10 +- Code/unittests/io/PathManagerTests.h | 12 +- Code/unittests/main.cc | 1 - .../multiscale/MockIntercommunicatorTests.h | 12 +- Code/unittests/reporting/ReporterTests.h | 4 - .../resources/config-velocity-iolet.xml | 12 - Code/unittests/resources/config.xml | 12 - Code/unittests/resources/config0_2_0.xml | 12 - .../unittests/resources/config_file_inlet.xml | 12 - .../resources/config_file_velocity_inlet.xml | 12 - .../resources/config_new_velocity_inlets.xml | 12 - Code/unittests/resources/four_cube.xml | 12 - .../resources/four_cube_multiscale.xml | 12 - .../vistests/HslToRgbConvertorTests.h | 63 -- Code/unittests/vistests/vistests.h | 15 - Code/vis/BasicPixel.cc | 84 -- Code/vis/BasicPixel.h | 59 -- Code/vis/CMakeLists.txt | 7 - Code/vis/Control.cc | 637 --------------- Code/vis/Control.h | 154 ---- Code/vis/DomainStats.h | 35 - Code/vis/GlyphDrawer.cc | 217 ----- Code/vis/GlyphDrawer.h | 88 -- Code/vis/PixelSet.h | 150 ---- Code/vis/PixelSetStore.h | 74 -- Code/vis/Rendering.cc | 140 ---- Code/vis/Rendering.h | 64 -- Code/vis/ResultPixel.cc | 237 ------ Code/vis/ResultPixel.h | 62 -- Code/vis/Screen.cc | 106 --- Code/vis/Screen.h | 81 -- Code/vis/Viewpoint.cc | 158 ---- Code/vis/Viewpoint.h | 112 --- Code/vis/VisSettings.h | 51 -- Code/vis/XYCoordinates.h | 140 ---- Code/vis/rayTracer/Cluster.h | 171 ---- Code/vis/rayTracer/ClusterBuilder.h | 343 -------- Code/vis/rayTracer/ClusterNormal.cc | 47 -- Code/vis/rayTracer/ClusterNormal.h | 42 - Code/vis/rayTracer/ClusterRayTracer.h | 764 ------------------ Code/vis/rayTracer/ClusterTraverser.h | 58 -- Code/vis/rayTracer/ClusterWithWallNormals.cc | 67 -- Code/vis/rayTracer/ClusterWithWallNormals.h | 47 -- Code/vis/rayTracer/HSLToRGBConverter.cc | 130 --- Code/vis/rayTracer/HSLToRGBConverter.h | 37 - Code/vis/rayTracer/Ray.h | 168 ---- Code/vis/rayTracer/RayData.h | 251 ------ Code/vis/rayTracer/RayDataEnhanced.cc | 62 -- Code/vis/rayTracer/RayDataEnhanced.h | 297 ------- Code/vis/rayTracer/RayDataNormal.cc | 165 ---- Code/vis/rayTracer/RayDataNormal.h | 86 -- Code/vis/rayTracer/RayTracer.h | 103 --- Code/vis/rayTracer/SiteData.h | 31 - .../streaklineDrawer/NeighbouringProcessor.cc | 170 ---- .../streaklineDrawer/NeighbouringProcessor.h | 80 -- Code/vis/streaklineDrawer/Particle.cc | 50 -- Code/vis/streaklineDrawer/Particle.h | 43 - Code/vis/streaklineDrawer/ParticleManager.cc | 124 --- Code/vis/streaklineDrawer/ParticleManager.h | 63 -- Code/vis/streaklineDrawer/StreakPixel.cc | 24 - Code/vis/streaklineDrawer/StreakPixel.h | 104 --- Code/vis/streaklineDrawer/StreaklineDrawer.cc | 354 -------- Code/vis/streaklineDrawer/StreaklineDrawer.h | 93 --- Code/vis/streaklineDrawer/VelocityField.cc | 350 -------- Code/vis/streaklineDrawer/VelocityField.h | 84 -- Code/vis/streaklineDrawer/VelocitySiteData.h | 56 -- 102 files changed, 45 insertions(+), 8599 deletions(-) delete mode 100644 Code/steering/ImageSendComponent.h delete mode 100644 Code/steering/basic/ImageSendComponent.cc delete mode 100644 Code/steering/common/Singleton.h delete mode 100644 Code/steering/common/Steerable.h delete mode 100644 Code/steering/common/Steerer.cc delete mode 100644 Code/steering/common/Steerer.h delete mode 100644 Code/steering/common/Tag.h delete mode 100644 Code/steering/common/TagBase.h delete mode 100644 Code/steering/common/Tags.cc delete mode 100644 Code/steering/common/Tags.h delete mode 100644 Code/steering/none/ImageSendComponent.cc delete mode 100644 Code/unittests/vistests/HslToRgbConvertorTests.h delete mode 100644 Code/unittests/vistests/vistests.h delete mode 100644 Code/vis/BasicPixel.cc delete mode 100644 Code/vis/BasicPixel.h delete mode 100644 Code/vis/CMakeLists.txt delete mode 100644 Code/vis/Control.cc delete mode 100644 Code/vis/Control.h delete mode 100644 Code/vis/DomainStats.h delete mode 100644 Code/vis/GlyphDrawer.cc delete mode 100644 Code/vis/GlyphDrawer.h delete mode 100644 Code/vis/PixelSet.h delete mode 100644 Code/vis/PixelSetStore.h delete mode 100644 Code/vis/Rendering.cc delete mode 100644 Code/vis/Rendering.h delete mode 100644 Code/vis/ResultPixel.cc delete mode 100644 Code/vis/ResultPixel.h delete mode 100644 Code/vis/Screen.cc delete mode 100644 Code/vis/Screen.h delete mode 100644 Code/vis/Viewpoint.cc delete mode 100644 Code/vis/Viewpoint.h delete mode 100644 Code/vis/VisSettings.h delete mode 100644 Code/vis/XYCoordinates.h delete mode 100644 Code/vis/rayTracer/Cluster.h delete mode 100644 Code/vis/rayTracer/ClusterBuilder.h delete mode 100644 Code/vis/rayTracer/ClusterNormal.cc delete mode 100644 Code/vis/rayTracer/ClusterNormal.h delete mode 100644 Code/vis/rayTracer/ClusterRayTracer.h delete mode 100644 Code/vis/rayTracer/ClusterTraverser.h delete mode 100644 Code/vis/rayTracer/ClusterWithWallNormals.cc delete mode 100644 Code/vis/rayTracer/ClusterWithWallNormals.h delete mode 100644 Code/vis/rayTracer/HSLToRGBConverter.cc delete mode 100644 Code/vis/rayTracer/HSLToRGBConverter.h delete mode 100644 Code/vis/rayTracer/Ray.h delete mode 100644 Code/vis/rayTracer/RayData.h delete mode 100644 Code/vis/rayTracer/RayDataEnhanced.cc delete mode 100644 Code/vis/rayTracer/RayDataEnhanced.h delete mode 100644 Code/vis/rayTracer/RayDataNormal.cc delete mode 100644 Code/vis/rayTracer/RayDataNormal.h delete mode 100644 Code/vis/rayTracer/RayTracer.h delete mode 100644 Code/vis/rayTracer/SiteData.h delete mode 100644 Code/vis/streaklineDrawer/NeighbouringProcessor.cc delete mode 100644 Code/vis/streaklineDrawer/NeighbouringProcessor.h delete mode 100644 Code/vis/streaklineDrawer/Particle.cc delete mode 100644 Code/vis/streaklineDrawer/Particle.h delete mode 100644 Code/vis/streaklineDrawer/ParticleManager.cc delete mode 100644 Code/vis/streaklineDrawer/ParticleManager.h delete mode 100644 Code/vis/streaklineDrawer/StreakPixel.cc delete mode 100644 Code/vis/streaklineDrawer/StreakPixel.h delete mode 100644 Code/vis/streaklineDrawer/StreaklineDrawer.cc delete mode 100644 Code/vis/streaklineDrawer/StreaklineDrawer.h delete mode 100644 Code/vis/streaklineDrawer/VelocityField.cc delete mode 100644 Code/vis/streaklineDrawer/VelocityField.h delete mode 100644 Code/vis/streaklineDrawer/VelocitySiteData.h diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 64fe3017a..32ab6b98b 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -184,7 +184,6 @@ set(package_subdirs extraction reporting steering - vis lb geometry net @@ -231,7 +230,6 @@ if (HEMELB_BUILD_MULTISCALE) extraction reporting steering - vis lb geometry net diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index 3468f1b95..195ca4355 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -46,13 +46,11 @@ SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, latticeBoltzmannModel = NULL; steeringCpt = NULL; propertyDataSource = NULL; - visualisationControl = NULL; propertyExtractor = NULL; simulationState = NULL; stepManager = NULL; netConcern = NULL; neighbouringDataManager = NULL; - imagesPerSimulation = options.NumberOfImages(); steeringSessionId = options.GetSteeringSessionId(); fileManager = new hemelb::io::PathManager(options, IsCurrentProcTheIOProc(), GetProcessorCount()); @@ -85,10 +83,6 @@ SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, SimulationMaster::~SimulationMaster() { - if (ioComms.OnIORank()) - { - delete imageSendCpt; - } delete latticeData; delete colloidController; delete latticeBoltzmannModel; @@ -96,7 +90,6 @@ SimulationMaster::~SimulationMaster() delete outletValues; delete network; delete steeringCpt; - delete visualisationControl; delete propertyExtractor; delete propertyDataSource; delete stabilityTester; @@ -134,7 +127,7 @@ int SimulationMaster::GetProcessorCount() /** * Initialises various elements of the simulation if necessary - steering, - * domain decomposition, LBM and visualisation. + * domain decomposition, and LBM. */ void SimulationMaster::Initialise() { @@ -233,29 +226,6 @@ void SimulationMaster::Initialise() incompressibilityChecker = NULL; } - hemelb::log::Logger::Log("Initialising visualisation controller."); - visualisationControl = - new hemelb::vis::Control(latticeBoltzmannModel->GetLbmParams()->StressType, - &communicationNet, - simulationState, - latticeBoltzmannModel->GetPropertyCache(), - latticeData, - timings[hemelb::reporting::Timers::visualisation]); - - if (ioComms.OnIORank()) - { - imageSendCpt = new hemelb::steering::ImageSendComponent(simulationState, - visualisationControl, - latticeBoltzmannModel->GetLbmParams(), - network, - latticeBoltzmannModel->InletCount()); - - } - else - { - imageSendCpt = NULL; - } - inletValues = new hemelb::lb::iolets::BoundaryValues(hemelb::geometry::INLET_TYPE, latticeData, simConfig->GetInlets(), @@ -270,22 +240,17 @@ void SimulationMaster::Initialise() ioComms, *unitConverter); - latticeBoltzmannModel->Initialise(visualisationControl, inletValues, outletValues, unitConverter); + latticeBoltzmannModel->Initialise(inletValues, outletValues, unitConverter); neighbouringDataManager->ShareNeeds(); neighbouringDataManager->TransferNonFieldDependentInformation(); steeringCpt = new hemelb::steering::SteeringComponent(network, - visualisationControl, - imageSendCpt, &communicationNet, simulationState, simConfig, unitConverter, timings); - // Read in the visualisation parameters. - latticeBoltzmannModel->ReadVisParameters(); - propertyDataSource = new hemelb::extraction::LbDataSourceIterator(latticeBoltzmannModel->GetPropertyCache(), *latticeData, @@ -307,8 +272,6 @@ void SimulationMaster::Initialise() timings, ioComms); } - imagesPeriod = OutputPeriod(imagesPerSimulation); - stepManager = new hemelb::net::phased::StepManager(2, &timings, hemelb::net::separate_communications); @@ -335,7 +298,7 @@ void SimulationMaster::Initialise() stepManager->RegisterIteratedActorSteps(*incompressibilityChecker, 1); stepManager->RegisterCommsSteps(*incompressibilityChecker, 1); } - stepManager->RegisterIteratedActorSteps(*visualisationControl, 1); + if (propertyExtractor != NULL) { stepManager->RegisterIteratedActorSteps(*propertyExtractor, 1); @@ -372,77 +335,6 @@ void SimulationMaster::OnUnstableSimulation() Abort(); } -void SimulationMaster::WriteLocalImages() -{ - /** - * this map iteration iterates over all those image generation requests completing this step. - * The map key (it->first) is the completion time step number. - * The map value (it->second) is the initiation time step number. - */ - for (MapType::const_iterator it = writtenImagesCompleted.find(simulationState->GetTimeStep()); - it != writtenImagesCompleted.end() && it->first == simulationState->GetTimeStep(); ++it) - { - - if (ioComms.OnIORank()) - { - reporter->Image(); - hemelb::io::writers::Writer * writer = fileManager->XdrImageWriter(1 - + ( (it->second - 1) % simulationState->GetTimeStep())); - - const hemelb::vis::PixelSet* result = - visualisationControl->GetResult(it->second); - - visualisationControl->WriteImage(writer, - *result, - visualisationControl->domainStats, - visualisationControl->visSettings); - - delete writer; - } - } - - writtenImagesCompleted.erase(simulationState->GetTimeStep()); -} - -void SimulationMaster::GenerateNetworkImages() -{ - for (std::multimap::const_iterator it = - networkImagesCompleted.find(simulationState->GetTimeStep()); - it != networkImagesCompleted.end() && it->first == simulationState->GetTimeStep(); ++it) - { - if (ioComms.OnIORank()) - { - - const hemelb::vis::PixelSet* result = - visualisationControl->GetResult(it->second); - - if (steeringCpt->updatedMouseCoords) - { - float density, stress; - - if (visualisationControl->MouseIsOverPixel(result, &density, &stress)) - { - double mousePressure = 0.0, mouseStress = 0.0; - latticeBoltzmannModel->CalculateMouseFlowField(density, - stress, - mousePressure, - mouseStress, - visualisationControl->domainStats.density_threshold_min, - visualisationControl->domainStats.density_threshold_minmax_inv, - visualisationControl->domainStats.stress_threshold_max_inv); - - visualisationControl->SetMouseParams(mousePressure, mouseStress); - } - steeringCpt->updatedMouseCoords = false; - } - - imageSendCpt->DoWork(result); - - } - } - - networkImagesCompleted.erase(simulationState->GetTimeStep()); -} /** * Begin the simulation. @@ -484,54 +376,10 @@ void SimulationMaster::Finalise() void SimulationMaster::DoTimeStep() { - bool writeImage = ( (simulationState->GetTimeStep() % imagesPeriod) == 0) ? - true : - false; - - // Make sure we're rendering if we're writing this iteration. - if (writeImage) - { - /*** - * writtenImagesCompleted and networkImagesCompleted are multimaps. - * The keys are the iterations on which production of an image will complete, and should be written or sent over the network. - * The values are the iterations on which the image creation began. - */ - writtenImagesCompleted.insert(std::pair(visualisationControl->Start(), - simulationState->GetTimeStep())); - } - - if (simulationState->IsRendering()) - { - // Here, Start() actually triggers the render. - networkImagesCompleted.insert(std::pair(visualisationControl->Start(), - simulationState->GetTimeStep())); - hemelb::log::Logger::Log("%d images currently being composited for the steering client", - networkImagesCompleted.size()); - simulationState->SetIsRendering(false); - } - - /* In the following two if blocks we do the core magic to ensure we only Render - when (1) we are not sending a frame or (2) we need to output to disk */ - - /* TODO for debugging purposes we want to ensure we capture the variables in a single - instant of time since variables might be altered by the thread half way through? - This is to be done. */ - - bool renderForNetworkStream = false; - if (ioComms.OnIORank() - && !steeringCpt->readyForNextImage) - { - renderForNetworkStream = imageSendCpt->ShouldRenderNewNetworkImage(); - steeringCpt->readyForNextImage = renderForNetworkStream; - } - if (simulationState->GetTimeStep() % 100 == 0) { - hemelb::log::Logger::Log("time step %i render_network_stream %i write_image_to_disk %i rendering %i", - simulationState->GetTimeStep(), - renderForNetworkStream, - writeImage, - simulationState->IsRendering()); + hemelb::log::Logger::Log("time step %i", + simulationState->GetTimeStep()); LogStabilityReport(); } @@ -556,22 +404,6 @@ void SimulationMaster::DoTimeStep() if ( (simulationState->GetTimeStep() % 500 == 0) && colloidController != NULL) colloidController->OutputInformation(simulationState->GetTimeStep()); -#ifndef NO_STREAKLINES - visualisationControl->ProgressStreaklines(simulationState->GetTimeStep(), - simulationState->GetTotalTimeSteps()); -#endif - - if (writtenImagesCompleted.count(simulationState->GetTimeStep()) > 0) - { - WriteLocalImages(); - - } - - if (networkImagesCompleted.count(simulationState->GetTimeStep()) > 0) - { - GenerateNetworkImages(); - } - if (simulationState->GetTimeStep() % FORCE_FLUSH_PERIOD == 0 && IsCurrentProcTheIOProc()) { fflush(NULL); @@ -586,22 +418,6 @@ void SimulationMaster::RecalculatePropertyRequirements() propertyCache.ResetRequirements(); - // Check whether we're rendering images on this iteration. - if (visualisationControl->IsRendering()) - { - propertyCache.densityCache.SetRefreshFlag(); - propertyCache.velocityCache.SetRefreshFlag(); - - if (simConfig->GetStressType() == hemelb::lb::ShearStress) - { - propertyCache.wallShearStressMagnitudeCache.SetRefreshFlag(); - } - else if (simConfig->GetStressType() == hemelb::lb::VonMises) - { - propertyCache.vonMisesStressCache.SetRefreshFlag(); - } - } - if (monitoringConfig->doIncompressibilityCheck) { propertyCache.densityCache.SetRefreshFlag(); @@ -613,11 +429,6 @@ void SimulationMaster::RecalculatePropertyRequirements() { propertyExtractor->SetRequiredProperties(propertyCache); } - - // If using streaklines, the velocity will be needed. -#ifndef NO_STREAKLINES - propertyCache.velocityCache.SetRefreshFlag(); -#endif } /** diff --git a/Code/SimulationMaster.h b/Code/SimulationMaster.h index b8df57df6..4329965cf 100644 --- a/Code/SimulationMaster.h +++ b/Code/SimulationMaster.h @@ -14,7 +14,6 @@ #include "lb/lb.hpp" #include "lb/StabilityTester.h" #include "net/net.h" -#include "steering/ImageSendComponent.h" #include "steering/SteeringComponent.h" #include "lb/EntropyTester.h" #include "lb/iolets/BoundaryValues.h" @@ -67,8 +66,6 @@ class SimulationMaster unsigned int OutputPeriod(unsigned int frequency); void HandleActors(); void OnUnstableSimulation(); - void WriteLocalImages(); - void GenerateNetworkImages(); /** * Updates the property caches record of which properties need to be calculated * and cached on this iteration. @@ -87,11 +84,7 @@ class SimulationMaster hemelb::reporting::BuildInfo build_info; typedef std::multimap MapType; - MapType writtenImagesCompleted; - MapType networkImagesCompleted; - hemelb::steering::Network* network; - hemelb::steering::ImageSendComponent *imageSendCpt; hemelb::steering::SteeringComponent* steeringCpt; hemelb::lb::SimulationState* simulationState; @@ -108,16 +101,13 @@ class SimulationMaster const hemelb::util::UnitConverter* unitConverter; - hemelb::vis::Control* visualisationControl; hemelb::extraction::IterableDataSource* propertyDataSource; hemelb::extraction::PropertyActor* propertyExtractor; hemelb::net::phased::StepManager* stepManager; hemelb::net::phased::NetConcern* netConcern; - unsigned int imagesPerSimulation; int steeringSessionId; - unsigned int imagesPeriod; static const hemelb::LatticeTimeStep FORCE_FLUSH_PERIOD=1000; }; diff --git a/Code/configuration/CommandLine.cc b/Code/configuration/CommandLine.cc index f511edb10..d54c91661 100644 --- a/Code/configuration/CommandLine.cc +++ b/Code/configuration/CommandLine.cc @@ -16,7 +16,7 @@ namespace hemelb { CommandLine::CommandLine(int aargc, const char * const * const aargv) : - inputFile("input.xml"), outputDir(""), images(10), steeringSessionId(1), debugMode(false), argc(aargc), + inputFile("input.xml"), outputDir(""), steeringSessionId(1), debugMode(false), argc(aargc), argv(aargv) { @@ -40,11 +40,6 @@ namespace hemelb { outputDir = std::string(paramValue); } - else if (std::strcmp(paramName, "-i") == 0) - { - char *dummy; - images = (unsigned int) (strtoul(paramValue, &dummy, 10)); - } else if (std::strcmp(paramName, "-ss") == 0) { char *dummy; diff --git a/Code/configuration/CommandLine.h b/Code/configuration/CommandLine.h index a031c67b9..990cd176b 100644 --- a/Code/configuration/CommandLine.h +++ b/Code/configuration/CommandLine.h @@ -24,7 +24,6 @@ namespace hemelb * Arguments should be: * - -in input xml configuration file (default input.xml) * - -out output folder (empty default, but the hemelb::io::PathManager will guess a value from the input file if not given.) - * - -i number of images (default 10) * - -ss steering session i.d. (default 1) */ class CommandLine @@ -41,13 +40,7 @@ namespace hemelb * Report to standard output an error message describing the usage */ static std::string GetUsage(); - /** - * @return Number of images that should be produced - */ - unsigned int NumberOfImages() const - { - return (images); - } + /** * The output directory that should be used for result files. * Empty default, but the hemelb::io::PathManager will guess a value from the input file if not given.) @@ -105,7 +98,6 @@ namespace hemelb private: std::string inputFile; //! local or full path to input file std::string outputDir; //! local or full path to input file - unsigned int images; //! images to produce int steeringSessionId; //! unique identifier for steering session bool debugMode; //! Use debugger int argc; //! count of command line arguments, including program name diff --git a/Code/configuration/SimConfig.cc b/Code/configuration/SimConfig.cc index e53907e81..8619c66bd 100644 --- a/Code/configuration/SimConfig.cc +++ b/Code/configuration/SimConfig.cc @@ -86,8 +86,6 @@ namespace hemelb inlets = DoIOForInOutlets(topNode.GetChildOrThrow("inlets")); outlets = DoIOForInOutlets(topNode.GetChildOrThrow("outlets")); - DoIOForVisualisation(topNode.GetChildOrThrow("visualisation")); - // Optional element io::xml::Element propertiesEl = topNode.GetChildOrNull("properties"); if (propertiesEl != io::xml::Element::Missing()) @@ -291,24 +289,6 @@ namespace hemelb return newIolet; } - void SimConfig::DoIOForVisualisation(const io::xml::Element& visEl) - { - GetDimensionalValue(visEl.GetChildOrThrow("centre"), "m", visualisationCentre); - - io::xml::Element orientationEl = visEl.GetChildOrThrow("orientation"); - GetDimensionalValue(orientationEl.GetChildOrThrow("longitude"), - "deg", - visualisationLongitude); - GetDimensionalValue(orientationEl.GetChildOrThrow("latitude"), "deg", visualisationLatitude); - - io::xml::Element displayEl = visEl.GetChildOrThrow("display"); - displayEl.GetAttributeOrThrow("zoom", visualisationZoom); - displayEl.GetAttributeOrThrow("brightness", visualisationBrightness); - - io::xml::Element rangeEl = visEl.GetChildOrThrow("range"); - GetDimensionalValue(rangeEl.GetChildOrThrow("maxvelocity"), "m/s", maxVelocity); - GetDimensionalValue(rangeEl.GetChildOrThrow("maxstress"), "Pa", maxStress); - } void SimConfig::DoIOForProperties(const io::xml::Element& propertiesEl) { diff --git a/Code/configuration/SimConfig.h b/Code/configuration/SimConfig.h index 51551621f..192ce4b68 100644 --- a/Code/configuration/SimConfig.h +++ b/Code/configuration/SimConfig.h @@ -68,10 +68,6 @@ namespace hemelb void Save(std::string path); // TODO this method should be able to be CONST // but because it uses DoIo, which uses one function signature for both reading and writing, it cannot be. - const util::Vector3D & GetVisualisationCentre() const - { - return visualisationCentre; - } const std::vector & GetInlets() const { return inlets; @@ -84,30 +80,6 @@ namespace hemelb { return stressType; } - float GetVisualisationLongitude() const - { - return visualisationLongitude; - } - float GetVisualisationLatitude() const - { - return visualisationLatitude; - } - float GetVisualisationZoom() const - { - return visualisationZoom; - } - float GetVisualisationBrightness() const - { - return visualisationBrightness; - } - float GetMaximumVelocity() const - { - return maxVelocity; - } - float GetMaximumStress() const - { - return maxStress; - } const std::string & GetDataFilePath() const { return dataFilePath; @@ -247,7 +219,6 @@ namespace hemelb extraction::SurfacePointSelector* DoIOForSurfacePoint(const io::xml::Element&); void DoIOForInitialConditions(io::xml::Element parent); - void DoIOForVisualisation(const io::xml::Element& visEl); /** * Reads monitoring configuration from XML file @@ -274,13 +245,6 @@ namespace hemelb io::xml::Document* rawXmlDoc; std::string dataFilePath; - util::Vector3D visualisationCentre; - float visualisationLongitude; - float visualisationLatitude; - float visualisationZoom; - float visualisationBrightness; - float maxVelocity; - float maxStress; lb::StressTypes stressType; std::vector propertyOutputs; std::string colloidConfigPath; diff --git a/Code/geometry/BlockTraverserWithVisitedBlockTracker.cc b/Code/geometry/BlockTraverserWithVisitedBlockTracker.cc index 2078dee30..51a36eaac 100644 --- a/Code/geometry/BlockTraverserWithVisitedBlockTracker.cc +++ b/Code/geometry/BlockTraverserWithVisitedBlockTracker.cc @@ -10,7 +10,6 @@ #include "geometry/BlockTraverserWithVisitedBlockTracker.h" #include "geometry/LatticeData.h" #include "util/Vector3D.h" -#include "vis/rayTracer/RayTracer.h" namespace hemelb { diff --git a/Code/lb/SimulationState.cc b/Code/lb/SimulationState.cc index df12d0b7e..2eca0ef15 100644 --- a/Code/lb/SimulationState.cc +++ b/Code/lb/SimulationState.cc @@ -18,7 +18,7 @@ namespace hemelb SimulationState::SimulationState(double timeStepLength, unsigned long totalTimeSteps) : timeStepLength(timeStepLength), timeStep(1), totalTimeSteps(totalTimeSteps), - isTerminating(false), isRendering(false), stability(Stable) + isTerminating(false), stability(Stable) { } @@ -36,10 +36,7 @@ namespace hemelb { isTerminating = value; } - void SimulationState::SetIsRendering(bool value) - { - isRendering = value; - } + void SimulationState::SetStability(Stability value) { stability = value; @@ -64,10 +61,7 @@ namespace hemelb { return isTerminating; } - bool SimulationState::IsRendering() const - { - return isRendering; - } + Stability SimulationState::GetStability() const { return stability; diff --git a/Code/lb/SimulationState.h b/Code/lb/SimulationState.h index beb493eee..bcaa49d8e 100644 --- a/Code/lb/SimulationState.h +++ b/Code/lb/SimulationState.h @@ -34,14 +34,12 @@ namespace hemelb void Increment(); void Reset(); void SetIsTerminating(bool value); - void SetIsRendering(bool value); void SetStability(Stability value); LatticeTimeStep GetTimeStep() const; LatticeTimeStep Get0IndexedTimeStep() const; LatticeTimeStep GetTotalTimeSteps() const; bool IsTerminating() const; - bool IsRendering() const; Stability GetStability() const; PhysicalTime GetTime() const {return GetTimeStepLength()*Get0IndexedTimeStep();} @@ -54,7 +52,6 @@ namespace hemelb LatticeTimeStep timeStep; LatticeTimeStep totalTimeSteps; bool isTerminating; - bool isRendering; Stability stability; }; } diff --git a/Code/lb/lb.h b/Code/lb/lb.h index ba7e4900f..fca4de283 100644 --- a/Code/lb/lb.h +++ b/Code/lb/lb.h @@ -87,21 +87,10 @@ namespace hemelb * Second constructor. * */ - void Initialise(vis::Control* iControl, - iolets::BoundaryValues* iInletValues, + void Initialise(iolets::BoundaryValues* iInletValues, iolets::BoundaryValues* iOutletValues, const util::UnitConverter* iUnits); - void ReadVisParameters(); - - void CalculateMouseFlowField(const ScreenDensity densityIn, - const ScreenStress stressIn, - const LatticeDensity density_threshold_min, - const LatticeDensity density_threshold_minmax_inv, - const LatticeStress stress_threshold_max_inv, - PhysicalPressure &mouse_pressure, - PhysicalStress &mouse_stress); - hemelb::lb::LbmParameters *GetLbmParams(); lb::MacroscopicPropertyCache& GetPropertyCache(); @@ -132,27 +121,14 @@ namespace hemelb template void StreamAndCollide(Collision* collision, const site_t iFirstIndex, const site_t iSiteCount) { - if (mVisControl->IsRendering()) - { - collision->template StreamAndCollide (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); - } - else - { - collision->template StreamAndCollide (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); - } + // TODO: factor out template bool "do rendering" + collision->template StreamAndCollide (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); } template void PostStep(Collision* collision, const site_t iFirstIndex, const site_t iSiteCount) { - if (mVisControl->IsRendering()) - { - collision->template DoPostStep (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); - } - else - { - collision->template DoPostStep (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); - } + collision->template DoPostStep (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); } unsigned int inletCount; @@ -165,7 +141,6 @@ namespace hemelb iolets::BoundaryValues *mInletValues, *mOutletValues; LbmParameters mParams; - vis::Control* mVisControl; const util::UnitConverter* mUnits; diff --git a/Code/lb/lb.hpp b/Code/lb/lb.hpp index de6150415..629fa2cba 100644 --- a/Code/lb/lb.hpp +++ b/Code/lb/lb.hpp @@ -44,22 +44,6 @@ namespace hemelb ReadParameters(); } - template - void LBM::CalculateMouseFlowField(const ScreenDensity densityIn, - const ScreenStress stressIn, - const LatticeDensity density_threshold_min, - const LatticeDensity density_threshold_minmax_inv, - const LatticeStress stress_threshold_max_inv, - PhysicalPressure &mouse_pressure, - PhysicalStress &mouse_stress) - { - LatticeDensity density = density_threshold_min + densityIn / density_threshold_minmax_inv; - LatticeStress stress = stressIn / stress_threshold_max_inv; - - mouse_pressure = mUnits->ConvertPressureToPhysicalUnits(density * Cs2); - mouse_stress = mUnits->ConvertStressToPhysicalUnits(stress); - } - template void LBM::InitInitParamsSiteRanges(kernels::InitParams& initParams, unsigned& state) { @@ -129,8 +113,7 @@ namespace hemelb } template - void LBM::Initialise(vis::Control* iControl, - iolets::BoundaryValues* iInletValues, + void LBM::Initialise(iolets::BoundaryValues* iInletValues, iolets::BoundaryValues* iOutletValues, const util::UnitConverter* iUnits) { @@ -141,8 +124,6 @@ namespace hemelb InitCollisions(); SetInitialConditions(); - - mVisControl = iControl; } template @@ -373,37 +354,6 @@ namespace hemelb mParams.StressType = mSimConfig->GetStressType(); } - template - void LBM::ReadVisParameters() - { - distribn_t density_min = std::numeric_limits::max(); - distribn_t density_max = std::numeric_limits::min(); - - distribn_t velocity_max = mUnits->ConvertVelocityToLatticeUnits(mSimConfig->GetMaximumVelocity()); - distribn_t stress_max = mUnits->ConvertStressToLatticeUnits(mSimConfig->GetMaximumStress()); - - for (int i = 0; i < InletCount(); i++) - { - density_min = util::NumericalFunctions::min(density_min, mInletValues->GetDensityMin(i)); - density_max = util::NumericalFunctions::max(density_max, mInletValues->GetDensityMax(i)); - } - for (int i = 0; i < OutletCount(); i++) - { - density_min = util::NumericalFunctions::min(density_min, mOutletValues->GetDensityMin(i)); - density_max = util::NumericalFunctions::max(density_max, mOutletValues->GetDensityMax(i)); - } - - distribn_t lDensity_threshold_min = density_min; - distribn_t lDensity_threshold_minmax_inv = 1.0F / (density_max - density_min); - distribn_t lVelocity_threshold_max_inv = 1.0F / velocity_max; - distribn_t lStress_threshold_max_inv = 1.0F / stress_max; - - mVisControl->SetSomeParams(mSimConfig->GetVisualisationBrightness(), - lDensity_threshold_min, - lDensity_threshold_minmax_inv, - lVelocity_threshold_max_inv, - lStress_threshold_max_inv); - } } } diff --git a/Code/lb/streamers/BaseStreamer.h b/Code/lb/streamers/BaseStreamer.h index 1fc5dca06..1bd40b540 100644 --- a/Code/lb/streamers/BaseStreamer.h +++ b/Code/lb/streamers/BaseStreamer.h @@ -13,7 +13,6 @@ #include #include "geometry/LatticeData.h" -#include "vis/Control.h" #include "lb/LbmParameters.h" #include "lb/kernels/BaseKernel.h" #include "lb/MacroscopicPropertyCache.h" @@ -57,6 +56,7 @@ namespace hemelb class BaseStreamer { public: + // TODO: remove tDoRayTracing template inline void StreamAndCollide(const site_t firstIndex, const site_t siteCount, diff --git a/Code/lb/streamers/JunkYangFactory.h b/Code/lb/streamers/JunkYangFactory.h index 99624405a..774cb4904 100644 --- a/Code/lb/streamers/JunkYangFactory.h +++ b/Code/lb/streamers/JunkYangFactory.h @@ -10,6 +10,8 @@ #ifndef HEMELB_LB_STREAMERS_JUNKYANGFACTORY_H #define HEMELB_LB_STREAMERS_JUNKYANGFACTORY_H +#include + #include "lb/kernels/BaseKernel.h" #include "lb/streamers/BaseStreamer.h" #include diff --git a/Code/reporting/Reporter.cc b/Code/reporting/Reporter.cc index a8f32d8c8..c456b4150 100644 --- a/Code/reporting/Reporter.cc +++ b/Code/reporting/Reporter.cc @@ -14,16 +14,11 @@ namespace hemelb namespace reporting { Reporter::Reporter(const std::string &apath, const std::string &inputFile) : - path(apath), imageCount(0), dictionary("Reporting dictionary") + path(apath), dictionary("Reporting dictionary") { dictionary.SetValue("CONFIG", inputFile); } - void Reporter::Image() - { - imageCount++; - } - void Reporter::AddReportable(Reportable* reportable) { reportableObjects.push_back(reportable); @@ -42,8 +37,6 @@ namespace hemelb void Reporter::FillDictionary() { - dictionary.SetIntValue("IMAGES", imageCount); - for (std::vector::iterator reporters = reportableObjects.begin(); reporters != reportableObjects.end(); reporters++) { diff --git a/Code/reporting/Reporter.h b/Code/reporting/Reporter.h index f1355c8eb..8c109820d 100644 --- a/Code/reporting/Reporter.h +++ b/Code/reporting/Reporter.h @@ -43,7 +43,6 @@ namespace hemelb * @param aState Reference to state of ongoing simulation. */ Reporter(const std::string &path, const std::string &inputFile); - void Image(); //! Inform the reporter that an image has been saved. void AddReportable(Reportable* reportable); @@ -68,7 +67,6 @@ namespace hemelb private: const std::string &path; void Write(const std::string &ctemplate, const std::string &as); //! Write the report to disk, (or wherever the WriterPolicy decides.) - unsigned int imageCount; //! Number of images written. ctemplate::TemplateDictionary dictionary; std::vector reportableObjects; }; diff --git a/Code/reporting/Timers.h b/Code/reporting/Timers.h index 485afed2a..726ecc583 100644 --- a/Code/reporting/Timers.h +++ b/Code/reporting/Timers.h @@ -95,7 +95,6 @@ namespace hemelb latDatInitialise, //!< Time spent initialising the lattice data lb, //!< Time spent doing the core lattice boltzman simulation lb_calc, //!< Time spent doing calculations in the core lattice boltzmann simulation - visualisation, //!< Time spent on visualisation monitoring, //!< Time spent monitoring for stability, compressibility, etc. mpiSend, //!< Time spent sending MPI data mpiWait, //!< Time spent waiting for MPI @@ -218,7 +217,7 @@ namespace hemelb const std::string TimersBase::timerNames[TimersBase::numberOfTimers] = { "Total", "Seed Decomposition", "Domain Decomposition", "File Read", "Re Read", "Unzip", "Moves", "Parmetis", - "Lattice Data initialisation", "Lattice Boltzmann", "LB calc only", "Visualisation", "Monitoring", "MPI Send", + "Lattice Data initialisation", "Lattice Boltzmann", "LB calc only", "Monitoring", "MPI Send", "MPI Wait", "Simulation total", "Reading communications", "Parsing", "Read IO", "Read Blocks prelim", "Read blocks all", "Steering Client Wait", "Move Forcing Counts", "Move Forcing Data", "Block Requirements", "Move Counts Sending", "Move Data Sending", "Populating moves list for decomposition optimisation", diff --git a/Code/steering/CMakeLists.txt b/Code/steering/CMakeLists.txt index 8f2c3003e..98898d4e9 100644 --- a/Code/steering/CMakeLists.txt +++ b/Code/steering/CMakeLists.txt @@ -8,7 +8,6 @@ if(HEMELB_STEERING_LIB MATCHES [Bb]asic) set(steerers basic/ClientConnection.cc basic/HttpPost.cc - basic/ImageSendComponent.cc basic/Network.cc basic/SimulationParameters.cc basic/SteeringComponentB.cc @@ -17,7 +16,6 @@ if(HEMELB_STEERING_LIB MATCHES [Bb]asic) elseif(HEMELB_STEERING_LIB MATCHES [Nn]one) set(steerers none/ClientConnection.cc - none/ImageSendComponent.cc none/Network.cc none/SteeringComponentN.cc ) @@ -27,7 +25,6 @@ else() set(steerers basic/ClientConnection.cc basic/HttpPost.cc - basic/ImageSendComponent.cc basic/Network.cc basic/SimulationParameters.cc basic/SteeringComponentB.cc @@ -36,9 +33,7 @@ else() endif() add_library(hemelb_steering - #common/Steerer.cc # Not used in old nrmake build either -- TODO find out why common/SteeringComponentC.cc - #common/Tags.cc ${steerers} ) diff --git a/Code/steering/ImageSendComponent.h b/Code/steering/ImageSendComponent.h deleted file mode 100644 index a9dc26429..000000000 --- a/Code/steering/ImageSendComponent.h +++ /dev/null @@ -1,69 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_STEERING_IMAGESENDCOMPONENT_H -#define HEMELB_STEERING_IMAGESENDCOMPONENT_H - -#include "constants.h" -#include "lb/SimulationState.h" -#include "lb/LbmParameters.h" -#include "steering/Network.h" -#include "steering/basic/SimulationParameters.h" -#include "vis/Control.h" - -namespace hemelb -{ - namespace steering - { - class ImageSendComponent - { - public: - ImageSendComponent(lb::SimulationState* iSimState, - vis::Control* iControl, - const lb::LbmParameters* iLbmParams, - Network* iNetwork, - unsigned inletCount); - ~ImageSendComponent(); - - void DoWork(const vis::PixelSet* pix); - void SetMaxFramerate(float maxFramerate){ - MaxFramerate=maxFramerate; - } - bool ShouldRenderNewNetworkImage(); - - bool isConnected; - int send_array_length; - - private: - Network* mNetwork; - lb::SimulationState* mSimState; - vis::Control* mVisControl; - const unsigned inletCount; - float MaxFramerate; - char* xdrSendBuffer; - double lastRender; - - // data per pixel is - // 1 * int (pixel index) - // 3 * int (pixel RGB) - static const int bytes_per_pixel_data = 4 * 4; - - // Sent data: - // 2 * int (pixelsX, pixelsY) - // 1 * int (bytes of pixel data) - // pixel data (variable, up to COLOURED_PIXELS_MAX * bytes_per_pixel_data) - // SimulationParameters::paramsSizeB (metadata - mouse pressure and stress etc) - static const unsigned int XdrIntLength = 4; - static const unsigned int maxSendSize = 2 * XdrIntLength + 1 * XdrIntLength - + vis::Screen::COLOURED_PIXELS_MAX * bytes_per_pixel_data + SimulationParameters::paramsSizeB; - }; - } -} - -#endif /* HEMELB_STEERING_IMAGESENDCOMPONENT_H */ diff --git a/Code/steering/SteeringComponent.h b/Code/steering/SteeringComponent.h index 2a5b6e6c0..2d6e49d29 100644 --- a/Code/steering/SteeringComponent.h +++ b/Code/steering/SteeringComponent.h @@ -10,18 +10,17 @@ #ifndef HEMELB_STEERING_STEERINGCOMPONENT_H #define HEMELB_STEERING_STEERINGCOMPONENT_H +#include "net/Net.h" #include "net/CollectiveAction.h" #include "lb/SimulationState.h" #include "configuration/SimConfig.h" #include "steering/Network.h" -#include "vis/DomainStats.h" -#include "vis/Control.h" -#include "steering/ImageSendComponent.h" namespace hemelb { namespace steering { + // TODO: remove most of these enum parameter { SceneCentreX = 0, @@ -59,8 +58,6 @@ namespace hemelb { public: SteeringComponent(Network* iNetwork, - vis::Control* iVisControl, - steering::ImageSendComponent* imageSendComponent, net::Net * iNet, lb::SimulationState * iSimState, configuration::SimConfig* iSimConfig, @@ -76,9 +73,6 @@ namespace hemelb */ void ClearValues(); - bool readyForNextImage; - bool updatedMouseCoords; - void PreSend(); void Send(); void PostReceive(); @@ -93,8 +87,6 @@ namespace hemelb Network* mNetwork; lb::SimulationState* mSimState; - vis::Control* mVisControl; - steering::ImageSendComponent* imageSendComponent; std::vector privateSteeringParams; const util::UnitConverter* mUnits; configuration::SimConfig* simConfig; diff --git a/Code/steering/basic/ImageSendComponent.cc b/Code/steering/basic/ImageSendComponent.cc deleted file mode 100644 index e618b82a4..000000000 --- a/Code/steering/basic/ImageSendComponent.cc +++ /dev/null @@ -1,121 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include -#include - -#include "log/Logger.h" -#include "steering/ImageSendComponent.h" -#include "steering/Network.h" -#include "io/writers/xdr/XdrMemWriter.h" -#include "util/utilityFunctions.h" - -namespace hemelb -{ - namespace steering - { - // Use initialisation list to do the work. - ImageSendComponent::ImageSendComponent(lb::SimulationState* iSimState, - vis::Control* iControl, - const lb::LbmParameters* iLbmParams, - Network* iNetwork, - unsigned inletCountIn) : - mNetwork(iNetwork), mSimState(iSimState), mVisControl(iControl), inletCount(inletCountIn), MaxFramerate(25.0) - { - xdrSendBuffer = new char[maxSendSize]; - - // Suppress signals from a broken pipe. - signal(SIGPIPE, SIG_IGN); - - isConnected = false; - lastRender = 0.0; - } - - ImageSendComponent::~ImageSendComponent() - { - delete[] xdrSendBuffer; - } - - // This is original code with minimal tweaks to make it work with - // the new (Feb 2011) structure. - void ImageSendComponent::DoWork(const vis::PixelSet* pix) - { - isConnected = mNetwork->IsConnected(); - - if (!isConnected) - { - return; - } - - io::writers::xdr::XdrMemWriter imageWriter = io::writers::xdr::XdrMemWriter(xdrSendBuffer, maxSendSize); - - unsigned int initialPosition = imageWriter.getCurrentStreamPosition(); - - // Write the dimensions of the image, in terms of pixel count. - imageWriter << mVisControl->GetPixelsX() << mVisControl->GetPixelsY(); - - // Write the length of the pixel data - imageWriter << (int) (pix->GetPixelCount() * bytes_per_pixel_data); - - // Write the pixels themselves - mVisControl->WritePixels(&imageWriter, *pix, mVisControl->domainStats, mVisControl->visSettings); - - // Write the numerical data from the simulation, wanted by the client. - { - SimulationParameters sim; - - sim.timeStep = (int) mSimState->GetTimeStep(); - sim.time = mSimState->GetTime(); - sim.nInlets = inletCount; - - sim.mousePressure = mVisControl->visSettings.mouse_pressure; - sim.mouseStress = mVisControl->visSettings.mouse_stress; - - mVisControl->visSettings.mouse_pressure = -1.0; - mVisControl->visSettings.mouse_stress = -1.0; - - sim.pack(&imageWriter); - } - - // Send to the client. - log::Logger::Log("Sending network image at timestep %d",mSimState->GetTimeStep()); - mNetwork->send_all(xdrSendBuffer, imageWriter.getCurrentStreamPosition() - initialPosition); - } - - bool ImageSendComponent::ShouldRenderNewNetworkImage() - { - isConnected = mNetwork->IsConnected(); - - if (!isConnected) - { - return false; - } - - // If we're going to exceed 25Hz by rendering now, wait until next iteration. - { - double frameTimeStart = util::myClock(); - - double deltaTime = frameTimeStart - lastRender; - - if ( (1.0 / MaxFramerate) > deltaTime) - { - return false; - } - else - { - log::Logger::Log("Image-send component requesting new render, %f seconds since last one at step %d max rate is %f.", - deltaTime, mSimState->GetTimeStep(), MaxFramerate); - lastRender = frameTimeStart; - return true; - } - } - } - - } -} diff --git a/Code/steering/basic/SimulationParameters.cc b/Code/steering/basic/SimulationParameters.cc index a0b05d8c0..695a79a40 100644 --- a/Code/steering/basic/SimulationParameters.cc +++ b/Code/steering/basic/SimulationParameters.cc @@ -9,7 +9,6 @@ #include "io/writers/xdr/XdrMemWriter.h" -#include "vis/Control.h" #include "steering/basic/SimulationParameters.h" namespace hemelb @@ -24,8 +23,6 @@ namespace hemelb timeStep = 0; time = 0.0; nInlets = 3; - mousePressure = -1.0; - mouseStress = -1.0; } SimulationParameters::~SimulationParameters() @@ -41,8 +38,6 @@ namespace hemelb writer->operator <<(0); // Cycle is always zero, leave this in to stop steering clients breaking. writer->operator <<(nInlets); - writer->operator <<(mousePressure); - writer->operator <<(mouseStress); } } diff --git a/Code/steering/basic/SimulationParameters.h b/Code/steering/basic/SimulationParameters.h index 9fa1426f4..2e371b7f4 100644 --- a/Code/steering/basic/SimulationParameters.h +++ b/Code/steering/basic/SimulationParameters.h @@ -26,10 +26,8 @@ namespace hemelb int timeStep; double time; int nInlets; - double mousePressure; - double mouseStress; - static const u_int paramsSizeB = 3 * sizeof(int) + 3 * sizeof(double); + static const u_int paramsSizeB = 3 * sizeof(int) + 1 * sizeof(double); SimulationParameters(); ~SimulationParameters(); diff --git a/Code/steering/basic/SteeringComponentB.cc b/Code/steering/basic/SteeringComponentB.cc index a44d2bba9..04982750d 100644 --- a/Code/steering/basic/SteeringComponentB.cc +++ b/Code/steering/basic/SteeringComponentB.cc @@ -18,15 +18,14 @@ namespace hemelb { namespace steering { - SteeringComponent::SteeringComponent(Network* iNetwork, vis::Control* iVisControl, - steering::ImageSendComponent* imageSendComponent, + SteeringComponent::SteeringComponent(Network* iNetwork, net::Net * iNet, lb::SimulationState * iSimState, configuration::SimConfig* iSimConfig, const util::UnitConverter* iUnits, reporting::Timers& timings) : net::CollectiveAction(iNet->GetCommunicator(), timings[reporting::Timers::steeringWait]), - mNetwork(iNetwork), mSimState(iSimState), mVisControl(iVisControl), - imageSendComponent(imageSendComponent), privateSteeringParams(STEERABLE_PARAMETERS + 1), + mNetwork(iNetwork), mSimState(iSimState), + privateSteeringParams(STEERABLE_PARAMETERS + 1), mUnits(iUnits), simConfig(iSimConfig) { ClearValues(); @@ -43,14 +42,6 @@ namespace hemelb if (collectiveComm.Rank() != RootRank) return; - /* - * The final steering parameter is DoRendering, which is true if we're connected and ready - * for the next frame. - */ - { - privateSteeringParams[STEERABLE_PARAMETERS] = (float) (isConnected && readyForNextImage); - } - // Create a buffer for the data received. const int num_chars = STEERABLE_PARAMETERS * sizeof(float) / sizeof(char); const int bytes = sizeof(char) * num_chars; diff --git a/Code/steering/common/Singleton.h b/Code/steering/common/Singleton.h deleted file mode 100644 index 05a677887..000000000 --- a/Code/steering/common/Singleton.h +++ /dev/null @@ -1,50 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_STEERING_COMMON_SINGLETON_H -#define HEMELB_STEERING_COMMON_SINGLETON_H - -namespace hemelb -{ - namespace steering - { - namespace _private - { - /** - * Base class for singleton behaviour. - * Uses CRTP to do static polymorphism. - */ - template - class Singleton - { - protected: - static Singleton* singleton; - Singleton() - { - } - - public: - static Derived* Init() - { - if (Singleton::singleton == NULL) - { - Singleton::singleton = new Derived(); - } - return static_cast(Singleton::singleton); - } - }; - - template - Singleton* Singleton::singleton = NULL; - - } - } -} - -#endif // HEMELB_STEERING_COMMON_SINGLETON_H diff --git a/Code/steering/common/Steerable.h b/Code/steering/common/Steerable.h deleted file mode 100644 index 14b47367a..000000000 --- a/Code/steering/common/Steerable.h +++ /dev/null @@ -1,77 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_STEERING_COMMON_STEERABLE_H -#define HEMELB_STEERING_COMMON_STEERABLE_H - -namespace hemelb -{ - namespace steering - { - namespace _private - { - // Base class for Steerable member vars - class SteerableBase - { - }; - } - - /** - * Template class for wrappers around member vars that allows them to - * be altered by the Steerer. Template parameter should be a suitable - * subclass of Tag<>; see example in Tags.cc. - */ - - template - class Steerable : public _private::SteerableBase - { - public: - typedef T TagClass; - typedef typename TagClass::WrappedType WrappedType; - - protected: - // The value we need to steer. - WrappedType mValue; - // Constant pointer to a mutable TagClass instance. - static TagClass* const tag; - - public: - Steerable() : - mValue(TagClass::InitialValue) - { - Steerable::tag->AddInstance(this); - } - - ~Steerable() - { - tag->RemoveInstance(this); - } - - WrappedType const& Get(void) - { - return mValue; - } - - protected: - void Set(WrappedType const& value) - { - mValue = value; - } - - // It's not the Tag that needs to be our friend, it's the Tag's - // special base class. Happily this is allowed by the standard but - // friend class TagClass is not. - friend class TagClass::Super; - }; - - template T* const Steerable::tag = T::Init(); - - } -} -#endif // HEMELB_STEERING_COMMON_STEERABLE_H diff --git a/Code/steering/common/Steerer.cc b/Code/steering/common/Steerer.cc deleted file mode 100644 index 5d31deecb..000000000 --- a/Code/steering/common/Steerer.cc +++ /dev/null @@ -1,27 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "steering/common/Steerer.h" - -// void Steerer::RegisterTag(TagBase* tag) -// { -// mTags.insert(std::make_pair(tag.TagName, tag)); -// } -namespace hemelb -{ - namespace steering - { - namespace _private - { - Steerer::Steerer() - { - } - } - } -} diff --git a/Code/steering/common/Steerer.h b/Code/steering/common/Steerer.h deleted file mode 100644 index ed147b5bf..000000000 --- a/Code/steering/common/Steerer.h +++ /dev/null @@ -1,93 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_STEERING_COMMON_STEERER_H -#define HEMELB_STEERING_COMMON_STEERER_H - -#include -#include -#include - -#include "steering/common/Singleton.h" -#include "steering/common/Tag.h" - -/* - * Global object to control steerable parameters. - * - * To work, this must have the following methods: - * - * SteeringType* Init(void); - * - * Returns the steerer object, creating it - * if need be. Currently do this by inheriting from the Singleton - * class template. - * - * template - * void SetTag(typename TagType::WrappedType value) - * - * Get the tag type from the list stored and sets its value to the one - * given. - * - * template - * void RegisterTag(TagType* tag) - * - * Called by the Tag<> instance to register itself. Must be accessible - * from there, i.e. make that a friend. However the standard doesn't - * allow partial specializations to be declared friend so we have to - * let any Tag<> be our friend, with: - * - * template friend class Tag; - * - */ -namespace hemelb -{ - namespace steering - { - namespace _private - { - class TagBase; - - template - class Tag; - } - class Steerer : public _private::Singleton - { - public: - // Singleton exposes a method static Steerer* Init(void); - - typedef std::map ContainerType; - typedef ContainerType::value_type PairType; - - template - void SetTag(typename TagType::WrappedType value) - { - TagType* tag = static_cast(mTags[TagType::TagString]); - tag->SetInstanceValues(value); - } - - protected: - template - void RegisterTag(TagType* tag) - { - mTags.insert(PairType(TagType::TagString, static_cast<_private::TagBase*>(tag))); - } - - ContainerType mTags; - Steerer(); - - // Singleton must be a friend. - friend class _private::Singleton; - // Standard apparently doesn't allow partial specializations to be - // declared friend so we have to let any Tag<> be our friend. - template friend class _private::Tag; - }; - - } -} -#endif // HEMELB_STEERING_COMMON_STEERER_H diff --git a/Code/steering/common/SteeringComponentC.cc b/Code/steering/common/SteeringComponentC.cc index f2874fe09..fe27588bc 100644 --- a/Code/steering/common/SteeringComponentC.cc +++ b/Code/steering/common/SteeringComponentC.cc @@ -15,142 +15,18 @@ namespace hemelb { void SteeringComponent::AssignValues() { - mVisControl->visSettings.ctr_x += privateSteeringParams[SceneCentreX]; - mVisControl->visSettings.ctr_y += privateSteeringParams[SceneCentreY]; - mVisControl->visSettings.ctr_z += privateSteeringParams[SceneCentreZ]; - - float longitude = privateSteeringParams[Longitude]; - float latitude = privateSteeringParams[Latitude]; - - float zoom = privateSteeringParams[Zoom]; - - mVisControl->visSettings.brightness = privateSteeringParams[Brightness]; - - // The minimum value here is by default 0.0 all the time - mVisControl->domainStats.physical_velocity_threshold_max = privateSteeringParams[PhysicalVelocityThresholdMax]; - - // The minimum value here is by default 0.0 all the time - mVisControl->domainStats.physical_stress_threshold_max = privateSteeringParams[PhysicalStressThrehsholdMaximum]; - - mVisControl->domainStats.physical_pressure_threshold_min = - privateSteeringParams[PhysicalPressureThresholdMinimum]; - mVisControl->domainStats.physical_pressure_threshold_max = privateSteeringParams[10]; - - mVisControl->visSettings.glyphLength = privateSteeringParams[GlyphLength]; - - float pixels_x = privateSteeringParams[PixelsX]; - float pixels_y = privateSteeringParams[PixelsY]; - - int newMouseX = int(privateSteeringParams[NewMouseX]); - int newMouseY = int(privateSteeringParams[NewMouseY]); - - if (newMouseX != mVisControl->visSettings.mouse_x || newMouseY != mVisControl->visSettings.mouse_y) - { - updatedMouseCoords = true; - mVisControl->visSettings.mouse_x = newMouseX; - mVisControl->visSettings.mouse_y = newMouseY; - } - mSimState->SetIsTerminating(1 == (int) privateSteeringParams[SetIsTerminal]); - - // To swap between glyphs and streak line rendering... - // 0 - Only display the isosurfaces (wall pressure and stress) - // 1 - Isosurface and glyphs - // 2 - Wall pattern streak lines - mVisControl->visSettings.mode = (vis::VisSettings::Mode) (privateSteeringParams[Mode]); - - mVisControl->visSettings.streaklines_per_simulation = privateSteeringParams[StreaklinePerSimulation]; - mVisControl->visSettings.streakline_length = privateSteeringParams[StreaklineLength]; - - mSimState->SetIsRendering(1 == (int) privateSteeringParams[SetDoRendering]); - if (mSimState->IsRendering()) - { - readyForNextImage = false; - } - - mVisControl->UpdateImageSize((int) pixels_x, (int) pixels_y); - - distribn_t lattice_density_min = - mUnits->ConvertPressureToLatticeUnits(mVisControl->domainStats.physical_pressure_threshold_min) / Cs2; - distribn_t lattice_density_max = - mUnits->ConvertPressureToLatticeUnits(mVisControl->domainStats.physical_pressure_threshold_max) / Cs2; - distribn_t lattice_velocity_max = - mUnits->ConvertVelocityToLatticeUnits(mVisControl->domainStats.physical_velocity_threshold_max); - distribn_t lattice_stress_max = - mUnits->ConvertStressToLatticeUnits(mVisControl->domainStats.physical_stress_threshold_max); - - mVisControl->SetProjection((int) pixels_x, - (int) pixels_y, - mVisControl->visSettings.ctr_x, - mVisControl->visSettings.ctr_y, - mVisControl->visSettings.ctr_z, - longitude, - latitude, - zoom); - if (imageSendComponent != NULL) - { - imageSendComponent->SetMaxFramerate(privateSteeringParams[MaxFramerate]); - } - mVisControl->domainStats.density_threshold_min = lattice_density_min; - mVisControl->domainStats.density_threshold_minmax_inv = 1.0F / (lattice_density_max - lattice_density_min); - mVisControl->domainStats.velocity_threshold_max_inv = 1.0F / lattice_velocity_max; - mVisControl->domainStats.stress_threshold_max_inv = 1.0F / lattice_stress_max; } void SteeringComponent::ClearValues() { - readyForNextImage = false; isConnected = false; - updatedMouseCoords = false; - - // scene center (dx,dy,dz) - privateSteeringParams[SceneCentreX] = simConfig->GetVisualisationCentre().x; - privateSteeringParams[SceneCentreY] = simConfig->GetVisualisationCentre().y; - privateSteeringParams[SceneCentreZ] = simConfig->GetVisualisationCentre().z; - - // longitude and latitude - privateSteeringParams[Longitude] = simConfig->GetVisualisationLongitude(); - privateSteeringParams[Latitude] = simConfig->GetVisualisationLatitude(); - - // zoom and brightness - privateSteeringParams[Zoom] = simConfig->GetVisualisationZoom(); - privateSteeringParams[Brightness] = simConfig->GetVisualisationBrightness(); - - // velocity and stress ranges - privateSteeringParams[PhysicalVelocityThresholdMax] = simConfig->GetMaximumVelocity(); - privateSteeringParams[PhysicalStressThrehsholdMaximum] = simConfig->GetMaximumStress(); - - // Minimum pressure and maximum pressure for Colour mapping - privateSteeringParams[PhysicalPressureThresholdMinimum] = 80.0F; - privateSteeringParams[10] = 120.0F; - - // Glyph length - privateSteeringParams[GlyphLength] = 1.0F; - - // Rendered frame size, pixel x and pixel y - privateSteeringParams[PixelsX] = 512.0F; - privateSteeringParams[PixelsY] = 512.0F; - - // x-y position of the mouse of the client - privateSteeringParams[NewMouseX] = -1.0F; - privateSteeringParams[NewMouseY] = -1.0F; // signal useful to terminate the simulation privateSteeringParams[SetIsTerminal] = 0.0F; // Vis_mode privateSteeringParams[Mode] = 0.0F; - - // vis_streaklines_per_pulsatile_period - privateSteeringParams[StreaklinePerSimulation] = 5.0F; - - // vis_streakline_length - privateSteeringParams[StreaklineLength] = 100.0F; - - privateSteeringParams[MaxFramerate] = 25.0F; - - // Value of DoRendering - privateSteeringParams[SetDoRendering] = 0.0F; } } } diff --git a/Code/steering/common/Tag.h b/Code/steering/common/Tag.h deleted file mode 100644 index 0ed9109aa..000000000 --- a/Code/steering/common/Tag.h +++ /dev/null @@ -1,128 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_STEERING_COMMON_TAG_H -#define HEMELB_STEERING_COMMON_TAG_H - -#include -#include - -#include "steering/common/Singleton.h" -#include "steering/common/TagBase.h" - -#include "steering/common/Steerer.h" -#include "steering/common/Steerable.h" - -//class Steerer; -namespace hemelb -{ - namespace steering - { - namespace _private - { - /** - * Family of Tags for a type of variable T. Doesn't specify a tag - * string; that's the job of the the subclass. Hence we're a "static - * abstract base class" and using the Curiously Recurring Template - * Pattern/static polymorphism. Note this is without the common - * specialization when we don't have a subclass. - */ - - template - class Tag : public TagBase, - public Singleton > - { - public: - // Typedefs to avoid lots of parameterized templates - typedef T WrappedType; - typedef Tag TagType; - typedef Steerable SteerableType; - typedef std::set ContainerType; - // String for the tag - static const std::string TagString; - static const WrappedType InitialValue; - - protected: - // Contains of pointers to instances of SteerableType - ContainerType Instances; - // Our global Steerer. - static SteererClass* steerer; - - static DerivedTag* Init() - { - // Wrap the singleton behaviour, registering the instance with the - // steerer. Note have to use static polymorphism pattern here. - DerivedTag* self = static_cast (Singleton >::Init()); - - if (steerer == NULL) - steerer = SteererClass::Init(); - - steerer->SteererClass::RegisterTag(self); - return self; - } - - // Called by constructor of SteerableType - void AddInstance(SteerableType* const inst) - { - Instances.insert(inst); - } - // Called by destructor of SteerableType - void RemoveInstance(SteerableType* const inst) - { - Instances.erase(inst); - } - - public: - // Called by the Steerer to update our instances with the new value - void SetInstanceValues(const T value) - { - for (typename ContainerType::iterator i = Instances.begin(); i - != Instances.end(); ++i) - { - (*i)->Set(value); - } - } - - friend class Singleton > ; - friend class Steerable ; - - // Standard doesn't allow friend class SteererClass. - }; - - // Template static initializers - template - SteererClass* Tag::steerer = - SteererClass::Init(); - - template - const std::string Tag::TagString = ""; - - template - const typename Tag::WrappedType Tag::InitialValue = 0; - - } - } -} - -#define HEMELB_STEERING_DECLARE_TAG(name, type) \ -class name : \ - public hemelb::steering::_private::Tag \ - { \ - public: \ - typedef hemelb::steering::_private::Tag Super; \ - } - -#define HEMELB_STEERING_INITIALISE_TAG(name, value) \ - template<> const std::string name::Super::TagString = #name; \ - template<> const name::Super::WrappedType name::Super::InitialValue = value - -#endif // HEMELB_STEERING_COMMON_TAG_H diff --git a/Code/steering/common/TagBase.h b/Code/steering/common/TagBase.h deleted file mode 100644 index 05cb3b938..000000000 --- a/Code/steering/common/TagBase.h +++ /dev/null @@ -1,28 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_STEERING_COMMON_TAGBASE_H -#define HEMELB_STEERING_COMMON_TAGBASE_H - -#include -#include -namespace hemelb -{ - namespace steering - { - namespace _private - { - // Base class for Tags - class TagBase - { - }; - } - } -} -#endif // HEMELB_STEERING_COMMON_TAGBASE_H diff --git a/Code/steering/common/Tags.cc b/Code/steering/common/Tags.cc deleted file mode 100644 index 8ebdb322d..000000000 --- a/Code/steering/common/Tags.cc +++ /dev/null @@ -1,46 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include - -#include "steering/common/Tags.h" -#include "steering/common/Steerer.h" -namespace hemelb -{ - namespace steering - { - HEMELB_STEERING_INITIALISE_TAG(ctr_x, 0.); - HEMELB_STEERING_INITIALISE_TAG(ctr_y, 0.); - HEMELB_STEERING_INITIALISE_TAG(ctr_z, 0.); - - HEMELB_STEERING_INITIALISE_TAG(longitude, 45.); - HEMELB_STEERING_INITIALISE_TAG(latitude, 45.); - - HEMELB_STEERING_INITIALISE_TAG(zoom, 1.); - HEMELB_STEERING_INITIALISE_TAG(brightness, 0.03); - - HEMELB_STEERING_INITIALISE_TAG(velocity_max, 0.1); - HEMELB_STEERING_INITIALISE_TAG(stress_max, 0.1); - - HEMELB_STEERING_INITIALISE_TAG(pressure_min, 80.); - HEMELB_STEERING_INITIALISE_TAG(pressure_max, 120.); - - HEMELB_STEERING_INITIALISE_TAG(glyph_length, 1.); - - HEMELB_STEERING_INITIALISE_TAG(pixels_x, 512); - HEMELB_STEERING_INITIALISE_TAG(pixels_y, 512); - - HEMELB_STEERING_INITIALISE_TAG(terminate_signal, false); - HEMELB_STEERING_INITIALISE_TAG(vis_mode, 0); - HEMELB_STEERING_INITIALISE_TAG(streaklines_per_pulsatile_period, 5); - HEMELB_STEERING_INITIALISE_TAG(streakline_length, 100); - HEMELB_STEERING_INITIALISE_TAG(doRendering, false); - - } -} diff --git a/Code/steering/common/Tags.h b/Code/steering/common/Tags.h deleted file mode 100644 index 94305da22..000000000 --- a/Code/steering/common/Tags.h +++ /dev/null @@ -1,50 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_STEERING_COMMON_TAGS_H -#define HEMELB_STEERING_COMMON_TAGS_H - -#include "steering/common/Tag.h" -#include "steering/common/Steerer.h" - -namespace hemelb -{ - namespace steering - { - HEMELB_STEERING_DECLARE_TAG(ctr_x, float); - HEMELB_STEERING_DECLARE_TAG(ctr_y, float); - HEMELB_STEERING_DECLARE_TAG(ctr_z, float); - - HEMELB_STEERING_DECLARE_TAG(longitude, float); - HEMELB_STEERING_DECLARE_TAG(latitude, float); - - HEMELB_STEERING_DECLARE_TAG(zoom, float); - HEMELB_STEERING_DECLARE_TAG(brightness, float); - - HEMELB_STEERING_DECLARE_TAG(velocity_max, float); - HEMELB_STEERING_DECLARE_TAG(stress_max, float); - - HEMELB_STEERING_DECLARE_TAG(pressure_min, float); - HEMELB_STEERING_DECLARE_TAG(pressure_max, float); - - HEMELB_STEERING_DECLARE_TAG(glyph_length, float); - - HEMELB_STEERING_DECLARE_TAG(pixels_x, unsigned int); - HEMELB_STEERING_DECLARE_TAG(pixels_y, unsigned int); - - HEMELB_STEERING_DECLARE_TAG(terminate_signal, bool); - HEMELB_STEERING_DECLARE_TAG(vis_mode, int); - HEMELB_STEERING_DECLARE_TAG(streaklines_per_pulsatile_period, unsigned int); - HEMELB_STEERING_DECLARE_TAG(streakline_length, float); - HEMELB_STEERING_DECLARE_TAG(doRendering, bool); - - } -} - -#endif // HEMELB_STEERING_COMMON_TAGS_H diff --git a/Code/steering/none/ImageSendComponent.cc b/Code/steering/none/ImageSendComponent.cc deleted file mode 100644 index c97ba7bd8..000000000 --- a/Code/steering/none/ImageSendComponent.cc +++ /dev/null @@ -1,39 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "steering/ImageSendComponent.h" - -namespace hemelb -{ - namespace steering - { - ImageSendComponent::ImageSendComponent(lb::SimulationState* iSimState, - vis::Control* iControl, - const lb::LbmParameters* iLbmParams, - Network* iNetwork, - unsigned int inletCountIn): inletCount(inletCountIn), MaxFramerate(25.0) - { - - } - - ImageSendComponent::~ImageSendComponent() - { - } - - void ImageSendComponent::DoWork(const vis::PixelSet* pix) - { - - } - - bool ImageSendComponent::ShouldRenderNewNetworkImage() - { - return false; - } - } -} diff --git a/Code/steering/none/SteeringComponentN.cc b/Code/steering/none/SteeringComponentN.cc index 0756ea312..e18789faf 100644 --- a/Code/steering/none/SteeringComponentN.cc +++ b/Code/steering/none/SteeringComponentN.cc @@ -24,14 +24,13 @@ namespace hemelb * @param iSimState * @return */ - SteeringComponent::SteeringComponent(Network* network, vis::Control* iVisControl, - steering::ImageSendComponent* imageSendComponent, + SteeringComponent::SteeringComponent(Network* network, net::Net * iNet, lb::SimulationState * iSimState, configuration::SimConfig* iSimConfig, const util::UnitConverter* iUnits, reporting::Timers& timings) : net::CollectiveAction(iNet->GetCommunicator(), timings[reporting::Timers::steeringWait]), - mSimState(iSimState), mVisControl(iVisControl), imageSendComponent(imageSendComponent), + mSimState(iSimState), mUnits(iUnits), simConfig(iSimConfig), privateSteeringParams(STEERABLE_PARAMETERS + 1) { ClearValues(); diff --git a/Code/unittests/SimulationMasterTests.h b/Code/unittests/SimulationMasterTests.h index b7065fbc4..f4de7e312 100644 --- a/Code/unittests/SimulationMasterTests.h +++ b/Code/unittests/SimulationMasterTests.h @@ -30,14 +30,12 @@ namespace hemelb public: void setUp() { - argc = 7; + argc = 5; argv[0] = "hemelb"; - argv[2] = "four_cube.xml"; argv[1] = "-in"; - argv[3] = "-i"; - argv[4] = "1"; - argv[5] = "-ss"; - argv[6] = "1111"; + argv[2] = "four_cube.xml"; + argv[3] = "-ss"; + argv[4] = "1111"; FolderTestFixture::setUp(); CopyResourceToTempdir("four_cube.xml"); CopyResourceToTempdir("four_cube.gmy"); diff --git a/Code/unittests/configuration/CommandLineTests.h b/Code/unittests/configuration/CommandLineTests.h index 2f8421eba..7a2225cd8 100644 --- a/Code/unittests/configuration/CommandLineTests.h +++ b/Code/unittests/configuration/CommandLineTests.h @@ -34,14 +34,12 @@ namespace hemelb void setUp() { configFile = Resource("four_cube.xml").Path(); - argc = 7; + argc = 5; argv[0] = "hemelb"; - argv[2] = configFile.c_str(); argv[1] = "-in"; - argv[3] = "-i"; - argv[4] = "1"; - argv[5] = "-ss"; - argv[6] = "1111"; + argv[2] = configFile.c_str(); + argv[3] = "-ss"; + argv[4] = "1111"; FolderTestFixture::setUp(); options = new hemelb::configuration::CommandLine(argc, argv); } diff --git a/Code/unittests/io/PathManagerTests.h b/Code/unittests/io/PathManagerTests.h index 7bad97e5b..1996abfbb 100644 --- a/Code/unittests/io/PathManagerTests.h +++ b/Code/unittests/io/PathManagerTests.h @@ -32,15 +32,13 @@ namespace hemelb void setUp() { FolderTestFixture::setUp(); - argc = 7; + argc = 5; processorCount = 5; argv[0] = "hemelb"; - argv[2] = "config.xml"; argv[1] = "-in"; - argv[3] = "-i"; - argv[4] = "1"; - argv[5] = "-ss"; - argv[6] = "1111"; + argv[2] = "config.xml"; + argv[3] = "-ss"; + argv[4] = "1111"; } void tearDown() @@ -53,7 +51,6 @@ namespace hemelb { ConstructManager(); AssertPresent("results"); - AssertPresent("results/Images"); } void TestNameInventionLocalConfig() @@ -67,7 +64,6 @@ namespace hemelb { ConstructPathConfigManager(); AssertPresent("results"); - AssertPresent("results/Images"); } void TestNameInventionPathConfig() diff --git a/Code/unittests/main.cc b/Code/unittests/main.cc index 58fe2345c..1a355c655 100644 --- a/Code/unittests/main.cc +++ b/Code/unittests/main.cc @@ -16,7 +16,6 @@ #include #include "unittests/helpers/helpers.h" #include "unittests/lbtests/lbtests.h" -#include "unittests/vistests/vistests.h" #include "unittests/io/io.h" #include "unittests/reporting/reporting.h" #include "unittests/configuration/configuration.h" diff --git a/Code/unittests/multiscale/MockIntercommunicatorTests.h b/Code/unittests/multiscale/MockIntercommunicatorTests.h index 5b0486f92..5e453971a 100644 --- a/Code/unittests/multiscale/MockIntercommunicatorTests.h +++ b/Code/unittests/multiscale/MockIntercommunicatorTests.h @@ -210,15 +210,13 @@ namespace hemelb // TODO: This test is fatal if run with LADDIOLET. See ticket #605. LADD_FAIL(); int argc; - const char* argv[7]; - argc = 7; + const char* argv[5]; + argc = 5; argv[0] = "hemelb"; - argv[2] = "four_cube_multiscale.xml"; argv[1] = "-in"; - argv[3] = "-i"; - argv[4] = "1"; - argv[5] = "-ss"; - argv[6] = "1111"; + argv[2] = "four_cube_multiscale.xml"; + argv[3] = "-ss"; + argv[4] = "1111"; CopyResourceToTempdir("four_cube_multiscale.xml"); CopyResourceToTempdir("four_cube.gmy"); hemelb::configuration::CommandLine options(argc, argv); diff --git a/Code/unittests/reporting/ReporterTests.h b/Code/unittests/reporting/ReporterTests.h index fc36954ff..55c53c9fe 100644 --- a/Code/unittests/reporting/ReporterTests.h +++ b/Code/unittests/reporting/ReporterTests.h @@ -88,9 +88,6 @@ namespace hemelb } } mockTimers->Reduce(); // invoke the Timers MPI mock - reporter->Image(); - reporter->Image(); - reporter->Image(); for (unsigned int step = 0; step < 1000; step++) { state->Increment(); @@ -104,7 +101,6 @@ namespace hemelb "{{#PROCESSOR}}R{{RANK}}S{{SITES}} {{/PROCESSOR}}"); AssertTemplate(hemelb::reporting::mercurial_revision_number, "{{#BUILD}}{{REVISION}}{{/BUILD}}"); AssertTemplate(hemelb::reporting::build_time, "{{#BUILD}}{{TIME}}{{/BUILD}}"); - AssertValue("3", "IMAGES"); AssertValue("0.000100", "TIME_STEP_LENGTH"); AssertValue("1000", "TOTAL_TIME_STEPS"); AssertValue("1000", "STEPS"); diff --git a/Code/unittests/resources/config-velocity-iolet.xml b/Code/unittests/resources/config-velocity-iolet.xml index d47d880e7..bddcd22dd 100644 --- a/Code/unittests/resources/config-velocity-iolet.xml +++ b/Code/unittests/resources/config-velocity-iolet.xml @@ -35,16 +35,4 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/config.xml b/Code/unittests/resources/config.xml index 8b03a2cbc..6aeade27a 100644 --- a/Code/unittests/resources/config.xml +++ b/Code/unittests/resources/config.xml @@ -39,18 +39,6 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/config0_2_0.xml b/Code/unittests/resources/config0_2_0.xml index c89a0f1d3..40424f9d8 100644 --- a/Code/unittests/resources/config0_2_0.xml +++ b/Code/unittests/resources/config0_2_0.xml @@ -39,16 +39,4 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/config_file_inlet.xml b/Code/unittests/resources/config_file_inlet.xml index 99c220c94..3158b1e35 100644 --- a/Code/unittests/resources/config_file_inlet.xml +++ b/Code/unittests/resources/config_file_inlet.xml @@ -36,16 +36,4 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/config_file_velocity_inlet.xml b/Code/unittests/resources/config_file_velocity_inlet.xml index 6434ecfa6..2215ed0cc 100644 --- a/Code/unittests/resources/config_file_velocity_inlet.xml +++ b/Code/unittests/resources/config_file_velocity_inlet.xml @@ -37,16 +37,4 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/config_new_velocity_inlets.xml b/Code/unittests/resources/config_new_velocity_inlets.xml index ea8833e9a..8c0f3efbc 100644 --- a/Code/unittests/resources/config_new_velocity_inlets.xml +++ b/Code/unittests/resources/config_new_velocity_inlets.xml @@ -39,16 +39,4 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/four_cube.xml b/Code/unittests/resources/four_cube.xml index 0b7980253..4c9f7352d 100644 --- a/Code/unittests/resources/four_cube.xml +++ b/Code/unittests/resources/four_cube.xml @@ -39,18 +39,6 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/four_cube_multiscale.xml b/Code/unittests/resources/four_cube_multiscale.xml index ace4c1c28..25536518e 100644 --- a/Code/unittests/resources/four_cube_multiscale.xml +++ b/Code/unittests/resources/four_cube_multiscale.xml @@ -37,18 +37,6 @@ - - - - - - - - - - - - diff --git a/Code/unittests/vistests/HslToRgbConvertorTests.h b/Code/unittests/vistests/HslToRgbConvertorTests.h deleted file mode 100644 index 0b0bfd700..000000000 --- a/Code/unittests/vistests/HslToRgbConvertorTests.h +++ /dev/null @@ -1,63 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_UNITTESTS_VISTESTS_HSLTORGBCONVERTORTESTS_H -#define HEMELB_UNITTESTS_VISTESTS_HSLTORGBCONVERTORTESTS_H - -#include - -namespace hemelb -{ - namespace unittests - { - namespace vistests - { - class HslToRgbConvertorTests : public CppUnit::TestFixture - { - CPPUNIT_TEST_SUITE( HslToRgbConvertorTests ); - CPPUNIT_TEST( TestColours ); - CPPUNIT_TEST_SUITE_END(); - public: - void TestColours() - { - DoColourTest(0, 0, 0, 0, 0, 0); - DoColourTest(1, 1, 1, 255, 255, 255); - DoColourTest(250, 0.5, 0.5, 84, 63, 191); - } - - private: - void DoColourTest(float iHue, - float iSaturation, - float iLightness, - unsigned char iExpectedRed, - unsigned char iExpectedGreen, - unsigned char iExpectedBlue) - { - unsigned char lRGB[3]; - - hemelb::vis::raytracer::HSLToRGBConverter::Convert(iHue, iSaturation, iLightness, lRGB); - - CPPUNIT_ASSERT_EQUAL_MESSAGE("HslRgbConvertor test differed on red channel", - iExpectedRed, - lRGB[0]); - CPPUNIT_ASSERT_EQUAL_MESSAGE("HslRgbConvertor test differed on green channel", - iExpectedGreen, - lRGB[1]); - CPPUNIT_ASSERT_EQUAL_MESSAGE("HslRgbConvertor test differed on blue channel", - iExpectedBlue, - lRGB[2]); - } - - }; - CPPUNIT_TEST_SUITE_REGISTRATION( HslToRgbConvertorTests ); - } - } -} - -#endif diff --git a/Code/unittests/vistests/vistests.h b/Code/unittests/vistests/vistests.h deleted file mode 100644 index c5303718f..000000000 --- a/Code/unittests/vistests/vistests.h +++ /dev/null @@ -1,15 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_UNITTESTS_VISTESTS_VISTESTS_H -#define HEMELB_UNITTESTS_VISTESTS_VISTESTS_H - -#include "unittests/vistests/HslToRgbConvertorTests.h" - -#endif /* HEMELB_UNITTESTS_VISTESTS_VISTESTS_H */ diff --git a/Code/vis/BasicPixel.cc b/Code/vis/BasicPixel.cc deleted file mode 100644 index a5cbdb5df..000000000 --- a/Code/vis/BasicPixel.cc +++ /dev/null @@ -1,84 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "vis/BasicPixel.h" - -namespace hemelb -{ - namespace vis - { - BasicPixel::BasicPixel() - { - - } - BasicPixel::BasicPixel(int iIn, int jIn) : - i(iIn), j(jIn) - { - - } - - int BasicPixel::GetI() const - { - return i; - } - - int BasicPixel::GetJ() const - { - return j; - } - - bool BasicPixel::operator <(const BasicPixel& right) const - { - return (i < right.i || (i == right.i && j < right.j)); - } - - bool BasicPixel::operator ==(const BasicPixel& right) const - { - return (i == right.i && j == right.j); - } - - MPI_Datatype BasicPixel::GetMPIType() - { - const int typeCount = 2; - int blocklengths[typeCount] = { 1, 1 }; - - MPI_Datatype types[typeCount] = { MPI_INT, MPI_INT }; - - BasicPixel example; - - MPI_Aint displacements[typeCount]; - - MPI_Get_address(&example.i, &displacements[0]); - MPI_Get_address(&example.j, &displacements[1]); - - displacements[1] -= displacements[0]; - displacements[0] = 0; - - MPI_Datatype ret; - - HEMELB_MPI_CALL( - MPI_Type_create_struct, - (typeCount, blocklengths, displacements, types, &ret) - ); - - return ret; - } - } - - namespace net - { - template<> - MPI_Datatype net::MpiDataTypeTraits::RegisterMpiDataType() - { - MPI_Datatype ret = vis::BasicPixel::GetMPIType(); - HEMELB_MPI_CALL(MPI_Type_commit, (&ret)); - return ret; - } - } -} diff --git a/Code/vis/BasicPixel.h b/Code/vis/BasicPixel.h deleted file mode 100644 index 7f8f45da0..000000000 --- a/Code/vis/BasicPixel.h +++ /dev/null @@ -1,59 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_BASICPIXEL_H -#define HEMELB_VIS_BASICPIXEL_H - -#include "net/mpi.h" - -namespace hemelb -{ - namespace vis - { - class BasicPixel - { - public: - BasicPixel(); - BasicPixel(int iIn, int jIn); - - int GetI() const; - int GetJ() const; - - bool operator <(const BasicPixel& right) const; - bool operator ==(const BasicPixel& right) const; - - /** - * NB This is not virtual, but it will be shadowed by derived classes. - * In the base case, we don't combine the pixels as they aren't associated - * with any data. - */ - void Combine(const BasicPixel& other) const - { - - } - - /** - * Produces an MPI Datatype object but doesn't commit it or manage its memory. - * @return - */ - static MPI_Datatype GetMPIType(); - - protected: - int i, j; - }; - } - - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); - } -} - -#endif /* HEMELB_VIS_BASICPIXEL_H */ diff --git a/Code/vis/CMakeLists.txt b/Code/vis/CMakeLists.txt deleted file mode 100644 index 9bf03a07e..000000000 --- a/Code/vis/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -add_library(hemelb_vis - GlyphDrawer.cc Control.cc Screen.cc Viewpoint.cc BasicPixel.cc Rendering.cc ResultPixel.cc - rayTracer/ClusterNormal.cc rayTracer/ClusterWithWallNormals.cc rayTracer/HSLToRGBConverter.cc - rayTracer/RayDataEnhanced.cc rayTracer/RayDataNormal.cc - streaklineDrawer/NeighbouringProcessor.cc streaklineDrawer/Particle.cc streaklineDrawer/ParticleManager.cc - streaklineDrawer/StreaklineDrawer.cc streaklineDrawer/VelocityField.cc streaklineDrawer/StreakPixel.cc - ) diff --git a/Code/vis/Control.cc b/Code/vis/Control.cc deleted file mode 100644 index 6f2af0ebc..000000000 --- a/Code/vis/Control.cc +++ /dev/null @@ -1,637 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include -#include -#include - -#include "log/Logger.h" -#include "util/utilityFunctions.h" -#include "vis/Control.h" -#include "vis/rayTracer/RayTracer.h" -#include "vis/GlyphDrawer.h" - -#include "io/writers/xdr/XdrFileWriter.h" - -namespace hemelb - -{ - namespace vis - { - Control::Control(lb::StressTypes iStressType, - net::Net* netIn, - lb::SimulationState* simState, - const lb::MacroscopicPropertyCache& propertyCache, - geometry::LatticeData* iLatDat, - reporting::Timer &atimer) : - net::PhasedBroadcastIrregular(netIn, simState, SPREADFACTOR), - propertyCache(propertyCache), latticeData(iLatDat), timer(atimer) - { - - visSettings.mStressType = iStressType; - - this->vis = new Vis; - - //sites_x etc are globals declared in net.h - vis->half_dim = util::Vector3D(iLatDat->GetSiteDimensions()) * 0.5F; - - vis->system_size = 2.F * fmaxf(vis->half_dim[0], fmaxf(vis->half_dim[1], vis->half_dim[2])); - - visSettings.mouse_x = -1; - visSettings.mouse_y = -1; - - initLayers(); - } - - void Control::initLayers() - { - // We don't have all the minima / maxima on one core, so we have to gather them. - // NOTE this only happens once, during initialisation, otherwise it would be - // totally unforgivable. - site_t block_min_x = std::numeric_limits < site_t > ::max(); - site_t block_min_y = std::numeric_limits < site_t > ::max(); - site_t block_min_z = std::numeric_limits < site_t > ::max(); - site_t block_max_x = std::numeric_limits < site_t > ::min(); - site_t block_max_y = std::numeric_limits < site_t > ::min(); - site_t block_max_z = std::numeric_limits < site_t > ::min(); - - for (geometry::BlockTraverser blockIt(*latticeData); blockIt.CurrentLocationValid(); blockIt.TraverseOne()) - { - if (blockIt.GetCurrentBlockData().IsEmpty()) - { - continue; - } - - block_min_x = util::NumericalFunctions::min(block_min_x, blockIt.GetCurrentLocation().x); - block_min_y = util::NumericalFunctions::min(block_min_y, blockIt.GetCurrentLocation().y); - block_min_z = util::NumericalFunctions::min(block_min_z, blockIt.GetCurrentLocation().z); - block_max_x = util::NumericalFunctions::max(block_max_x, blockIt.GetCurrentLocation().x); - block_max_y = util::NumericalFunctions::max(block_max_y, blockIt.GetCurrentLocation().y); - block_max_z = util::NumericalFunctions::max(block_max_z, blockIt.GetCurrentLocation().z); - } - - std::vector localMins(3), localMaxes(3); - - localMins[0] = block_min_x; - localMins[1] = block_min_y; - localMins[2] = block_min_z; - - localMaxes[0] = block_max_x; - localMaxes[1] = block_max_y; - localMaxes[2] = block_max_z; - - const net::MpiCommunicator& comms = this->mNet->GetCommunicator(); - std::vector mins = comms.AllReduce(localMins, MPI_MIN); - std::vector maxes = comms.AllReduce(localMaxes, MPI_MAX); - - visSettings.ctr_x = 0.5F * (float) (latticeData->GetBlockSize() * (mins[0] + maxes[0])); - visSettings.ctr_y = 0.5F * (float) (latticeData->GetBlockSize() * (mins[1] + maxes[1])); - visSettings.ctr_z = 0.5F * (float) (latticeData->GetBlockSize() * (mins[2] + maxes[2])); - - normalRayTracer = - new raytracer::RayTracer(latticeData, - &domainStats, - &screen, - &viewpoint, - &visSettings); - - myGlypher = new GlyphDrawer(latticeData, &screen, &domainStats, &viewpoint, &visSettings); - -#ifndef NO_STREAKLINES - myStreaker = new streaklinedrawer::StreaklineDrawer(*latticeData, screen, viewpoint, visSettings, propertyCache, mNet->GetCommunicator()); -#else - myStreaker = NULL; -#endif - // Note that rtInit does stuff to this->ctr_x (because this has - // to be global) - visSettings.ctr_x -= vis->half_dim[0]; - visSettings.ctr_y -= vis->half_dim[1]; - visSettings.ctr_z -= vis->half_dim[2]; - } - - void Control::SetProjection(const int &iPixels_x, - const int &iPixels_y, - const float &iLocal_ctr_x, - const float &iLocal_ctr_y, - const float &iLocal_ctr_z, - const float &iLongitude, - const float &iLatitude, - const float &iZoom) - { - float rad = 5.F * vis->system_size; - float dist = 0.5F * rad; - - //For now set the maximum draw distance to twice the radius; - visSettings.maximumDrawDistance = 2.0F * rad; - - util::Vector3D centre = util::Vector3D(iLocal_ctr_x, iLocal_ctr_y, iLocal_ctr_z); - - viewpoint.SetViewpointPosition(iLongitude * (float) DEG_TO_RAD, - iLatitude * (float) DEG_TO_RAD, - centre, - rad, - dist); - - screen.Set( (0.5F * vis->system_size) / iZoom, - (0.5F * vis->system_size) / iZoom, - iPixels_x, - iPixels_y, - rad, - &viewpoint); - } - - void Control::SetSomeParams(const float iBrightness, - const distribn_t iDensityThresholdMin, - const distribn_t iDensityThresholdMinMaxInv, - const distribn_t iVelocityThresholdMaxInv, - const distribn_t iStressThresholdMaxInv) - { - visSettings.brightness = iBrightness; - domainStats.density_threshold_min = iDensityThresholdMin; - - domainStats.density_threshold_minmax_inv = iDensityThresholdMinMaxInv; - domainStats.velocity_threshold_max_inv = iVelocityThresholdMaxInv; - domainStats.stress_threshold_max_inv = iStressThresholdMaxInv; - } - - void Control::UpdateImageSize(int pixels_x, int pixels_y) - { - screen.Resize(pixels_x, pixels_y); - } - - void Control::Render(unsigned long startIteration) - { - log::Logger::Log("Rendering."); - - PixelSet* ray = normalRayTracer->Render(propertyCache); - - PixelSet *glyph = NULL; - - if (visSettings.mode == VisSettings::ISOSURFACESANDGLYPHS) - { - glyph = myGlypher->Render(propertyCache); - } - else - { - glyph = myGlypher->GetUnusedPixelSet(); - glyph->Clear(); - } - - PixelSet *streak = NULL; - - if (myStreaker != NULL - && (visSettings.mStressType == lb::ShearStress || visSettings.mode == VisSettings::WALLANDSTREAKLINES)) - { - streak = myStreaker->Render(); - } - - localResultsByStartIt.insert(std::pair(startIteration, Rendering(glyph, ray, streak))); - } - - void Control::InitialAction(unsigned long startIteration) - { - timer.Start(); - - Render(startIteration); - - log::Logger::Log("Render stored for phased imaging."); - - timer.Stop(); - } - - void Control::WriteImage(io::writers::Writer* writer, - const PixelSet& imagePixels, - const DomainStats& domainStats, - const VisSettings& visSettings) const - { - *writer << (int) visSettings.mode; - - *writer << domainStats.physical_pressure_threshold_min << domainStats.physical_pressure_threshold_max - << domainStats.physical_velocity_threshold_max << domainStats.physical_stress_threshold_max; - - *writer << screen.GetPixelsX(); - *writer << screen.GetPixelsY(); - *writer << (int) imagePixels.GetPixelCount(); - - WritePixels(writer, imagePixels, domainStats, visSettings); - } - - int Control::GetPixelsX() const - { - return screen.GetPixelsX(); - } - - int Control::GetPixelsY() const - { - return screen.GetPixelsY(); - } - - void Control::WritePixels(io::writers::Writer* writer, - const PixelSet& imagePixels, - const DomainStats& domainStats, - const VisSettings& visSettings) const - { - const int bits_per_char = sizeof(char) * 8; - - for (unsigned int i = 0; i < imagePixels.GetPixelCount(); i++) - { - const ResultPixel& pixel = imagePixels.GetPixels()[i]; - - // Use a ray-tracer function to get the necessary pixel data. - unsigned index; - unsigned char rgb_data[12]; - - pixel.WritePixel(&index, rgb_data, domainStats, visSettings); - - *writer << (uint32_t) index; - - unsigned pix_data[3]; - pix_data[0] = (rgb_data[0] << (3 * bits_per_char)) + (rgb_data[1] << (2 * bits_per_char)) - + (rgb_data[2] << bits_per_char) + rgb_data[3]; - - pix_data[1] = (rgb_data[4] << (3 * bits_per_char)) + (rgb_data[5] << (2 * bits_per_char)) - + (rgb_data[6] << bits_per_char) + rgb_data[7]; - - pix_data[2] = (rgb_data[8] << (3 * bits_per_char)) + (rgb_data[9] << (2 * bits_per_char)) - + (rgb_data[10] << bits_per_char) + rgb_data[11]; - - for (int i = 0; i < 3; i++) - { - *writer << (uint32_t) pix_data[i]; - } - *writer << io::writers::Writer::eol; - } - } - - void Control::ProgressFromChildren(unsigned long startIteration, unsigned long splayNumber) - { - timer.Start(); - - if (splayNumber == 0) - { - for (unsigned int ii = 0; ii < GetChildren().size(); ++ii) - { - Rendering lRendering(myGlypher->GetUnusedPixelSet(), - normalRayTracer->GetUnusedPixelSet(), - myStreaker != NULL ? - myStreaker->GetUnusedPixelSet() : - NULL); - - lRendering.ReceivePixelCounts(mNet, GetChildren()[ii]); - - childrenResultsByStartIt.insert(std::pair(startIteration, lRendering)); - } - - log::Logger::Log("Receiving child image pixel count."); - } - else if (splayNumber == 1) - { - std::multimap::iterator renderings = - childrenResultsByStartIt.equal_range(startIteration).first; - - for (unsigned int ii = 0; ii < GetChildren().size(); ++ii) - { - Rendering& received = (*renderings).second; - - log::Logger::Log("Receiving child image pixel data (from it %li).", - startIteration); - - received.ReceivePixelData(mNet, GetChildren()[ii]); - - renderings++; - } - } - - timer.Stop(); - } - - void Control::ProgressToParent(unsigned long startIteration, unsigned long splayNumber) - { - timer.Start(); - - Rendering& rendering = (*localResultsByStartIt.find(startIteration)).second; - if (splayNumber == 0) - { - log::Logger::Log("Sending pixel count (from it %li).", startIteration); - - rendering.SendPixelCounts(mNet, GetParent()); - } - else if (splayNumber == 1) - { - log::Logger::Log("Sending pixel data (from it %li).", startIteration); - - rendering.SendPixelData(mNet, GetParent()); - } - - timer.Stop(); - } - - void Control::PostReceiveFromChildren(unsigned long startIteration, unsigned long splayNumber) - { - timer.Start(); - - // The first time round, ensure that we have enough memory to receive the image data that - // will come next time. - if (splayNumber == 0) - { - - } - if (splayNumber == 1) - { - std::pair < std::multimap::iterator , std::multimap::iterator - > its = childrenResultsByStartIt.equal_range(startIteration); - - Rendering local = (*localResultsByStartIt.find(startIteration)).second; - - std::multimap::iterator rendering = its.first; - while (rendering != its.second) - { - local.Combine( (*rendering).second); - - (*rendering).second.ReleaseAll(); - - rendering++; - } - - if (its.first != its.second) - { - its.first++; - childrenResultsByStartIt.erase(its.first, its.second); - } - - log::Logger::Log("Combining in child pixel data."); - } - - timer.Stop(); - } - - void Control::PostSendToParent(unsigned long startIteration, unsigned long splayNumber) - { - if (splayNumber == 1) - { - Rendering& rendering = (*localResultsByStartIt.find(startIteration)).second; - rendering.ReleaseAll(); - - localResultsByStartIt.erase(startIteration); - } - } - - bool Control::IsRendering() const - { - return IsInitialAction() || IsInstantBroadcast(); - } - - void Control::ClearOut(unsigned long startIt) - { - timer.Start(); - - bool found; - - do - { - found = false; - - if (localResultsByStartIt.size() > 0) - { - mapType::iterator it = localResultsByStartIt.begin(); - if (it->first <= startIt) - { - log::Logger::Log("Clearing out image cache from it %lu", it->first); - - (*it).second.ReleaseAll(); - - localResultsByStartIt.erase(it); - found = true; - } - } - } - while (found); - - do - { - found = false; - - if (childrenResultsByStartIt.size() > 0) - { - multimapType::iterator it = childrenResultsByStartIt.begin(); - if ( (*it).first <= startIt) - { - log::Logger::Log("Clearing out image cache from it %lu", (*it).first); - - (*it).second.ReleaseAll(); - - childrenResultsByStartIt.erase(it); - found = true; - } - } - } - while (found); - - do - { - found = false; - - if (renderingsByStartIt.size() > 0) - { - std::multimap*>::iterator it = renderingsByStartIt.begin(); - if ( (*it).first <= startIt) - { - log::Logger::Log("Clearing out image cache from it %lu", (*it).first); - - (*it).second->Release(); - renderingsByStartIt.erase(it); - found = true; - } - } - } - while (found); - - timer.Stop(); - } - - const PixelSet* Control::GetResult(unsigned long startIt) - { - log::Logger::Log("Getting image results from it %lu", startIt); - - if (renderingsByStartIt.count(startIt) != 0) - { - return (*renderingsByStartIt.find(startIt)).second; - } - - if (localResultsByStartIt.count(startIt) != 0) - { - Rendering finalRender = (*localResultsByStartIt.find(startIt)).second; - PixelSet *result = GetUnusedPixelSet(); - - finalRender.PopulateResultSet(result); - - renderingsByStartIt.insert(std::pair*>(startIt, result)); - return result; - } - else - { - return NULL; - } - } - - void Control::InstantBroadcast(unsigned long startIteration) - { - timer.Start(); - - log::Logger::Log("Performing instant imaging."); - - Render(startIteration); - - /* - * We do several iterations. - * - * On the first, every even proc passes data to the odd proc below, where it is merged. - * On the second, the difference is two, so proc 3 passes to 1, 7 to 5, 11 to 9 etc. - * On the third the differenec is four, so proc 5 passes to 1, 13 to 9 etc. - * . - * . - * . - * - * This continues until all data is passed back to processor one, which passes it to proc 0. - */ - - const net::MpiCommunicator& netComm = this->mNet->GetCommunicator(); - net::Net tempNet(netComm); - - Rendering* localBuffer = localResultsByStartIt.count(startIteration) > 0 ? - & (*localResultsByStartIt.find(startIteration)).second : - NULL; - Rendering receiveBuffer(myGlypher->GetUnusedPixelSet(), normalRayTracer->GetUnusedPixelSet(), myStreaker == NULL ? - NULL : - myStreaker->GetUnusedPixelSet()); - - // Start with a difference in rank of 1, doubling every time. - for (proc_t deltaRank = 1; deltaRank < netComm.Size(); deltaRank <<= 1) - { - // The receiving proc is all the ranks that are 1 modulo (deltaRank * 2) - for (proc_t receivingProc = 1; receivingProc < (netComm.Size() - deltaRank); - receivingProc += deltaRank << 1) - { - proc_t sendingProc = receivingProc + deltaRank; - - // If we're the sending proc, do the send. - if (netComm.Rank() == sendingProc) - { - localBuffer->SendPixelCounts(&tempNet, receivingProc); - - tempNet.Dispatch(); - - localBuffer->SendPixelData(&tempNet, receivingProc); - - tempNet.Dispatch(); - } - - // If we're the receiving proc, receive. - - else if (netComm.Rank() == receivingProc) - { - receiveBuffer.ReceivePixelCounts(&tempNet, sendingProc); - - tempNet.Dispatch(); - - receiveBuffer.ReceivePixelData(&tempNet, sendingProc); - - tempNet.Dispatch(); - - localBuffer->Combine(receiveBuffer); - } - } - } - - // Send the final image from proc 1 to 0. - if (netComm.Rank() == 1) - { - localBuffer->SendPixelCounts(&tempNet, 0); - - tempNet.Dispatch(); - - localBuffer->SendPixelData(&tempNet, 0); - - tempNet.Dispatch(); - } - // Receive the final image on proc 0. - else if (netComm.Rank() == 0 && netComm.Size() > 1) - { - receiveBuffer.ReceivePixelCounts(&tempNet, 1); - - tempNet.Dispatch(); - - receiveBuffer.ReceivePixelData(&tempNet, 1); - - tempNet.Dispatch(); - - localResultsByStartIt.erase(startIteration); - localResultsByStartIt.insert(std::pair(startIteration, Rendering(receiveBuffer))); - - log::Logger::Log("Inserting image at it %lu.", startIteration); - } - - if (netComm.Rank() != 0) - { - receiveBuffer.ReleaseAll(); - } - - timer.Stop(); - } - - void Control::SetMouseParams(double iPhysicalPressure, double iPhysicalStress) - { - visSettings.mouse_pressure = iPhysicalPressure; - visSettings.mouse_stress = iPhysicalStress; - } - - bool Control::MouseIsOverPixel(const PixelSet* result, float* density, float* stress) - { - if (visSettings.mouse_x < 0 || visSettings.mouse_y < 0) - { - return false; - } - - const std::vector& screenPix = result->GetPixels(); - - for (std::vector::const_iterator it = screenPix.begin(); it != screenPix.end(); ++it) - { - if ( (*it).GetRayPixel() != NULL && (*it).GetI() == visSettings.mouse_x && (*it).GetJ() == visSettings.mouse_y) - { - *density = (*it).GetRayPixel()->GetNearestDensity(); - *stress = (*it).GetRayPixel()->GetNearestStress(); - - return true; - } - } - - return false; - } - - void Control::ProgressStreaklines(unsigned long time_step, unsigned long period) - { - if (myStreaker != NULL) - { - timer.Start(); - myStreaker->ProgressStreaklines(time_step, period); - timer.Stop(); - } - } - - Control::~Control() - { - delete myStreaker; - delete vis; - delete myGlypher; - delete normalRayTracer; - } - - } // namespace vis -} -// namespace hemelb diff --git a/Code/vis/Control.h b/Code/vis/Control.h deleted file mode 100644 index b54705c97..000000000 --- a/Code/vis/Control.h +++ /dev/null @@ -1,154 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_CONTROL_H -#define HEMELB_VIS_CONTROL_H - -#include - -#include "geometry/LatticeData.h" - -#include "lb/LbmParameters.h" -#include "lb/SimulationState.h" - -#include "net/net.h" -#include "net/PhasedBroadcastIrregular.h" - -#include "vis/DomainStats.h" -#include "vis/GlyphDrawer.h" -#include "vis/rayTracer/ClusterWithWallNormals.h" -#include "vis/rayTracer/RayDataNormal.h" -#include "vis/rayTracer/RayDataEnhanced.h" -#include "vis/rayTracer/RayTracer.h" -#include "vis/Rendering.h" -#include "vis/ResultPixel.h" -#include "vis/Screen.h" -#include "vis/streaklineDrawer/StreaklineDrawer.h" -#include "vis/Viewpoint.h" -#include "vis/VisSettings.h" -#include "reporting/Timers.h" - -namespace hemelb -{ - namespace vis - { - /** - * Class to control and use the effects of different visualisation methods. - * - * We use irregular phased broadcasting because we don't know in advance which iterations we'll - * need to generate images on. - * - * The initial action is used to render the image on each core. 2 communications are required - * between each pair of nodes so that the number of pixels can be communicated before the pixels - * themselves. No overlap is possible between communications at different depths as the pixels - * must be merged before they can be passed on. We don't need to pass info top-down, we only - * pass image components upwards towards the top node. - */ - class Control : public net::PhasedBroadcastIrregular, - private PixelSetStore > - { - public: - Control(lb::StressTypes iStressType, - net::Net* net, - lb::SimulationState* simState, - const lb::MacroscopicPropertyCache& propertyCache, - geometry::LatticeData* iLatDat, - reporting::Timer &atimer); - ~Control(); - - void SetSomeParams(const float iBrightness, - const distribn_t iDensityThresholdMin, - const distribn_t iDensityThresholdMinMaxInv, - const distribn_t iVelocityThresholdMaxInv, - const distribn_t iStressThresholdMaxInv); - - void SetProjection(const int &pixels_x, - const int &pixels_y, - const float &ctr_x, - const float &ctr_y, - const float &ctr_z, - const float &longitude, - const float &latitude, - const float &zoom); - - bool MouseIsOverPixel(const PixelSet* result, float* density, float* stress); - - void ProgressStreaklines(unsigned long time_step, unsigned long period); - - void UpdateImageSize(int pixels_x, int pixels_y); - void SetMouseParams(double iPhysicalPressure, double iPhysicalStress); - - const PixelSet* GetResult(unsigned long startIteration); - - void WritePixels(io::writers::Writer* writer, - const PixelSet& imagePixels, - const DomainStats& domainStats, - const VisSettings& visSettings) const; - void WriteImage(io::writers::Writer* writer, - const PixelSet& imagePixels, - const DomainStats& domainStats, - const VisSettings& visSettings) const; - - bool IsRendering() const; - - int GetPixelsX() const; - int GetPixelsY() const; - - Viewpoint viewpoint; - DomainStats domainStats; - VisSettings visSettings; - - protected: - void InitialAction(unsigned long startIteration); - void ProgressFromChildren(unsigned long startIteration, unsigned long splayNumber); - void ProgressToParent(unsigned long startIteration, unsigned long splayNumber); - void PostReceiveFromChildren(unsigned long startIteration, unsigned long splayNumber); - void PostSendToParent(unsigned long startIteration, unsigned long splayNumber); - void ClearOut(unsigned long startIteration); - void InstantBroadcast(unsigned long startIteration); - - private: - typedef std::map mapType; - typedef std::multimap multimapType; - - // This is mainly constrained by the memory available per core. - static const unsigned int SPREADFACTOR = 2; - - struct Vis - { - util::Vector3D half_dim; - float system_size; - }; - - void initLayers(); - void Render(unsigned long startIteration); - - mapType localResultsByStartIt; - multimapType childrenResultsByStartIt; - std::multimap*> renderingsByStartIt; - - /** - * Cache of all the macroscopic fluid properties of each lattice site on this core. - */ - const lb::MacroscopicPropertyCache& propertyCache; - - geometry::LatticeData* latticeData; - Screen screen; - Vis* vis; - raytracer::RayTracer - *normalRayTracer; - GlyphDrawer *myGlypher; - streaklinedrawer::StreaklineDrawer *myStreaker; - - reporting::Timer &timer; - }; - } -} - -#endif // HEMELB_VIS_CONTROL_H diff --git a/Code/vis/DomainStats.h b/Code/vis/DomainStats.h deleted file mode 100644 index c0d036bb9..000000000 --- a/Code/vis/DomainStats.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_DOMAINSTATS_H -#define HEMELB_VIS_DOMAINSTATS_H - -#include "constants.h" - -namespace hemelb -{ - namespace vis - { - struct DomainStats - { - public: - distribn_t velocity_threshold_max_inv; - distribn_t stress_threshold_max_inv; - distribn_t density_threshold_min; - distribn_t density_threshold_minmax_inv; - float physical_pressure_threshold_min; - float physical_pressure_threshold_max; - float physical_velocity_threshold_max; - float physical_stress_threshold_max; - }; - } -} - -#endif /* HEMELB_VIS_DOMAINSTATS_H */ - diff --git a/Code/vis/GlyphDrawer.cc b/Code/vis/GlyphDrawer.cc deleted file mode 100644 index 6aad5d0bf..000000000 --- a/Code/vis/GlyphDrawer.cc +++ /dev/null @@ -1,217 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "vis/GlyphDrawer.h" -#include "vis/Control.h" - -namespace hemelb -{ - namespace vis - { - // Constructor - GlyphDrawer::GlyphDrawer(geometry::LatticeData* iLatDat, - Screen* iScreen, - DomainStats* iDomainStats, - Viewpoint* iViewpoint, - VisSettings* iVisSettings) : - mLatDat(iLatDat), mScreen(iScreen), mDomainStats(iDomainStats), mViewpoint(iViewpoint), mVisSettings(iVisSettings) - { - for (geometry::BlockTraverser blockTrav(*mLatDat); blockTrav.CurrentLocationValid(); blockTrav.TraverseOne()) - { - // Get the block data for this block - if it has no site data, move on. - const geometry::Block& block = blockTrav.GetCurrentBlockData(); - - if (block.IsEmpty()) - { - continue; - } - - // We put the glyph at the site at the centre of the block... - const util::Vector3D midBlockSite = util::Vector3D(mLatDat->GetBlockSize()) / (site_t) 2; - - const site_t siteIdOnBlock = mLatDat->GetLocalSiteIdFromLocalSiteCoords(midBlockSite); - - // ... (only if there's fluid there). - if (block.SiteIsSolid(siteIdOnBlock)) - { - continue; - } - - // Create a glyph at the desired location - Glyph lGlyph; - - util::Vector3D globalSiteCoords = mLatDat->GetGlobalCoords(blockTrav.GetCurrentLocation(), - midBlockSite); - - lGlyph.x = float(globalSiteCoords.x) - 0.5F * float(mLatDat->GetSiteDimensions().x); - lGlyph.y = float(globalSiteCoords.y) - 0.5F * float(mLatDat->GetSiteDimensions().y); - lGlyph.z = float(globalSiteCoords.z) - 0.5F * float(mLatDat->GetSiteDimensions().z); - - lGlyph.siteId = block.GetLocalContiguousIndexForSite(siteIdOnBlock); - - mGlyphs.push_back(lGlyph); - } - } - - /** - * Destructor - */ - GlyphDrawer::~GlyphDrawer() - { - } - - /** - * Render a line between two points on the screen. - * - * @param endPoint1 - * @param endPoint2 - * @param iStressType - * @param mode - */ - void GlyphDrawer::RenderLine(const XYCoordinates& endPoint1, - const XYCoordinates& endPoint2, - const VisSettings* visSettings, - PixelSet* pixelSet) - { - // Store end points of the line and 'current' point (x and y). - int x = int(endPoint1.x); - int y = int(endPoint1.y); - - // Ensure increasing x from point 1 to point 2. - int x1, y1, x2, y2; - if (endPoint2.x < endPoint1.x) - { - x1 = int(endPoint2.x); - y1 = int(endPoint2.y); - x2 = x; - y2 = y; - } - else - { - x1 = x; - y1 = y; - x2 = int(endPoint2.x); - y2 = int(endPoint2.y); - } - - // Ensure increasing y. - if (y2 <= y1) - { - int temp = y2; - y2 = y; - y = temp; - } - - // Set dx with the difference between x-values at the endpoints. - int dx = x2 - x1; - int dy = y2 - y; - - if (dx > dy) - { - RenderLineHelper(x, y, dy, dy - dx, x2, visSettings, pixelSet); - } - else - { - RenderLineHelper(x, y, dx, dx - dy, y2, visSettings, pixelSet); - } - } - - /** - * Helper function for rendering a line. The template parameter indicates whether the end of - * the line will be limited by the x or y dimension. - * - * @tparam xLimited True if the line is limited by an x value, false if limited by a y value. - * @param x Starting, minimal value of x. - * @param y Starting, minimal value of y. - * @param smallerDelta is the smaller of the two deltas between initial- and end- x and y. - * @param deltaDifference is (smaller delta) - (larger delta) - * @param limit Maximal value in the limiting direction - */ - template - void GlyphDrawer::RenderLineHelper(int x, - int y, - const int smallerDelta, - const int deltaDifference, - const int limit, - const VisSettings* visSettings, - PixelSet* pixelSet) - { - // This variable tracks whether we are above or below the line we draw. - int overOrUnderMeasure = deltaDifference; - - while ( (xLimited && x <= limit) || (!xLimited && y <= limit)) - { - // If on screen, add a pixel. - if (x >= 0 && x < (int) mScreen->GetPixelsX() && y >= 0 && y < (int) mScreen->GetPixelsY()) - { - BasicPixel pixel(x, y); - pixelSet->AddPixel(pixel); - } - - // We are effectively adding the smallerDelta every time (and increment in the other direction), - // and we periodically subtract a largerDelta (and increment in the limited direction). - if (overOrUnderMeasure < 0) - { - overOrUnderMeasure += smallerDelta; - if (xLimited) - { - ++x; - } - else - { - ++y; - } - } - else - { - overOrUnderMeasure += deltaDifference; - ++y; - ++x; - } - } // end while - } - - /** - * Perform the rendering for each glyph. - */ - PixelSet* GlyphDrawer::Render(const lb::MacroscopicPropertyCache& propertyCache) - { - PixelSet* pixelSet = GetUnusedPixelSet(); - - pixelSet->Clear(); - - // For each glyph... - for (site_t n = 0; n < (site_t) mGlyphs.size(); n++) - { - // ... get the velocity at that point... - const util::Vector3D& velocity = propertyCache.velocityCache.Get(mGlyphs[n].siteId); - - // ... calculate the velocity vector multiplier... - const double temp = mVisSettings->glyphLength * ((distribn_t) mLatDat->GetBlockSize()) - * mDomainStats->velocity_threshold_max_inv; - - // ... calculate the two ends of the line we're going to draw... - util::Vector3D p1 = util::Vector3D(mGlyphs[n].x, mGlyphs[n].y, mGlyphs[n].z); - util::Vector3D p2 = p1 - + util::Vector3D(float(velocity.x * temp), float(velocity.y * temp), float(velocity.z * temp)); - - // ... transform to the location on the screen, and render. - XYCoordinates p3 = mViewpoint->FlatProject(p1); - XYCoordinates p4 = mViewpoint->FlatProject(p2); - - p3 = mScreen->TransformScreenToPixelCoordinates(p3); - p4 = mScreen->TransformScreenToPixelCoordinates(p4); - - RenderLine(p3, p4, mVisSettings, pixelSet); - } - - return pixelSet; - } - } -} diff --git a/Code/vis/GlyphDrawer.h b/Code/vis/GlyphDrawer.h deleted file mode 100644 index 45e8cd137..000000000 --- a/Code/vis/GlyphDrawer.h +++ /dev/null @@ -1,88 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_GLYPHDRAWER_H -#define HEMELB_VIS_GLYPHDRAWER_H - -#include "geometry/LatticeData.h" -#include "lb/MacroscopicPropertyCache.h" - -#include "vis/BasicPixel.h" -#include "vis/PixelSet.h" -#include "vis/PixelSetStore.h" -#include "vis/Screen.h" -#include "vis/DomainStats.h" -#include "vis/Viewpoint.h" -#include "vis/VisSettings.h" - -#include -#include - -namespace hemelb -{ - namespace vis - { - // Class for drawing glyphs. - class GlyphDrawer : public PixelSetStore > - { - public: - // Constructor and destructor - GlyphDrawer(geometry::LatticeData* iLatDat, - Screen* iScreen, - DomainStats* iDomainStats, - Viewpoint* iViewpoint, - VisSettings* iVisSettings); - ~GlyphDrawer(); - - // Function to perform the rendering. - PixelSet* Render(const lb::MacroscopicPropertyCache& propertyCache); - - private: - // A struct to represent a single glyph. - struct Glyph - { - /** - * The 3D coordinates of the glyph. - */ - float x, y, z; - /** - * The local contiguous site id near there. - */ - site_t siteId; - }; - - void RenderLine(const XYCoordinates& endPoint1, - const XYCoordinates& endPoint2, - const VisSettings* visSettings, - PixelSet*); - - template - void RenderLineHelper(int x, - int y, - int incE, - int incNE, - int limit, - const VisSettings* visSettings, - PixelSet*); - - geometry::LatticeData* mLatDat; - - Screen* mScreen; - DomainStats* mDomainStats; - Viewpoint* mViewpoint; - VisSettings* mVisSettings; - - std::vector mGlyphs; - - }; - - } -} - -#endif //HEMELB_VIS_GLYPHDRAWER_H diff --git a/Code/vis/PixelSet.h b/Code/vis/PixelSet.h deleted file mode 100644 index 80b6ed83d..000000000 --- a/Code/vis/PixelSet.h +++ /dev/null @@ -1,150 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_PIXELSET_H -#define HEMELB_VIS_PIXELSET_H - -#include -#include - -#include "log/Logger.h" -#include "net/mpi.h" -#include "net/net.h" -#include "vis/BasicPixel.h" -#include "vis/VisSettings.h" - -namespace hemelb -{ - namespace vis - { - /** - * Base pixel set implementation, including the functionality that allows storing of pixels - * in a vector for speedy MPI usage, but also storing a map of pixels -> Pixel* for efficient - * lookup in O(N) time. - */ - template - class PixelSet - { - public: - - PixelSet() - { - inUse = false; - count = 0; - } - - ~PixelSet() - { - } - - void Combine(const PixelSet &other) - { - typename std::vector::const_iterator otherPixels = other.pixels.begin(); - - while (otherPixels != other.pixels.end()) - { - AddPixel(*otherPixels); - otherPixels++; - } - } - - void AddPixel(const PixelType& newPixel) - { - BasicPixel location = BasicPixel(newPixel.GetI(), newPixel.GetJ()); - - if (pixelLookup.count(location) > 0) - { - pixels[pixelLookup[location]].Combine(newPixel); - } - else - { - pixels.push_back(PixelType(newPixel)); - pixelLookup.insert(std::pair(location, - (unsigned int) (pixels.size() - - 1))); - } - } - - bool IsInUse() const - { - return inUse; - } - - void SetInUse() - { - inUse = true; - } - - void Release() - { - inUse = false; - } - - void SendQuantity(net::Net* net, proc_t destination) - { - count = (int) pixels.size(); - log::Logger::Log("Sending pixel count of %i", count); - net->RequestSendR(count, destination); - } - - void ReceiveQuantity(net::Net* net, proc_t source) - { - net->RequestReceiveR(count,source); - } - - void SendPixels(net::Net* net, proc_t destination) - { - if (pixels.size() > 0) - { - log::Logger::Log("Sending %i pixels to proc %i", - (int) pixels.size(), - (int) destination); - net->RequestSendV(pixels, destination); - } - } - - void ReceivePixels(net::Net* net, proc_t source) - { - if (count > 0) - { - // First make sure the vector will be large enough to hold the incoming pixels. - log::Logger::Log("Receiving %i pixels from proc %i", - count, - (int) source); - pixels.resize(count); - net->RequestReceiveV(pixels, source); - } - } - - size_t GetPixelCount() const - { - return pixels.size(); - } - - const std::vector& GetPixels() const - { - return pixels; - } - - void Clear() - { - pixelLookup.clear(); - pixels.clear(); - } - - private: - std::map pixelLookup; - std::vector pixels; - int count; - bool inUse; - }; - } -} - -#endif // HEMELB_VIS_PIXELSET_H diff --git a/Code/vis/PixelSetStore.h b/Code/vis/PixelSetStore.h deleted file mode 100644 index 469621816..000000000 --- a/Code/vis/PixelSetStore.h +++ /dev/null @@ -1,74 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_PIXELSETSTORE_H -#define HEMELB_VIS_PIXELSETSTORE_H - -#include -#include "log/Logger.h" - -namespace hemelb -{ - namespace vis - { - template - class PixelSetStore - { - public: - PixelSetType* GetUnusedPixelSet() - { - if (!store.empty()) - { - // Take the first pixel set (and put it on the back of the queue for reuse later). - PixelSetType* candidate = store.front(); - - store.pop(); - store.push(candidate); - - // If we can use this one to write, return it. - if (!candidate->IsInUse()) - { - candidate->SetInUse(); - candidate->Clear(); - return candidate; - } - } - - // We've not got any inUse pixel sets, so create a new one, store it and return it. - PixelSetType* ret = new PixelSetType(); - - store.push(ret); - - ret->SetInUse(); - return ret; - } - - ~PixelSetStore() - { - while (!store.empty()) - { - if (store.front()->IsInUse()) - { - log::Logger::Log("This could be a problem: we've just " - "cleared out a pixel set which appears to still be in use. If pixel sets are being " - "managed properly there shouldn't be more than a few of these messages per core."); - } - - delete store.front(); - store.pop(); - } - } - - private: - std::queue store; - }; - } -} - -#endif // HEMELB_VIS_PIXELSETSTORE_H diff --git a/Code/vis/Rendering.cc b/Code/vis/Rendering.cc deleted file mode 100644 index f28fcd92d..000000000 --- a/Code/vis/Rendering.cc +++ /dev/null @@ -1,140 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "vis/Rendering.h" - -namespace hemelb -{ - namespace vis - { - Rendering::Rendering(PixelSet* glyph, - PixelSet* ray, - PixelSet* streak) : - glyphResult(glyph), rayResult(ray), streakResult(streak) - { - - } - - void Rendering::ReleaseAll() - { - if (glyphResult != NULL) - { - glyphResult-> Release(); - } - - if (rayResult != NULL) - { - rayResult->Release(); - } - - if (streakResult != NULL) - { - streakResult-> Release(); - } - } - - void Rendering::ReceivePixelCounts(net::Net* inNet, proc_t source) - { - if (glyphResult != NULL) - { - glyphResult->ReceiveQuantity(inNet, source); - } - if (rayResult != NULL) - { - rayResult->ReceiveQuantity(inNet, source); - } - if (streakResult != NULL) - { - streakResult->ReceiveQuantity(inNet, source); - } - } - - void Rendering::ReceivePixelData(net::Net* inNet, proc_t source) - { - if (glyphResult != NULL) - { - glyphResult->ReceivePixels(inNet, source); - } - if (rayResult != NULL) - { - rayResult->ReceivePixels(inNet, source); - } - if (streakResult != NULL) - { - streakResult->ReceivePixels(inNet, source); - } - } - - void Rendering::SendPixelCounts(net::Net* inNet, proc_t destination) - { - if (glyphResult != NULL) - { - glyphResult->SendQuantity(inNet, destination); - } - if (rayResult != NULL) - { - rayResult->SendQuantity(inNet, destination); - } - if (streakResult != NULL) - { - streakResult->SendQuantity(inNet, destination); - } - } - - void Rendering::SendPixelData(net::Net* inNet, proc_t destination) - { - if (glyphResult != NULL) - { - glyphResult->SendPixels(inNet, destination); - } - if (rayResult != NULL) - { - rayResult->SendPixels(inNet, destination); - } - if (streakResult != NULL) - { - streakResult->SendPixels(inNet, destination); - } - } - - void Rendering::Combine(const Rendering& other) - { - if (glyphResult != NULL) - { - glyphResult->Combine(*other.glyphResult); - } - if (rayResult != NULL) - { - rayResult->Combine(*other.rayResult); - } - if (streakResult != NULL) - { - streakResult->Combine(*other.streakResult); - } - } - - void Rendering::PopulateResultSet(PixelSet* resultSet) - { - if (glyphResult != NULL) - { - AddPixelsToResultSet(resultSet, glyphResult->GetPixels()); - } - - if (rayResult != NULL) - { - AddPixelsToResultSet(resultSet, rayResult->GetPixels()); - } - - if (streakResult != NULL) - { - AddPixelsToResultSet(resultSet, streakResult->GetPixels()); - } - } - } -} diff --git a/Code/vis/Rendering.h b/Code/vis/Rendering.h deleted file mode 100644 index 442534be0..000000000 --- a/Code/vis/Rendering.h +++ /dev/null @@ -1,64 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RENDERING_H -#define HEMELB_VIS_RENDERING_H - -#include "vis/BasicPixel.h" -#include "vis/PixelSet.h" -#include "vis/rayTracer/RayDataNormal.h" -#include "vis/ResultPixel.h" -#include "vis/streaklineDrawer/StreakPixel.h" - -namespace hemelb -{ - namespace vis - { - /** - * Rendering: A class that acts as the interface between the visualisation controller and the - * drawn renderings from each component drawer. - */ - class Rendering - { - public: - Rendering(PixelSet* glyph, - PixelSet* ray, - PixelSet* streak); - void ReleaseAll(); - - void ReceivePixelCounts(net::Net* inNet, proc_t source); - - void ReceivePixelData(net::Net* inNet, proc_t source); - - void SendPixelCounts(net::Net* inNet, proc_t destination); - - void SendPixelData(net::Net* inNet, proc_t destination); - void Combine(const Rendering& other); - void PopulateResultSet(PixelSet* resultSet); - - private: - template - void AddPixelsToResultSet(PixelSet* resultSet, - const std::vector& inPixels) - { - for (typename std::vector::const_iterator it = inPixels.begin(); it - != inPixels.end(); it++) - { - resultSet ->AddPixel(ResultPixel(&*it)); - } - } - - PixelSet* glyphResult; - PixelSet* rayResult; - PixelSet* streakResult; - }; - } -} - -#endif /* HEMELB_VIS_RENDERING_H */ diff --git a/Code/vis/ResultPixel.cc b/Code/vis/ResultPixel.cc deleted file mode 100644 index 1520a69b9..000000000 --- a/Code/vis/ResultPixel.cc +++ /dev/null @@ -1,237 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "vis/ResultPixel.h" - -namespace hemelb -{ - namespace vis - { - ResultPixel::ResultPixel(const BasicPixel* glyph) : - BasicPixel(glyph->GetI(), glyph->GetJ()), hasGlyph(true), normalRayPixel(NULL), streakPixel(NULL) - { - - } - - ResultPixel::ResultPixel(const raytracer::RayDataNormal* ray) : - BasicPixel(ray->GetI(), ray->GetJ()), hasGlyph(false), normalRayPixel(ray), streakPixel(NULL) - { - - } - - ResultPixel::ResultPixel(const streaklinedrawer::StreakPixel* streak) : - BasicPixel(streak->GetI(), streak->GetJ()), hasGlyph(false), normalRayPixel(NULL), streakPixel(streak) - { - - } - - const raytracer::RayDataNormal* ResultPixel::GetRayPixel() const - { - return normalRayPixel; - } - - void ResultPixel::Combine(const ResultPixel& other) - { - if (other.hasGlyph) - { - hasGlyph = true; - } - - if (other.normalRayPixel != NULL) - { - normalRayPixel = other.normalRayPixel; - } - - if (other.streakPixel != NULL) - { - streakPixel = other.streakPixel; - } - } - - void ResultPixel::WritePixel(unsigned* pixel_index, - unsigned char rgb_data[12], - const DomainStats& iDomainStats, - const VisSettings& visSettings) const - { - const int bits_per_char = sizeof(char) * 8; - *pixel_index = (i << (2 * bits_per_char)) + j; - - if (normalRayPixel != NULL) - { - // store velocity volume rendering colour - normalRayPixel->GetVelocityColour(rgb_data, visSettings, iDomainStats); - - float stress = normalRayPixel->GetNearestStress(); - - if (visSettings.mStressType != lb::ShearStress) - { - normalRayPixel->GetStressColour(&rgb_data[3], visSettings, iDomainStats); - } - else if (stress < (float) NO_VALUE) - { - float stress_col[3]; - PickColour(stress, stress_col); - - // store wall shear stress colour - MakePixelColour(int(255.0F * stress_col[0]), - int(255.0F * stress_col[1]), - int(255.0F * stress_col[2]), - &rgb_data[3]); - } - else - { - rgb_data[3] = rgb_data[4] = rgb_data[5] = 0; - } - } - else - { - for (int ii = 0; ii < 6; ++ii) - { - rgb_data[ii] = 255; - } - } - - float density = normalRayPixel == NULL - ? 0.0F - : normalRayPixel->GetNearestDensity(); - float stress = normalRayPixel == NULL - ? 0.0F - : normalRayPixel->GetNearestStress(); - - if (visSettings.mStressType != lb::ShearStress && visSettings.mode == VisSettings::ISOSURFACES) - { - float density_col[3], stress_col[3]; - PickColour(density, density_col); - PickColour(stress, stress_col); - - // store wall pressure colour - MakePixelColour(int(255.0F * density_col[0]), - int(255.0F * density_col[1]), - int(255.0F * density_col[2]), - &rgb_data[6]); - - // store von Mises stress colour - MakePixelColour(int(255.0F * stress_col[0]), - int(255.0F * stress_col[1]), - int(255.0F * stress_col[2]), - &rgb_data[9]); - - } - else if (visSettings.mStressType != lb::ShearStress && visSettings.mode == VisSettings::ISOSURFACESANDGLYPHS) - { - float density_col[3], stress_col[3]; - PickColour(density, density_col); - PickColour(stress, stress_col); - - if (normalRayPixel != NULL) - { - if (!hasGlyph) - { - density_col[0] += 1.0F; - density_col[1] += 1.0F; - density_col[2] += 1.0F; - - stress_col[0] += 1.0F; - stress_col[1] += 1.0F; - stress_col[2] += 1.0F; - } - - // store wall pressure (+glyph) colour - MakePixelColour(int(127.5F * density_col[0]), - int(127.5F * density_col[1]), - int(127.5F * density_col[2]), - &rgb_data[6]); - - // store von Mises stress (+glyph) colour - MakePixelColour(int(127.5F * stress_col[0]), - int(127.5F * stress_col[1]), - int(127.5F * stress_col[2]), - &rgb_data[9]); - } - else - { - for (int ii = 6; ii < 12; ++ii) - { - rgb_data[ii] = 0; - } - } - - } - else if (streakPixel != NULL) - { - float scaled_vel = (float) (streakPixel->GetParticleVelocity() * iDomainStats.velocity_threshold_max_inv); - float particle_col[3]; - PickColour(scaled_vel, particle_col); - - // store particle colour - MakePixelColour(int(255.0F * particle_col[0]), - int(255.0F * particle_col[1]), - int(255.0F * particle_col[2]), - &rgb_data[6]); - - for (int ii = 9; ii < 12; ++ii) - { - rgb_data[ii] = rgb_data[ii - 3]; - } - } - else - { - // store pressure colour - rgb_data[6] = rgb_data[7] = rgb_data[8] - = (unsigned char) util::NumericalFunctions::enforceBounds(int(127.5F * density), 0, 127); - - // store shear stress or von Mises stress - if (stress < ((float) NO_VALUE)) - { - rgb_data[9] = rgb_data[10] = rgb_data[11] - = (unsigned char) util::NumericalFunctions::enforceBounds(int(127.5F * stress), 0, 127); - } - else - { - rgb_data[9] = rgb_data[10] = rgb_data[11] = 0; - } - } - } - - void ResultPixel::PickColour(float value, float colour[3]) - { - colour[0] = util::NumericalFunctions::enforceBounds(4.F * value - 2.F, 0.F, 1.F); - colour[1] = util::NumericalFunctions::enforceBounds(2.F - 4.F * (float) fabs(value - 0.5F), 0.F, 1.F); - colour[2] = util::NumericalFunctions::enforceBounds(2.F - 4.F * value, 0.F, 1.F); - } - - void ResultPixel::MakePixelColour(int rawRed, int rawGreen, int rawBlue, unsigned char* dest) - { - dest[0] = (unsigned char) util::NumericalFunctions::enforceBounds(rawRed, 0, 255); - dest[1] = (unsigned char) util::NumericalFunctions::enforceBounds(rawGreen, 0, 255); - dest[2] = (unsigned char) util::NumericalFunctions::enforceBounds(rawBlue, 0, 255); - } - - void ResultPixel::LogDebuggingInformation() const - { - log::Logger::Log("Pixel at (%i,%i) with (ray,streak,glyph)=(%i,%i,%i)", - GetI(), - GetJ(), - normalRayPixel != NULL, - streakPixel != NULL, - hasGlyph); - - if (normalRayPixel != NULL) - { - normalRayPixel->LogDebuggingInformation(); - } - - if (streakPixel != NULL) - { - streakPixel->LogDebuggingInformation(); - } - } - - } -} diff --git a/Code/vis/ResultPixel.h b/Code/vis/ResultPixel.h deleted file mode 100644 index 43550a5ac..000000000 --- a/Code/vis/ResultPixel.h +++ /dev/null @@ -1,62 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RESULTPIXEL_H -#define HEMELB_VIS_RESULTPIXEL_H - -#include - -#include "util/utilityFunctions.h" -#include "vis/BasicPixel.h" -#include "vis/rayTracer/RayDataNormal.h" -#include "vis/streaklineDrawer/StreakPixel.h" -#include "vis/VisSettings.h" -#include "vis/DomainStats.h" - -namespace hemelb -{ - namespace vis - { - class ResultPixel : public BasicPixel - { - public: - ResultPixel(const BasicPixel* glyph); - - ResultPixel(const raytracer::RayDataNormal* ray); - - ResultPixel(const streaklinedrawer::StreakPixel* streak); - - const raytracer::RayDataNormal* GetRayPixel() const; - - void Combine(const ResultPixel& other); - - void WritePixel(unsigned* pixel_index, - unsigned char rgb_data[12], - const DomainStats& iDomainStats, - const VisSettings& visSettings) const; - - /* - * Debugging function to output details about the pixel to the logger. - */ - void LogDebuggingInformation() const; - - private: - - static void PickColour(float value, float colour[3]); - - static void MakePixelColour(int rawRed, int rawGreen, int rawBlue, unsigned char* dest); - - bool hasGlyph; - const raytracer::RayDataNormal* normalRayPixel; - const streaklinedrawer::StreakPixel* streakPixel; - }; - } -} - -#endif /* HEMELB_VIS_RESULTPIXEL_H */ diff --git a/Code/vis/Screen.cc b/Code/vis/Screen.cc deleted file mode 100644 index 44126d846..000000000 --- a/Code/vis/Screen.cc +++ /dev/null @@ -1,106 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "net/IOCommunicator.h" -#include "util/utilityFunctions.h" -#include "vis/Screen.h" - -namespace hemelb -{ - namespace vis - { - - Screen::Screen() - { - } - - Screen::~Screen() - { - } - - void Screen::Set(float maxX, - float maxY, - int pixelsX, - int pixelsY, - float rad, - const Viewpoint* viewpoint) - { - MaxXValue = maxX; - MaxYValue = maxY; - - mPixelUnitVectorProjectionX - = viewpoint->RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(MaxXValue, - 0.0F, - 0.0F)); - mPixelUnitVectorProjectionY - = viewpoint-> RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(0.0F, - MaxYValue, - 0.0F)); - - Resize(pixelsX, pixelsY); - - mPixelsPerUnitX = (float) GetPixelsX() / (2.F * MaxXValue); - mPixelsPerUnitY = (float) GetPixelsY() / (2.F * MaxYValue); - - util::Vector3D - lCameraToLocalCentreVector = - viewpoint->RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(0.F, - 0.F, - -viewpoint->GetDistanceFromCameraToScreen())); - - util::Vector3D lMiddleCentreToMiddleRightOfScreen = - viewpoint->RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(MaxXValue, - 0.0F, - 0.0F)); - - util::Vector3D lLowerCentreToTopCentreOfScreen = - viewpoint->RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(0.0F, - MaxYValue, - 0.0F)); - - mCameraToBottomLeftOfScreen = (lCameraToLocalCentreVector - - lMiddleCentreToMiddleRightOfScreen) - lLowerCentreToTopCentreOfScreen; - - mPixelUnitVectorProjectionX = lMiddleCentreToMiddleRightOfScreen * (2.F - / (float) GetPixelsX()); - - mPixelUnitVectorProjectionY = lLowerCentreToTopCentreOfScreen * (2.F / (float) GetPixelsY()); - } - - void Screen::Resize(unsigned int newPixelsX, unsigned int newPixelsY) - { - if (newPixelsX * newPixelsY <= COLOURED_PIXELS_MAX) - { - xPixels = newPixelsX; - yPixels = newPixelsY; - } - } - - const util::Vector3D& Screen::GetCameraToBottomLeftOfScreenVector() const - { - return mCameraToBottomLeftOfScreen; - } - const util::Vector3D& Screen::GetPixelUnitVectorProjectionX() const - { - return mPixelUnitVectorProjectionX; - } - const util::Vector3D& Screen::GetPixelUnitVectorProjectionY() const - { - return mPixelUnitVectorProjectionY; - } - int Screen::GetPixelsX() const - { - return xPixels; - } - int Screen::GetPixelsY() const - { - return yPixels; - } - } -} diff --git a/Code/vis/Screen.h b/Code/vis/Screen.h deleted file mode 100644 index 2440e7918..000000000 --- a/Code/vis/Screen.h +++ /dev/null @@ -1,81 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_SCREEN_H -#define HEMELB_VIS_SCREEN_H - -#include "io/writers/Writer.h" -#include "util/Vector3D.h" -#include "vis/Viewpoint.h" -#include "vis/VisSettings.h" - -namespace hemelb -{ - namespace vis - { - class Screen - { - public: - static const unsigned int COLOURED_PIXELS_MAX = 2048 * 2048; - - Screen(); - ~Screen(); - - void Set(float maxX, - float maxY, - int pixelsX, - int pixelsY, - float rad, - const Viewpoint* viewpoint); - - void Resize(unsigned int pixelsX, unsigned int pixelsY); - - /** - * Does a transform from input array into output array. This function - * will still work if the two arrays point to the same location in - * memory. It only operates on the first two elements of input. - * - * @param input - * @param output - */ - template - XYCoordinates TransformScreenToPixelCoordinates(const XYCoordinates& iXYIn) const - { - return XYCoordinates (static_cast (mPixelsPerUnitX * (iXYIn.x + MaxXValue)), - static_cast (mPixelsPerUnitY * (iXYIn.y + MaxYValue))); - } - - const util::Vector3D& GetCameraToBottomLeftOfScreenVector() const; - const util::Vector3D& GetPixelUnitVectorProjectionX() const; - const util::Vector3D& GetPixelUnitVectorProjectionY() const; - int GetPixelsX() const; - int GetPixelsY() const; - - private: - int xPixels, yPixels; - - //The number of pixels per unit of X or Y in screen coordinates - float mPixelsPerUnitX; - float mPixelsPerUnitY; - - //The extent of the screen in screen coordintes - //(from -MaxXValue to MaxXValue) - float MaxXValue; - float MaxYValue; - - util::Vector3D mCameraToBottomLeftOfScreen; - - // Projection of unit vectors along screen axes into normal space. - util::Vector3D mPixelUnitVectorProjectionX; - util::Vector3D mPixelUnitVectorProjectionY; - }; - } -} - -#endif /* HEMELB_VIS_SCREEN_H */ diff --git a/Code/vis/Viewpoint.cc b/Code/vis/Viewpoint.cc deleted file mode 100644 index 5fa6fdd25..000000000 --- a/Code/vis/Viewpoint.cc +++ /dev/null @@ -1,158 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include - -#include "log/Logger.h" -#include "util/Vector3D.h" -#include "vis/Viewpoint.h" -#include "vis/XYCoordinates.h" - -namespace hemelb -{ - namespace vis - { - Viewpoint::Viewpoint() : - mViewpointLocationInWorldCoordinates(0.0F) - { - } - - util::Vector3D Viewpoint::RotateCameraCoordinatesToWorldCoordinates(const util::Vector3D< - float>& iVector) const - { - // A rotation of iThetaX clockwise looking up the x-axis (increasing) - // Followed by a rotation of iThetaY anticlockwise looking up the y-axis. - return Rotate(mSinLatitude, mCosLatitude, mSinLongitude, mCosLongitude, iVector); - } - - util::Vector3D Viewpoint::RotateWorldToCameraCoordinates(const util::Vector3D& iVector) const - { - // The reverse of the above - return UnRotate(mSinLatitude, mCosLatitude, mSinLongitude, mCosLongitude, iVector); - } - - util::Vector3D Viewpoint::Rotate(float iSinThetaX, - float iCosThetaX, - float iSinThetaY, - float iCosThetaY, - const util::Vector3D& iVector) - { - // A rotation of iThetaX clockwise looking down the x-axis - // Followed by a rotation of iThetaY anticlockwise looking down the y-axis. - // In matrices: - // (cos(iThetaY) 0 sin(iThetaY)) (1 0 0 ) - // Out = (0 1 0 ) (0 cos(-iThetaX) -sin(-iThetaX)) In - // (-sin(iThetaY) 0 cos(iThetaY)) (0 sin(-iThetaX) cos(-iThetaX) ) - // - // (Xcos(iThetaY) + Zsin(iThetaY)cos(iThetaX) - Ysin(iThetaY)sin(iThetaX)) - // Out = (Ycos(iThetaX) + Zsin(iThetaX) ) - // (Zcos(iThetaX)cos(iThetaY) - Ysin(iThetaX)cos(iThetaY) - Xsin(iThetaY)) - - const float lTemp = iVector.z * iCosThetaX - iVector.y * iSinThetaX; - - return util::Vector3D(lTemp * iSinThetaY + iVector.x * iCosThetaY, - iVector.z * iSinThetaX + iVector.y * iCosThetaX, - lTemp * iCosThetaY - iVector.x * iSinThetaY); - } - - util::Vector3D Viewpoint::UnRotate(float iSinThetaX, - float iCosThetaX, - float iSinThetaY, - float iCosThetaY, - const util::Vector3D& iVector) - { - // A rotation of iThetaY aniclockwise looking down the y-axis - // Followed by a rotation of iThetaX clockwise looking down the x-axis. - // In matrices: - // (1 0 0 )(cos(-iThetaY) 0 sin(-iThetaY)) - // Out = (0 cos(iThetaX) -sin(iThetaX))(0 1 0 ) In - // (0 sin(iThetaX) cos(iThetaX) )(-sin(-iThetaY) 0 cos(-iThetaY)) - // - // This is the Rotation matrix inversed / transposted - // ie (AB)^-1 = (AB)^t = B^t A^T - - const float lTemp = iVector.x * iSinThetaY + iVector.z * iCosThetaY; - - return util::Vector3D(iVector.x * iCosThetaY - iVector.z * iSinThetaY, - -lTemp * iSinThetaX + iVector.y * iCosThetaX, - lTemp * iCosThetaX + iVector.y * iSinThetaX); - } - - util::Vector3D Viewpoint::Project(const util::Vector3D& iWorldLocation) const - { - util::Vector3D lLocationCamCoordinates = - GetLocationInCameraCoordinates(iWorldLocation); - - //Carry out a perspective projection on an infinite spanning screen - //between the camera and the subject. - //Reverse the sign such that depth is positive (I believe). - return util::Vector3D(mDistanceFromCameraToScreen / (-lLocationCamCoordinates.z) - * lLocationCamCoordinates.x, - mDistanceFromCameraToScreen / (-lLocationCamCoordinates.z) - * lLocationCamCoordinates.y, - -lLocationCamCoordinates.z); - } - - XYCoordinates Viewpoint::FlatProject(const util::Vector3D& iWorldLocation) const - { - util::Vector3D lLocationCamCoordinates = - GetLocationInCameraCoordinates(iWorldLocation); - - return XYCoordinates (mDistanceFromCameraToScreen / (-lLocationCamCoordinates.z) - * lLocationCamCoordinates.x, - mDistanceFromCameraToScreen / (-lLocationCamCoordinates.z) - * lLocationCamCoordinates.y); - } - - void Viewpoint::SetViewpointPosition(float iLongitude, - float iLatitude, - const util::Vector3D& iLocalCentre, - float iRadius, - float iDistanceFromCameraToScreen) - { - - mSinLongitude = sinf(iLongitude); - mCosLongitude = cosf(iLongitude); - - mSinLatitude = sinf(iLatitude); - mCosLatitude = cosf(iLatitude); - - //The camera is located at 0,0,radius from the world centre in camera co-ordinates - mViewpointLocationInWorldCoordinates - = RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(0., 0., iRadius)); - - //Translate the camera location to allow it to point at - //a local centre rather than the world centre - mViewpointLocationInWorldCoordinates += iLocalCentre; - - mDistanceFromCameraToScreen = iDistanceFromCameraToScreen; - } - - const util::Vector3D& Viewpoint::GetViewpointLocation() const - { - return mViewpointLocationInWorldCoordinates; - } - - float Viewpoint::GetDistanceFromCameraToScreen() const - { - return mDistanceFromCameraToScreen; - } - - util::Vector3D Viewpoint::GetLocationInCameraCoordinates(const util::Vector3D& iWorldLocation) const - { - //Calculate the location of the point relative to the viewpoint - util::Vector3D lLocationRelativeToViewPoint = iWorldLocation - - mViewpointLocationInWorldCoordinates; - - //Rotate the location vector in the opposite manner to that of the camera - return RotateWorldToCameraCoordinates(lLocationRelativeToViewPoint); - } - - } -} diff --git a/Code/vis/Viewpoint.h b/Code/vis/Viewpoint.h deleted file mode 100644 index 63a1ed2c9..000000000 --- a/Code/vis/Viewpoint.h +++ /dev/null @@ -1,112 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_VIEWPOINT_H -#define HEMELB_VIS_VIEWPOINT_H - -#include "util/Vector3D.h" -#include "vis/XYCoordinates.h" - -namespace hemelb -{ - namespace vis - { - /** - * Holds the position of the viewpoint or camera location and performs - * projection of points in world coordinates into 3D screen co-ordinates - */ - class Viewpoint - { - //See http://en.wikipedia.org/wiki/3D_projection#Perspective_projection - public: - Viewpoint(); - - /** - * Changes a location in camera coordinates into world coordinates - * Note that both co-ordinate systems share the same (0,0,0) point - * and the camera is at (0,0,radius) in camera coordinates - * - */ - util::Vector3D RotateCameraCoordinatesToWorldCoordinates(const util::Vector3D& iVector) const; - - /** - * Does the reverse of the above - * - */ - util::Vector3D RotateWorldToCameraCoordinates(const util::Vector3D& iVector) const; - - /** - * Projects a location in world coordinates onto the infinite screen, - * by translating and rotating such as to give coordinates relative - * to the camera, and then applying a perspective projection - */ - util::Vector3D Project(const util::Vector3D& p1) const; - - /** - * Same as project but doesn't return a z component - * - */ - XYCoordinates FlatProject(const util::Vector3D& iWorldLocation) const; - - /** - * Sets the position of the Camera or Viewpoint - * - * In world coordinates, the camera is located at radius from - * the local centre, and pointing at an angle indicated by the - * latitude and longitude. - * - * @param iLongitude - longitude in radians. - * @param iLatitude - latitude in radians. - * @param iLocalCentre - where the camera should point in world-coordinates - * @param iRadius - the distance of the camera from this local centre - * @param iDistanceFromCameraToScreen - the distance of the infinite screen - * from the viewer. Allows for zoom - */ - void SetViewpointPosition(float iLongitude, - float iLatitude, - const util::Vector3D& iLocalCentre, - float iRadius, - float iDistanceFromCameraToScreen); - - const util::Vector3D& GetViewpointLocation() const; - - float GetDistanceFromCameraToScreen() const; - - private: - util::Vector3D GetLocationInCameraCoordinates(const util::Vector3D& iWorldLocation) const; - - //Performs a vector rotation using stored - //Sin and Cosine Values - static util::Vector3D Rotate(float iSinThetaX, - float iCosThetaX, - float iSinThetaY, - float iCosThetaY, - const util::Vector3D& iVector); - - //Reverses a vector rotation of the above - static util::Vector3D UnRotate(float iSinThetaX, - float iCosThetaX, - float iSinThetaY, - float iCosThetaY, - const util::Vector3D& iVector); - - float mSinLongitude; - float mCosLongitude; - float mSinLatitude; - float mCosLatitude; - - //Stores the viewpoint Location in world co-ordinate - util::Vector3D mViewpointLocationInWorldCoordinates; - - float mDistanceFromCameraToScreen; - }; - } -} - -#endif /* HEMELB_VIS_VIEWPOINT_H */ diff --git a/Code/vis/VisSettings.h b/Code/vis/VisSettings.h deleted file mode 100644 index a8d706271..000000000 --- a/Code/vis/VisSettings.h +++ /dev/null @@ -1,51 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_VISSETTINGS_H -#define HEMELB_VIS_VISSETTINGS_H - -#include "lb/LbmParameters.h" - -namespace hemelb -{ - namespace vis - { - struct VisSettings - { - enum Mode - { - // 0 - Only display the isosurfaces (wall pressure and stress) - ISOSURFACES = 0, - // 1 - Isosurface and glyphs - ISOSURFACESANDGLYPHS = 1, - // 2 - Wall pattern streak lines - WALLANDSTREAKLINES = 2 - }; - - // better public member vars than globals! - Mode mode; - - float ctr_x, ctr_y, ctr_z; - float streaklines_per_simulation, streakline_length; - double mouse_pressure, mouse_stress; - float brightness; - float glyphLength; - - //Maximum distance - used in the enhanced ray tracer to handle - //dept cuing - float maximumDrawDistance; - - lb::StressTypes mStressType; - - int mouse_x, mouse_y; - }; - } -} - -#endif /* HEMELB_VIS_VISSETTINGS_H */ diff --git a/Code/vis/XYCoordinates.h b/Code/vis/XYCoordinates.h deleted file mode 100644 index 2499b9055..000000000 --- a/Code/vis/XYCoordinates.h +++ /dev/null @@ -1,140 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_XYCOORDINATES_H -#define HEMELB_VIS_XYCOORDINATES_H - -#include "constants.h" -#include "util/utilityFunctions.h" - -namespace hemelb -{ - namespace vis - { - template - class XYCoordinates - { - public: - T x, y; - - XYCoordinates() - { - } - ; - - XYCoordinates(T iX, T iY) : - x(iX), y(iY) - { - } - - XYCoordinates(T iN) : - x(iN), y(iN) - { - } - - //Copy constructor - can be used to perform type converstion - template - XYCoordinates(const XYCoordinates & iOldXYCoordinates) - { - x = static_cast(iOldXYCoordinates.x); - y = static_cast(iOldXYCoordinates.y); - } - - //Equality - bool operator==(const XYCoordinates right) - { - if (x != right.x) - { - return false; - } - if (y != right.y) - { - return false; - } - return true; - } - - //Vector addition - XYCoordinates operator+(const XYCoordinates right) const - { - return XYCoordinates(x + right.x, y + right.y); - } - - //Vector addition - XYCoordinates& operator+=(const XYCoordinates right) - { - x += right.x; - y += right.y; - - return *this; - } - - //Vector subtraction - XYCoordinates operator-(const XYCoordinates right) const - { - return XYCoordinates(x - right.x, y - right.y); - } - - //Scalar multiplication - template - XYCoordinates operator*(const MultiplierT multiplier) const - { - return XYCoordinates(x * multiplier, y * multiplier); - } - - //Updates the XYCoordinates with the smallest of each - //of the x and y co-ordinatess independently of both XYCoordinates - void UpdatePointwiseMin(const XYCoordinates& iCompareLocation) - { - x = util::NumericalFunctions::min(x, iCompareLocation.x); - - y = util::NumericalFunctions::min(y, iCompareLocation.y); - } - - //Updates the XYCoordinates with the largest of each - //of the x and y co-ordinates independently of both XYCoordinates - void UpdatePointwiseMax(const XYCoordinates& iCompareLocation) - { - x = util::NumericalFunctions::max(x, iCompareLocation.x); - - y = util::NumericalFunctions::max(y, iCompareLocation.y); - } - - static XYCoordinates MaxLimit() - { - return XYCoordinates(std::numeric_limits::max()); - } - - static XYCoordinates MinLimit() - { - return XYCoordinates(std::numeric_limits::min()); - } - }; - - template - XYCoordinates operator+(const XYCoordinates left, const XYCoordinates right) - { - return left + right; - } - - template - XYCoordinates operator-(const XYCoordinates left, const XYCoordinates right) - { - return left - right; - } - - template - XYCoordinates operator*(const TLeft left, const XYCoordinates right) - { - return right * left; - } - } -} - -#endif // HEMELB_VIS_XYCOORDINATES_H diff --git a/Code/vis/rayTracer/Cluster.h b/Code/vis/rayTracer/Cluster.h deleted file mode 100644 index ce0c81c59..000000000 --- a/Code/vis/rayTracer/Cluster.h +++ /dev/null @@ -1,171 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTER_H -#define HEMELB_VIS_RAYTRACER_CLUSTER_H - -#include - -#include "util/Vector3D.h" - -#include "vis/rayTracer/SiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - //The cluster structure stores data relating to the clusters - //used by the RayTracer, in an optimal format - //Cluster are produced by the ClusterSharedFactory - //Caution: the data within the flow field is altered by means - //of pointers obtained from the GetClusterSharedVoxelDataPointer - //method - template - class Cluster - { - public: - Cluster(unsigned short xBlockCount, - unsigned short yBlockCount, - unsigned short zBlockCount, - const util::Vector3D& minimalSite, - const util::Vector3D& maximalSite, - const util::Vector3D& minimalSiteOnMinimalBlock, - const util::Vector3D& minimalBlock) : - blocksX(xBlockCount), blocksY(yBlockCount), blocksZ(zBlockCount), minSite(minimalSite), maxSite(maximalSite), leastSiteOnLeastBlockInImage(minimalSiteOnMinimalBlock), minBlock(minimalBlock) - { - } - - unsigned int GetBlockIdFrom3DBlockLocation(const util::Vector3D& iLocation) const - { - return iLocation.x * blocksY * blocksZ + iLocation.y * blocksZ + iLocation.z; - } - - const util::Vector3D* GetWallData(site_t iBlockNumber, site_t iSiteNumber) const - { - return ((const Derived*) (this))->DoGetWallData(iBlockNumber, iSiteNumber); - } - - void SetWallData(site_t iBlockNumber, site_t iSiteNumber, const util::Vector3D& iData) - { - return ((Derived*) (this))->DoSetWallData(iBlockNumber, iSiteNumber, iData); - } - - static bool NeedsWallNormals() - { - return Derived::DoNeedsWallNormals(); - } - - const std::vector > GetCorners() const - { - std::vector > lCorners; - - lCorners.push_back(util::Vector3D(minSite.x, minSite.y, minSite.z)); - - lCorners.push_back(util::Vector3D(minSite.x, minSite.y, maxSite.z)); - - lCorners.push_back(util::Vector3D(minSite.x, maxSite.y, minSite.z)); - - lCorners.push_back(util::Vector3D(minSite.x, maxSite.y, maxSite.z)); - - lCorners.push_back(util::Vector3D(maxSite.x, minSite.y, minSite.z)); - - lCorners.push_back(util::Vector3D(maxSite.x, minSite.y, maxSite.z)); - - lCorners.push_back(util::Vector3D(maxSite.x, maxSite.y, minSite.z)); - - lCorners.push_back(util::Vector3D(maxSite.x, maxSite.y, maxSite.z)); - - return lCorners; - } - - /** - * True if the cluster type requires wall normals. - * - * This can be overridden by deriving classes. - * @return - */ - static bool DoNeedsWallNormals() - { - return false; - } - - unsigned short GetBlocksX() const - { - return blocksX; - } - - unsigned short GetBlocksY() const - { - return blocksY; - } - - unsigned short GetBlocksZ() const - { - return blocksZ; - } - - const util::Vector3D& GetMinSite() const - { - return minSite; - } - - const util::Vector3D& GetMaxSite() const - { - return maxSite; - } - - const util::Vector3D& GetLeastSiteOnLeastBlockInImage() const - { - return leastSiteOnLeastBlockInImage; - } - - /** - * Returns the block coordinates of the block with minimal x, y and z - * coordinates in the cluster. - * - * @return The minimal block coordinates - */ - const util::Vector3D& GetMinBlockLocation() const - { - return minBlock; - } - - private: - /** - * The size of the cluster in terms of the number of blocks - */ - unsigned short blocksX; - unsigned short blocksY; - unsigned short blocksZ; - - /** - * The min and maximum site location, in site units - * relative to the centre of the lattice - */ - util::Vector3D minSite; - util::Vector3D maxSite; - - /** - * The lowest x, y and z block location of the ClusterShared - * in terms of site units relative to the centre location - */ - util::Vector3D leastSiteOnLeastBlockInImage; - - /** - * The coordinates of the block with minimal x, y and z components. - */ - util::Vector3D minBlock; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTER_H diff --git a/Code/vis/rayTracer/ClusterBuilder.h b/Code/vis/rayTracer/ClusterBuilder.h deleted file mode 100644 index ff7a9097d..000000000 --- a/Code/vis/rayTracer/ClusterBuilder.h +++ /dev/null @@ -1,343 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTERBUILDER_H -#define HEMELB_VIS_RAYTRACER_CLUSTERBUILDER_H - -#include -#include - -#include "geometry/BlockTraverserWithVisitedBlockTracker.h" -#include "geometry/LatticeData.h" -#include "geometry/SiteTraverser.h" -#include "lb/LbmParameters.h" -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" -#include "vis/rayTracer/ClusterBuilder.h" -#include "vis/rayTracer/ClusterTraverser.h" -#include "vis/rayTracer/RayTracer.h" -#include "vis/rayTracer/SiteData.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - template - class ClusterBuilder - { - public: - ClusterBuilder(const geometry::LatticeData* latticeData, int localRank_) : - mBlockTraverser(*latticeData), localRank(localRank_) - { - mLatticeData = latticeData; - - //Each block is assigned a cluster id once it has been - //assigned to a cluster - mClusterIdOfBlock = new short int[mLatticeData->GetBlockCount()]; - for (site_t lId = 0; lId < mLatticeData->GetBlockCount(); lId++) - { - mClusterIdOfBlock[lId] = NOTASSIGNEDTOCLUSTER; - } - } - - ~ClusterBuilder() - { - delete[] mClusterIdOfBlock; - } - - void BuildClusters() - { - //Initially locate all clusters, locating their - //range by block span and site span - LocateClusters(); - - // Process the flow field for every cluster - for (unsigned int lThisClusterId = 0; lThisClusterId < mClusters.size(); lThisClusterId++) - { - ProcessCluster(lThisClusterId); - } - } - - std::vector& GetClusters() - { - return mClusters; - } - - private: - // Locates all the clusters in the lattice structure and the - void LocateClusters() - { - // Run through all unvisited blocks finding clusters - do - { - //Mark the block visited - mBlockTraverser.MarkCurrentBlockVisited(); - - //If there are sites assigned to the local processor, search for the - //cluster of connected sites - if (AreSitesAssignedToLocalProcessorRankInBlock(mBlockTraverser.GetCurrentBlockData())) - { - FindNewCluster(); - } - } - while (mBlockTraverser.GoToNextUnvisitedBlock()); - } - - //Locates all the clusters in the lattice structure and stores - //their locations - void FindNewCluster() - { - //These locations will eventually contain the bounds of the - //rectangular cluster, both in terms of block number and - //site numbers - util::Vector3D clusterBlockMin = util::Vector3D::MaxLimit(); - util::Vector3D clusterBlockMax = util::Vector3D::MinLimit(); - util::Vector3D clusterSiteMin = util::Vector3D::MaxLimit(); - util::Vector3D clusterSiteMax = util::Vector3D::MinLimit(); - - //To discover the cluster, we continually visit the neighbours - //of sequential blocks - //We keep a stack of all the sites that must be processed - //and sequentially add neighbours to it - std::stack > blocksToProcess; - - //Set up the initial condition - blocksToProcess.push(mBlockTraverser.GetCurrentLocation()); - - //Loop over the cluster via neighbours until - //all blocks have been processed - while (!blocksToProcess.empty()) - { - //Get location off the top of the stack - //(we could actually take anything off the stack) - util::Vector3D lCurrentLocation = blocksToProcess.top(); - blocksToProcess.pop(); - - if (AreSitesAssignedToLocalProcessorRankInBlock(mBlockTraverser.GetBlockDataForLocation(lCurrentLocation))) - { - //Update block range of the cluster - clusterBlockMin.UpdatePointwiseMin(lCurrentLocation); - clusterBlockMax.UpdatePointwiseMax(lCurrentLocation); - - //Update the cluster id of the given block - site_t blockId = mBlockTraverser.GetIndexFromLocation(lCurrentLocation); - mClusterIdOfBlock[blockId] = (short int) mClusters.size(); - - //Loop through all the sites on the block, to - //update the site bounds on the cluster - geometry::SiteTraverser siteTraverser = mBlockTraverser.GetSiteTraverser(); - do - { - //If the site is not a solid - if (!mBlockTraverser.GetBlockDataForLocation(lCurrentLocation).SiteIsSolid(siteTraverser.GetCurrentIndex())) - { - clusterSiteMin.UpdatePointwiseMin(siteTraverser.GetCurrentLocation() - + lCurrentLocation * mBlockTraverser.GetBlockSize()); - - clusterSiteMax.UpdatePointwiseMax(siteTraverser.GetCurrentLocation() - + lCurrentLocation * mBlockTraverser.GetBlockSize()); - } - } - while (siteTraverser.TraverseOne()); - - //Check all the neighbouring blocks to see if they need visiting. Add them to the stack. - AddNeighbouringBlocks(lCurrentLocation, blocksToProcess); - } - } - - AddCluster(clusterBlockMin, clusterBlockMax, clusterSiteMin, clusterSiteMax); - } - - //Adds neighbouring blocks of the input location to the input stack - void AddNeighbouringBlocks(util::Vector3D iCurrentLocation, - std::stack >& oBlocksToProcess) - { - // Loop over all neighbouring blocks - for (int l = 0; l < 26; l++) - { - util::Vector3D lNeighbouringBlock = iCurrentLocation + mNeighbours[l]; - - //The neighouring block location might not exist - //eg negative co-ordinates - if (mBlockTraverser.IsValidLocation(lNeighbouringBlock)) - { - //Ensure that the block hasn't been visited before - if (!mBlockTraverser.IsBlockVisited(lNeighbouringBlock)) - { - //Add to the stack - oBlocksToProcess.push(lNeighbouringBlock); - - //We must mark this locatoin as visited so it only - //gets processed once - mBlockTraverser.MarkBlockVisited(lNeighbouringBlock); - } - } - } - } - - //Returns true if there are sites in the given block associated with the - //local processor rank - bool AreSitesAssignedToLocalProcessorRankInBlock(const geometry::Block& block) - { - if (block.IsEmpty()) - { - return false; - } - - for (site_t siteId = 0; siteId < mLatticeData->GetSitesPerBlockVolumeUnit(); siteId++) - { - if (localRank == block.GetProcessorRankForSite(siteId)) - { - return true; - } - } - return false; - } - - //Adds a new cluster by taking in the required data in interger format - //and converting it to that used by the raytracer - //NB: Futher processing is required on the cluster before it can be used - //by the ray tracer, which is handled by the ProcessCluster method - void AddCluster(util::Vector3D clusterBlockMin, - util::Vector3D clusterBlockMax, - util::Vector3D clusterVoxelMin, - util::Vector3D clusterVoxelMax) - { - const util::Vector3D halfLatticeSiteCount = util::Vector3D(mLatticeData->GetSiteDimensions()) - * 0.5F; - - //The friendly locations must be turned into a format usable by the ray tracer - ClusterType lNewCluster((unsigned short) (1 + clusterBlockMax.x - clusterBlockMin.x), - (unsigned short) (1 + clusterBlockMax.y - clusterBlockMin.y), - (unsigned short) (1 + clusterBlockMax.z - clusterBlockMin.z), - util::Vector3D(clusterVoxelMin) - halfLatticeSiteCount, - util::Vector3D(clusterVoxelMax + util::Vector3D(1)) - - halfLatticeSiteCount, - util::Vector3D(clusterBlockMin * mLatticeData->GetBlockSize()) - - halfLatticeSiteCount, - clusterBlockMin); - - mClusters.push_back(lNewCluster); - - //We need to store the cluster block minimum in - //order to process the cluster - mClusterBlockMins.push_back(clusterBlockMin); - } - - //Adds "flow-field" data to the cluster - void ProcessCluster(unsigned int clusterId) - { - ClusterType& cluster = mClusters[clusterId]; - - ClusterTraverser clusterTraverser(cluster); - - do - { - util::Vector3D blockCoordinates = clusterTraverser.GetCurrentLocation() - + mClusterBlockMins[clusterId]; - - site_t blockId = mLatticeData->GetBlockIdFromBlockCoords(blockCoordinates); - - if (mClusterIdOfBlock[blockId] == (short) clusterId) - { - UpdateSiteData(blockId, clusterTraverser.GetCurrentIndex(), cluster); - } - } - while (clusterTraverser.TraverseOne()); - } - - void UpdateSiteData(site_t blockId, site_t blockIndexWithinCluster, ClusterType& cluster) - { - geometry::SiteTraverser siteTraverser(*mLatticeData); - do - { - UpdateSiteDataAtSite(blockId, blockIndexWithinCluster, cluster, siteTraverser.GetCurrentIndex()); - } - while (siteTraverser.TraverseOne()); - } - - virtual void UpdateSiteDataAtSite(site_t blockId, - site_t blockIndexWithinCluster, - ClusterType& cluster, - site_t siteIdOnBlock) - { - const geometry::Block& block = mLatticeData->GetBlock(blockId); - - //If site not a solid and on the current processor [net.cc] - if (!block.SiteIsSolid(siteIdOnBlock)) - { - if (ClusterType::NeedsWallNormals()) - { - UpdateWallNormalAtSite(block, blockIndexWithinCluster, cluster, siteIdOnBlock); - } - } - } - - void UpdateWallNormalAtSite(const geometry::Block& block, - site_t blockNum, - ClusterType& cluster, - site_t siteIdOnBlock) - { - site_t localIndex = block.GetLocalContiguousIndexForSite(siteIdOnBlock); - - const geometry::Site site = mLatticeData->GetSite(localIndex); - - if (site.IsWall()) - { - cluster.SetWallData(blockNum, siteIdOnBlock, site.GetWallNormal()); - } - } - - //Caution: the data within mClusters is altered by means - //of pointers obtained from the GetClusterVoxelDataPointer - //method. No insertion or copying must therefore take place - //on mClusters once building is complete - std::vector mClusters; - - const geometry::LatticeData* mLatticeData; - - geometry::BlockTraverserWithVisitedBlockTracker mBlockTraverser; - - std::vector > mClusterBlockMins; - - short int *mClusterIdOfBlock; - - int localRank; - - static const short int NOTASSIGNEDTOCLUSTER = -1; - - static const util::Vector3D mNeighbours[26]; - } - ; - - template - const util::Vector3D ClusterBuilder::mNeighbours[26] = { - util::Vector3D(-1, -1, -1), util::Vector3D(-1, -1, 0), util::Vector3D(-1, -1, 1), - util::Vector3D(-1, 0, -1), util::Vector3D(-1, 0, 0), util::Vector3D(-1, 0, 1), - util::Vector3D(-1, 1, -1), util::Vector3D(-1, 1, 0), - util::Vector3D(-1, 1, 1), - util::Vector3D(0, -1, -1), - util::Vector3D(0, -1, 0), - util::Vector3D(0, -1, 1), - util::Vector3D(0, 0, -1), - // 0 0 0 is same site - util::Vector3D(0, 0, 1), util::Vector3D(0, 1, -1), util::Vector3D(0, 1, 0), - util::Vector3D(0, 1, 1), util::Vector3D(1, -1, -1), util::Vector3D(1, -1, 0), - util::Vector3D(1, -1, 1), util::Vector3D(1, 0, -1), util::Vector3D(1, 0, 0), - util::Vector3D(1, 0, 1), util::Vector3D(1, 1, -1), util::Vector3D(1, 1, 0), - util::Vector3D(1, 1, 1) }; - - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTERBUILDER_H diff --git a/Code/vis/rayTracer/ClusterNormal.cc b/Code/vis/rayTracer/ClusterNormal.cc deleted file mode 100644 index a76f298c2..000000000 --- a/Code/vis/rayTracer/ClusterNormal.cc +++ /dev/null @@ -1,47 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "geometry/LatticeData.h" -#include "vis/rayTracer/ClusterNormal.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - ClusterNormal::ClusterNormal(unsigned short xBlockCount, - unsigned short yBlockCount, - unsigned short zBlockCount, - const util::Vector3D& minimalSite, - const util::Vector3D& maximalSite, - const util::Vector3D& minimalSiteOnMinimalBlock, - const util::Vector3D& minimalBlock) : - Cluster(xBlockCount, - yBlockCount, - zBlockCount, - minimalSite, - maximalSite, - minimalSiteOnMinimalBlock, - minimalBlock) - { - } - - const util::Vector3D* ClusterNormal::DoGetWallData(site_t iBlockNumber, site_t iSiteNumber) const - { - return NULL; - } - - void ClusterNormal::DoSetWallData(site_t iBlockNumber, site_t iSiteNumber, const util::Vector3D& iData) - { - } - - } - } -} diff --git a/Code/vis/rayTracer/ClusterNormal.h b/Code/vis/rayTracer/ClusterNormal.h deleted file mode 100644 index 5b0b2b2ba..000000000 --- a/Code/vis/rayTracer/ClusterNormal.h +++ /dev/null @@ -1,42 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTERNORMAL_H -#define HEMELB_VIS_RAYTRACER_CLUSTERNORMAL_H - -#include "vis/rayTracer/Cluster.h" -#include "vis/rayTracer/SiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - class ClusterNormal : public Cluster - { - public: - ClusterNormal(unsigned short xBlockCount, - unsigned short yBlockCount, - unsigned short zBlockCount, - const util::Vector3D& minimalSite, - const util::Vector3D& maximalSite, - const util::Vector3D& minimalSiteOnMinimalBlock, - const util::Vector3D& minimalBlock); - - const util::Vector3D* DoGetWallData(site_t iBlockNumber, site_t iSiteNumber) const; - - void DoSetWallData(site_t iBlockNumber, site_t iSiteNumber, const util::Vector3D& iData); - }; - - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTERNORMAL_H diff --git a/Code/vis/rayTracer/ClusterRayTracer.h b/Code/vis/rayTracer/ClusterRayTracer.h deleted file mode 100644 index 7f9c86e2b..000000000 --- a/Code/vis/rayTracer/ClusterRayTracer.h +++ /dev/null @@ -1,764 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTERRAYTRACER_H -#define HEMELB_VIS_RAYTRACER_CLUSTERRAYTRACER_H - -#include -#include -#include - -#include "geometry/SiteTraverser.h" -#include "lb/MacroscopicPropertyCache.h" -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" -#include "vis/DomainStats.h" -#include "vis/PixelSet.h" -#include "vis/rayTracer/Cluster.h" -#include "vis/rayTracer/ClusterTraverser.h" -#include "vis/rayTracer/Ray.h" -#include "vis/Screen.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - template - class ClusterRayTracer - { - public: - ClusterRayTracer(const Viewpoint& iViewpoint, - Screen& iScreen, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings, - const hemelb::geometry::LatticeData& iLatticeData, - const lb::MacroscopicPropertyCache& propertyCache) : - viewpoint(iViewpoint), screen(iScreen), domainStats(iDomainStats), visSettings(iVisSettings), latticeData(iLatticeData), propertyCache(propertyCache) - { - // TODO: This is absolutely horrible, but neccessary until RayDataNormal is - // removed. - RayDataNormal::mDomainStats = &iDomainStats; - } - - void RenderCluster(const ClusterType& iCluster, PixelSet& pixels) - { - mLowerSiteCordinatesOfClusterRelativeToViewpoint = iCluster.GetLeastSiteOnLeastBlockInImage() - - viewpoint.GetViewpointLocation(); - - //Calculate the projection of the cluster on the screen - //refered to as the subimage - CalculateSubImage(iCluster); - - //If the entire sub-image is off the screen, - //no rendering is needed - if (SubImageOffScreen()) - { - return; - } - - CropSubImageToScreen(); - - CalculateVectorsToClusterSpanAndLowerLeftPixel(iCluster); - - CastRaysForEachPixel(iCluster, pixels); - } - - private: - void GetRayUnitsFromViewpointToCluster(const Ray & iRay, - float & oMaximumRayUnits, - float & oMinimumRayUnits) - { - // (Remember that iRay.mDirection is normalised) - float lMaxUnitRaysBasedOnX; - float lMinUnitRaysBasedOnX; - if (iRay.GetDirection().x > 0.0F) - { - lMaxUnitRaysBasedOnX = mViewpointCentreToMaxSite.x * iRay.GetInverseDirection().x; - - lMinUnitRaysBasedOnX = mViewpointCentreToMinSite.x * iRay.GetInverseDirection().x; - } - else if (iRay.GetDirection().x < 0.0F) - { - lMaxUnitRaysBasedOnX = mViewpointCentreToMinSite.x * iRay.GetInverseDirection().x; - lMinUnitRaysBasedOnX = mViewpointCentreToMaxSite.x * iRay.GetInverseDirection().x; - } - else - { - lMaxUnitRaysBasedOnX = std::numeric_limits::max(); - lMinUnitRaysBasedOnX = 0.0F; - } - - float lMaxUnitRaysBasedOnY; - float lMinUnitRaysBasedOnY; - if (iRay.GetDirection().y > 0.0F) - { - lMaxUnitRaysBasedOnY = mViewpointCentreToMaxSite.y * iRay.GetInverseDirection().y; - - lMinUnitRaysBasedOnY = mViewpointCentreToMinSite.y * iRay.GetInverseDirection().y; - } - else if (iRay.GetDirection().y < 0.0F) - { - lMaxUnitRaysBasedOnY = mViewpointCentreToMinSite.y * iRay.GetInverseDirection().y; - lMinUnitRaysBasedOnY = mViewpointCentreToMaxSite.y * iRay.GetInverseDirection().y; - } - else - { - lMaxUnitRaysBasedOnY = std::numeric_limits::max(); - lMinUnitRaysBasedOnY = 0.0F; - } - - float lMaxUnitRaysBasedOnZ; - float lMinUnitRaysBasedOnZ; - if (iRay.GetDirection().z > 0.0F) - { - lMaxUnitRaysBasedOnZ = mViewpointCentreToMaxSite.z * iRay.GetInverseDirection().z; - - lMinUnitRaysBasedOnZ = mViewpointCentreToMinSite.z * iRay.GetInverseDirection().z; - } - else if (iRay.GetDirection().z < 0.0F) - { - lMaxUnitRaysBasedOnZ = mViewpointCentreToMinSite.z * iRay.GetInverseDirection().z; - lMinUnitRaysBasedOnZ = mViewpointCentreToMaxSite.z * iRay.GetInverseDirection().z; - } - else - { - lMaxUnitRaysBasedOnZ = std::numeric_limits::max(); - lMinUnitRaysBasedOnZ = 0.0F; - } - - //Maximum ray units from viewpoint to cluster - //We want the minimum number - since at this point the ray is - //completely out - oMaximumRayUnits = std::min(std::min(lMaxUnitRaysBasedOnX, lMaxUnitRaysBasedOnY), lMaxUnitRaysBasedOnZ); - - //Maximum ray units to get us into the cluster - //We want the maximum number - since only at this point - // is the ray completely in - oMinimumRayUnits = std::max(std::max(lMinUnitRaysBasedOnX, lMinUnitRaysBasedOnY), lMinUnitRaysBasedOnZ); - - } - - void CastRay(const ClusterType& iCluster, - Ray& ioRay, - float iMaximumRayUnits, - float iMinimumRayUnits) - { - //It's possible for the ray to totally miss the cluster - //This is because the sub-image is square while the cluster - // projection won't be in most circumstances - if (iMaximumRayUnits < iMinimumRayUnits) - { - return; - } - - ioRay.SetRayLengthTraversedToCluster(iMinimumRayUnits); - - util::Vector3D fromLowerSiteToFirstRayClusterIntersection = ioRay.GetDirection() * iMinimumRayUnits - - mLowerSiteCordinatesOfClusterRelativeToViewpoint; - - TraverseBlocks(iCluster, fromLowerSiteToFirstRayClusterIntersection, ioRay); - } - - void CalculateSubImage(const ClusterType& iCluster) - { - //The extent of the cluster when projected (ie the subimage) - //is determined by projecting all eight vertices of the cuboid - XYCoordinates lSubImageLowerLeft = XYCoordinates::MaxLimit(); - XYCoordinates lSubImageUpperRight = XYCoordinates::MinLimit(); - - const std::vector > lCorners = iCluster.GetCorners(); - - for (std::vector >::const_iterator lIt = lCorners.begin(); lIt != lCorners.end(); - lIt++) - { - UpdateSubImageExtentForCorner(*lIt, lSubImageLowerLeft, lSubImageUpperRight); - } - - lowerLeftPixelCoordinatesOfSubImage = - screen.template TransformScreenToPixelCoordinates(lSubImageLowerLeft); - - // We add a unit vector here because the transformation will round down from float - // to int. - upperRightPixelCoordinatesOfSubImage = - screen.template TransformScreenToPixelCoordinates(lSubImageUpperRight); - } - - void UpdateSubImageExtentForCorner(const util::Vector3D& iCorner, - XYCoordinates& ioSubImageLowerLeft, - XYCoordinates& ioSubImageUpperRight) - { - XYCoordinates lCornerProjection = viewpoint.FlatProject(iCorner); - - ioSubImageLowerLeft.UpdatePointwiseMin(lCornerProjection); - ioSubImageUpperRight.UpdatePointwiseMax(lCornerProjection); - } - - bool SubImageOffScreen() - { - return (lowerLeftPixelCoordinatesOfSubImage.x >= screen.GetPixelsX() - || upperRightPixelCoordinatesOfSubImage.x < 0 - || lowerLeftPixelCoordinatesOfSubImage.y >= screen.GetPixelsY() - || upperRightPixelCoordinatesOfSubImage.y < 0); - } - - void CropSubImageToScreen() - { - lowerLeftPixelCoordinatesOfSubImage.x = util::NumericalFunctions::max(lowerLeftPixelCoordinatesOfSubImage.x, - 0); - - upperRightPixelCoordinatesOfSubImage.x = - util::NumericalFunctions::min(upperRightPixelCoordinatesOfSubImage.x, screen.GetPixelsX() - 1); - - lowerLeftPixelCoordinatesOfSubImage.y = util::NumericalFunctions::max(lowerLeftPixelCoordinatesOfSubImage.y, - 0); - - upperRightPixelCoordinatesOfSubImage.y = - util::NumericalFunctions::min(upperRightPixelCoordinatesOfSubImage.y, screen.GetPixelsY() - 1); - } - - void CalculateVectorsToClusterSpanAndLowerLeftPixel(const ClusterType& iCluster) - { - mViewpointCentreToMaxSite = iCluster.GetMaxSite() - viewpoint.GetViewpointLocation(); - - mViewpointCentreToMinSite = iCluster.GetMinSite() - viewpoint.GetViewpointLocation(); - - //Obtaining the vector from the camera to the lower left pixel - fromCameraToBottomLeftPixelOfSubImage = screen.GetCameraToBottomLeftOfScreenVector() - + screen.GetPixelUnitVectorProjectionX() * (float) lowerLeftPixelCoordinatesOfSubImage.x - + screen.GetPixelUnitVectorProjectionY() * (float) lowerLeftPixelCoordinatesOfSubImage.y; - } - - void CastRaysForEachPixel(const ClusterType& iCluster, PixelSet& pixels) - { - XYCoordinates lPixel; - - //Loop over all the pixels - util::Vector3D lCameraToBottomRow = fromCameraToBottomLeftPixelOfSubImage; - for (lPixel.x = lowerLeftPixelCoordinatesOfSubImage.x; lPixel.x <= upperRightPixelCoordinatesOfSubImage.x; - ++lPixel.x) - { - util::Vector3D lCameraToPixel = lCameraToBottomRow; - for (lPixel.y = lowerLeftPixelCoordinatesOfSubImage.y; lPixel.y <= upperRightPixelCoordinatesOfSubImage.y; - ++lPixel.y) - { - CastRayForPixel(iCluster, lPixel, lCameraToPixel, pixels); - - lCameraToPixel += screen.GetPixelUnitVectorProjectionY(); - } - - lCameraToBottomRow += screen.GetPixelUnitVectorProjectionX(); - } - } - - virtual void CastRayForPixel(const ClusterType& iCluster, - const XYCoordinates& iPixelCoordinates, - const util::Vector3D& iRayDirection, - PixelSet& pixels) - { - Ray lRay(iRayDirection, iPixelCoordinates.x, iPixelCoordinates.y); - - //These tell us how many ray units get us into the cluster - //and after how many ray units we are out - float lMaximumRayUnits; - float lMinimumRayUnits; - GetRayUnitsFromViewpointToCluster(lRay, lMaximumRayUnits, lMinimumRayUnits); - - CastRay(iCluster, lRay, lMaximumRayUnits, lMinimumRayUnits); - - //Make sure the ray hasn't reached infinity - if (!lRay.CollectedNoData()) - { - pixels.AddPixel(lRay.GetRayData()); - } - } - - void TraverseRayThroughBlock(const util::Vector3D& fromFirstRayClusterIntersectionToLowerSiteOfCurrentBlock, - const util::Vector3D& iLocationInBlock, - const ClusterType& iCluster, - const util::Vector3D& blockLocation, - const site_t blockNumberOnCluster, - float euclideanClusterLengthTraversedByRay, - Ray& ioRay) - { - //Work out which site we're currently in - const util::Vector3D truncatedLocationInBlock = RoundToNearestVoxel(iLocationInBlock); - - geometry::SiteTraverser siteTraverser(latticeData); - siteTraverser.SetCurrentLocation(truncatedLocationInBlock); - - // In order to trace the rays through the voxels, we need to keep track of how far the - // ray can travel to the next voxel in each of the three directions in ray units - util::Vector3D rayUnitsUntilNextSite = - CalculateRayUnitsBeforeNextSite(fromFirstRayClusterIntersectionToLowerSiteOfCurrentBlock, - util::Vector3D(truncatedLocationInBlock), - ioRay); - - while (siteTraverser.CurrentLocationValid()) - { - // Firstly, work out in which direction we - // can travel the least ray units before reaching - // a vortex side - const util::Direction::Direction directionOfLeastTravel = DirectionOfLeastTravel(rayUnitsUntilNextSite); - - // Find out how far the ray can move - const float manhattanRayLengthThroughVoxel = rayUnitsUntilNextSite.GetByDirection(directionOfLeastTravel); - - const geometry::Block& block = latticeData.GetBlock(latticeData.GetBlockIdFromBlockCoords(blockLocation)); - - if (!block.IsEmpty()) // Ensure fluid site - { - if (!block.SiteIsSolid(siteTraverser.GetCurrentIndex())) - { - const site_t localContiguousId = - block.GetLocalContiguousIndexForSite(siteTraverser.GetCurrentIndex()); - - SiteData_t siteData; - siteData.density = propertyCache.densityCache.Get(localContiguousId); - siteData.velocity = propertyCache.velocityCache.Get(localContiguousId).GetMagnitude(); - - if (visSettings.mStressType == lb::ShearStress) - { - siteData.stress = propertyCache.wallShearStressMagnitudeCache.Get(localContiguousId); - } - else - { - siteData.stress = propertyCache.vonMisesStressCache.Get(localContiguousId); - } - - const util::Vector3D* lWallData = iCluster.GetWallData(blockNumberOnCluster, - siteTraverser.GetCurrentIndex()); - - if (lWallData == NULL || lWallData->x == NO_VALUE) - { - ioRay.UpdateDataForNormalFluidSite(siteData, - manhattanRayLengthThroughVoxel - - euclideanClusterLengthTraversedByRay, // Manhattan Ray-length through the voxel - euclideanClusterLengthTraversedByRay, // euclidean ray units spent in cluster - domainStats, - visSettings); - } - else - { - ioRay.UpdateDataForWallSite(siteData, - manhattanRayLengthThroughVoxel - euclideanClusterLengthTraversedByRay, - euclideanClusterLengthTraversedByRay, - domainStats, - visSettings, - lWallData); - } - } - else - { - ioRay.ProcessSolidSite(); - } - } - else - { - ioRay.ProcessSolidSite(); - } - - //Update ray length traversed so far - euclideanClusterLengthTraversedByRay = manhattanRayLengthThroughVoxel; - - //Update the block location and RayUnitsBeforeNextVoxel - //in each direction - switch (directionOfLeastTravel) - { - case util::Direction::X: - if (ioRay.XIncreasing()) - { - siteTraverser.IncrementX(); - rayUnitsUntilNextSite.x += ioRay.GetInverseDirection().x; - } - else - { - siteTraverser.DecrementX(); - rayUnitsUntilNextSite.x -= ioRay.GetInverseDirection().x; - } - - break; - - case util::Direction::Y: - if (ioRay.YIncreasing()) - { - siteTraverser.IncrementY(); - rayUnitsUntilNextSite.y += ioRay.GetInverseDirection().y; - } - else - { - siteTraverser.DecrementY(); - rayUnitsUntilNextSite.y -= ioRay.GetInverseDirection().y; - } - break; - - case util::Direction::Z: - if (ioRay.ZIncreasing()) - { - siteTraverser.IncrementZ(); - rayUnitsUntilNextSite.z += ioRay.GetInverseDirection().z; - } - else - { - siteTraverser.DecrementZ(); - rayUnitsUntilNextSite.z -= ioRay.GetInverseDirection().z; - } - break; - } - - } - } - - util::Vector3D CalculateRayUnitsBeforeNextSite(const util::Vector3D& iFirstRayClusterIntersectionToBlockLowerSite, - const util::Vector3D& iTruncatedLocationInBlock, - const Ray& iRay) const - { - util::Vector3D lRayUnits; - - //The ray has already travelled iFirstRayClusterIntersectionToBlockLowerSite - //If the ray is increasing in the co-ordinte it can travel as far - //as the truncated location + 1, otherwise just the truncated location - //for each co-ordinate - - //If the ray has zero in any direction, set the ray units to max - - if (iRay.GetDirection().x == 0.0F) - { - lRayUnits.x = std::numeric_limits::max(); - } - else - { - lRayUnits.x = iFirstRayClusterIntersectionToBlockLowerSite.x + (float) (iTruncatedLocationInBlock.x); - if (iRay.XIncreasing()) - { - lRayUnits.x += 1.0F; - } - //Convert from site units into ray units - lRayUnits.x *= iRay.GetInverseDirection().x; - } - - if (iRay.GetDirection().y == 0.0F) - { - lRayUnits.y = std::numeric_limits::max(); - } - else - { - lRayUnits.y = iFirstRayClusterIntersectionToBlockLowerSite.y + (float) (iTruncatedLocationInBlock.y); - if (iRay.YIncreasing()) - { - lRayUnits.y += 1.0F; - } - lRayUnits.y *= iRay.GetInverseDirection().y; - } - - if (iRay.GetDirection().z == 0.0F) - { - lRayUnits.z = std::numeric_limits::max(); - } - else - { - lRayUnits.z = iFirstRayClusterIntersectionToBlockLowerSite.z + (float) (iTruncatedLocationInBlock.z); - if (iRay.ZIncreasing()) - { - lRayUnits.z += 1.0F; - } - lRayUnits.z *= iRay.GetInverseDirection().z; - } - - return lRayUnits; - } - - util::Vector3D RoundToNearestVoxel(const util::Vector3D& iUnboundLocation) const - { - util::Vector3D lVoxelLocationInBlock; - - //Due to rounding errors, it's possible for the site location within a block - //to be outside the wrong block - lVoxelLocationInBlock.x = util::NumericalFunctions::enforceBounds((site_t) iUnboundLocation.x, - 0, - latticeData.GetBlockSize() - 1); - - lVoxelLocationInBlock.y = util::NumericalFunctions::enforceBounds((site_t) iUnboundLocation.y, - 0, - latticeData.GetBlockSize() - 1); - - lVoxelLocationInBlock.z = util::NumericalFunctions::enforceBounds((site_t) iUnboundLocation.z, - 0, - latticeData.GetBlockSize() - 1); - - return lVoxelLocationInBlock; - } - - util::Direction::Direction DirectionOfLeastTravel(const util::Vector3D& iRayUnitsBeforeNextVoxelOrBlock) const - { - - if (iRayUnitsBeforeNextVoxelOrBlock.x < iRayUnitsBeforeNextVoxelOrBlock.y) - { - //X is less than Y - if (iRayUnitsBeforeNextVoxelOrBlock.x < iRayUnitsBeforeNextVoxelOrBlock.z) - { - //X is less than Y and Z - return util::Direction::X; - } - else - { - //X is less than Y - //Z is less Than X (And Y) - return util::Direction::Z; - } - } - else - { - // Y is less than X - if (iRayUnitsBeforeNextVoxelOrBlock.y < iRayUnitsBeforeNextVoxelOrBlock.z) - { - //Y is less than X and Z - return util::Direction::Y; - } - else - { - //Y is less than X - //Z is less than Y (and so X) - return util::Direction::Z; - } - - } - } - - void TraverseBlocks(const ClusterType& iCluster, - const util::Vector3D& fromLowerClusterSiteToFirstRayIntersection, - Ray& ioRay) - { - float blockSizeAsFloat = (float) (latticeData.GetBlockSize()); - - //Calculate the coordinates of the block within the cluster where - //the ray first intersects - const util::Vector3D blockHoldingFirstIntersection = - GetBlockCoordinatesOfFirstIntersectionBlock(iCluster, fromLowerClusterSiteToFirstRayIntersection); - - //The Cluster Traverser keeps track of which block we're at - //in the cluster - ClusterTraverser clusterTraverser(iCluster); - - clusterTraverser.SetCurrentLocation(blockHoldingFirstIntersection); - - // The number of ray units in a block in each direction are cached. - const util::Vector3D rayUnitsAlongEachBlockSize = ioRay.GetInverseDirection() * blockSizeAsFloat; - - //For every block that is traversed, a vector is needed from - //where the ray first hits the cluster to the lower site ie site - //(0,0,0) within the block. This is required to locate how - //far the ray has travelled and where each part is in relation to - //voxel sites - util::Vector3D fromFirstIntersectionToLowerSiteOfCurrentBlock = - util::Vector3D(blockHoldingFirstIntersection) * blockSizeAsFloat - - fromLowerClusterSiteToFirstRayIntersection; - - //We need to know how many ray units can be traversed before - //a new block is hit. The initial value is calculated based on - //the location of first intersection - util::Vector3D totalRayUnitsToNextBlockFromFirstIntersection = - CalculateMinimalTotalRayUnitsToBlocksBehindCurrentOne(fromFirstIntersectionToLowerSiteOfCurrentBlock, - ioRay); - - // We need to track how far the ray has travelled - float siteUnitsTraversed = 0.0F; - - while (clusterTraverser.CurrentLocationValid()) - { - // The location of the ray within the block. - util::Vector3D siteLocationWithinBlock = (ioRay.GetDirection()) * siteUnitsTraversed - - fromFirstIntersectionToLowerSiteOfCurrentBlock; - - TraverseRayThroughBlock(fromFirstIntersectionToLowerSiteOfCurrentBlock, - siteLocationWithinBlock, - iCluster, - iCluster.GetMinBlockLocation() + clusterTraverser.GetCurrentLocation(), - clusterTraverser.GetCurrentIndex(), - siteUnitsTraversed, - ioRay); - - // The direction of least travel is the direction of - // the next block that will be hit by the ray - // relative to the current block. - util::Direction::Direction lDirectionOfLeastTravel = - DirectionOfLeastTravel(totalRayUnitsToNextBlockFromFirstIntersection); - - //Move to the next block based on the direction - //of least travel and update variables accordingly - siteUnitsTraversed = - totalRayUnitsToNextBlockFromFirstIntersection.GetByDirection(lDirectionOfLeastTravel); - - switch (lDirectionOfLeastTravel) - { - case util::Direction::X: - if (ioRay.XIncreasing()) - { - clusterTraverser.IncrementX(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.x += blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.x += rayUnitsAlongEachBlockSize.x; - } - else - { - clusterTraverser.DecrementX(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.x -= blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.x -= rayUnitsAlongEachBlockSize.x; - } - break; - - case util::Direction::Y: - if (ioRay.YIncreasing()) - { - clusterTraverser.IncrementY(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.y += blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.y += rayUnitsAlongEachBlockSize.y; - } - else - { - clusterTraverser.DecrementY(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.y -= blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.y -= rayUnitsAlongEachBlockSize.y; - } - break; - - case util::Direction::Z: - if (ioRay.ZIncreasing()) - { - clusterTraverser.IncrementZ(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.z += blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.z += rayUnitsAlongEachBlockSize.z; - } - else - { - clusterTraverser.DecrementZ(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.z -= blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.z -= rayUnitsAlongEachBlockSize.z; - } - break; - } - - } - - } - - util::Vector3D GetBlockCoordinatesOfFirstIntersectionBlock(const ClusterType& iCluster, - const util::Vector3D& iLowerSiteToFirstRayClusterIntersection) - { - util::Vector3D lBlockCoordinatesOfFirstIntersectionBlock; - - //Perform the truncated division and ensure that the - //coordinates are valid to allow for numerical errors - const util::Vector3D exactBlockCoordsOfFirstIntersectingBlock = - iLowerSiteToFirstRayClusterIntersection * (1.0F / (float) latticeData.GetBlockSize()); - - lBlockCoordinatesOfFirstIntersectionBlock.x = - (site_t) util::NumericalFunctions::enforceBounds((site_t) exactBlockCoordsOfFirstIntersectingBlock.x, - 0, - iCluster.GetBlocksX() - 1); - - lBlockCoordinatesOfFirstIntersectionBlock.y = - (site_t) util::NumericalFunctions::enforceBounds((site_t) exactBlockCoordsOfFirstIntersectingBlock.y, - 0, - iCluster.GetBlocksY() - 1); - - lBlockCoordinatesOfFirstIntersectionBlock.z = - (site_t) util::NumericalFunctions::enforceBounds((site_t) exactBlockCoordsOfFirstIntersectingBlock.z, - 0, - iCluster.GetBlocksZ() - 1); - - return lBlockCoordinatesOfFirstIntersectionBlock; - } - - util::Vector3D CalculateMinimalTotalRayUnitsToBlocksBehindCurrentOne(const util::Vector3D& lFirstIntersectionToBlockLowerSite, - const Ray& iRay) const - { - util::Vector3D lRayUnits; - - //The ray is currently at the first intersection - //The number of ray units for a given co-ordinate is the - //distance in sites to the next block divided by - //the ray unit distance projected in that direction - - if (iRay.GetDirection().x == 0.0F) - { - lRayUnits.x = std::numeric_limits::max(); - } - else - { - lRayUnits.x = lFirstIntersectionToBlockLowerSite.x; - //If the ray is increasing in this coordinate, we want the - //distance to the next block - if (iRay.XIncreasing()) - { - lRayUnits.x += (float) (latticeData.GetBlockSize()); - } - //Turn this from site units into ray units - lRayUnits.x *= iRay.GetInverseDirection().x; - } - - if (iRay.GetDirection().y == 0.0F) - { - lRayUnits.y = std::numeric_limits::max(); - } - else - { - lRayUnits.y = lFirstIntersectionToBlockLowerSite.y; - if (iRay.YIncreasing()) - { - lRayUnits.y += (float) (latticeData.GetBlockSize()); - } - lRayUnits.y *= iRay.GetInverseDirection().y; - } - - if (iRay.GetDirection().z == 0.0F) - { - lRayUnits.z = std::numeric_limits::max(); - } - else - { - lRayUnits.z = lFirstIntersectionToBlockLowerSite.z; - if (iRay.ZIncreasing()) - { - lRayUnits.z += (float) (latticeData.GetBlockSize()); - } - lRayUnits.z *= iRay.GetInverseDirection().z; - } - - return lRayUnits; - } - - const Viewpoint& viewpoint; - const Screen& screen; - const DomainStats& domainStats; - const VisSettings& visSettings; - const hemelb::geometry::LatticeData& latticeData; - /** - * The cache of macroscopic fluid properties at each local fluid site. - */ - const lb::MacroscopicPropertyCache& propertyCache; - - util::Vector3D fromCameraToBottomLeftPixelOfSubImage; - - util::Vector3D mLowerSiteCordinatesOfClusterRelativeToViewpoint; - - XYCoordinates lowerLeftPixelCoordinatesOfSubImage; - XYCoordinates upperRightPixelCoordinatesOfSubImage; - - //Vectors from the viewpoint centre - //to the maximum and minimum site span - //locations respectively - util::Vector3D mViewpointCentreToMaxSite; - util::Vector3D mViewpointCentreToMinSite; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTERRAYTRACER_H diff --git a/Code/vis/rayTracer/ClusterTraverser.h b/Code/vis/rayTracer/ClusterTraverser.h deleted file mode 100644 index 2282200c1..000000000 --- a/Code/vis/rayTracer/ClusterTraverser.h +++ /dev/null @@ -1,58 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTERTRAVERSER_H -#define HEMELB_VIS_RAYTRACER_CLUSTERTRAVERSER_H - -#include "geometry/VolumeTraverser.h" -#include "vis/rayTracer/Cluster.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - //ClusterTraverser is used to traverse the cluster - template - class ClusterTraverser : public geometry::VolumeTraverser - { - public: - ClusterTraverser(const ClusterType& iCluster) : - mCluster(iCluster) - { - } - - virtual ~ClusterTraverser() - { - } - - site_t GetXCount() const - { - return mCluster.GetBlocksX(); - } - - site_t GetYCount() const - { - return mCluster.GetBlocksY(); - } - - site_t GetZCount() const - { - return mCluster.GetBlocksZ(); - } - - private: - const ClusterType& mCluster; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTERTRAVERSER_H diff --git a/Code/vis/rayTracer/ClusterWithWallNormals.cc b/Code/vis/rayTracer/ClusterWithWallNormals.cc deleted file mode 100644 index 638802d9b..000000000 --- a/Code/vis/rayTracer/ClusterWithWallNormals.cc +++ /dev/null @@ -1,67 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "geometry/LatticeData.h" -#include "vis/rayTracer/ClusterWithWallNormals.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - ClusterWithWallNormals::ClusterWithWallNormals(unsigned short xBlockCount, - unsigned short yBlockCount, - unsigned short zBlockCount, - const util::Vector3D& minimalSite, - const util::Vector3D& maximalSite, - const util::Vector3D& minimalSiteOnMinimalBlock, - const util::Vector3D& minimalBlock) : - Cluster(xBlockCount, - yBlockCount, - zBlockCount, - minimalSite, - maximalSite, - minimalSiteOnMinimalBlock, - minimalBlock) - { - WallNormals.resize(GetBlocksX() * GetBlocksY() * GetBlocksZ()); - } - - const util::Vector3D* ClusterWithWallNormals::DoGetWallData(site_t blockNumber, site_t siteNumber) const - { - if (siteNumber < (site_t) WallNormals[blockNumber].size()) - { - return WallNormals[blockNumber][siteNumber]; - } - else - { - return NULL; - } - } - - void ClusterWithWallNormals::DoSetWallData(site_t blockNumber, - site_t siteNumber, - const util::Vector3D& data) - { - if (WallNormals[blockNumber].size() <= (size_t) siteNumber) - { - WallNormals[blockNumber].resize(siteNumber + 1, NULL); - } - - WallNormals[blockNumber][siteNumber] = &data; - } - - bool ClusterWithWallNormals::DoNeedsWallNormals() - { - return true; - } - } - } -} diff --git a/Code/vis/rayTracer/ClusterWithWallNormals.h b/Code/vis/rayTracer/ClusterWithWallNormals.h deleted file mode 100644 index 85b174403..000000000 --- a/Code/vis/rayTracer/ClusterWithWallNormals.h +++ /dev/null @@ -1,47 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTERWITHWALLNORMALS_H -#define HEMELB_VIS_RAYTRACER_CLUSTERWITHWALLNORMALS_H - -#include "vis/rayTracer/Cluster.h" -#include "vis/rayTracer/SiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - class ClusterWithWallNormals : public Cluster - { - public: - ClusterWithWallNormals(unsigned short xBlockCount, - unsigned short yBlockCount, - unsigned short zBlockCount, - const util::Vector3D& minimalSite, - const util::Vector3D& maximalSite, - const util::Vector3D& minimalSiteOnMinimalBlock, - const util::Vector3D& minimalBlock); - - const util::Vector3D* DoGetWallData(site_t iBlockNumber, site_t iSiteNumber) const; - - void DoSetWallData(site_t iBlockNumber, site_t iSiteNumber, const util::Vector3D& iData); - - static bool DoNeedsWallNormals(); - - private: - std::vector*> > WallNormals; - }; - - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTERWITHWALLNORMALS_H diff --git a/Code/vis/rayTracer/HSLToRGBConverter.cc b/Code/vis/rayTracer/HSLToRGBConverter.cc deleted file mode 100644 index 9aab48b27..000000000 --- a/Code/vis/rayTracer/HSLToRGBConverter.cc +++ /dev/null @@ -1,130 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include -#include -#include -#if HEMELB_HAVE_CSTDINT -# include -#else -# include -#endif - -#include "vis/rayTracer/HSLToRGBConverter.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - //See http://en.wikipedia.org/wiki/HSL_and_HSV for the - //formulae used - //TODO: Possibly replace truncation with rounding? - void HSLToRGBConverter::Convert(float iHue, - float iSaturation, - float iLightness, - unsigned char oRGBColour[3]) - { - //All values are stored as 32-bit unsigned integers - //stored within 16-bits, to allow multiplication between - //two values - - //All values are scaled up to be between 0 and the - //maximum value of a 16-bit interger - - //Scales a value between 0 and 360 to a range between 0 - //and the maximum value of a 16-bit interger - static const uint32_t DegreesScalar = std::numeric_limits::max() / 360; - - //Scales a value between 0 and 1 to a range between 0 - //and the maximum value of a 16-bit interger - static const uint32_t OtherScalar = std::numeric_limits::max(); - - //Cast the inputs accodingly - uint32_t lHue = (uint32_t)(iHue * (float) (DegreesScalar)); - uint32_t lSaturation = (uint32_t)(iSaturation * (float) (OtherScalar)); - uint32_t lLightness = (uint32_t)(iLightness * (float) (OtherScalar)); - - //Calculate the Chroma - a division by OtherScalar is - //required for the Chroma to remain between 0 - //and the maximum value of a 16-bit interger - int32_t lTemp = 2 * (int32_t) (lLightness) - (int32_t) (OtherScalar); - uint32_t lChroma = ( (OtherScalar - abs(lTemp)) * lSaturation) / OtherScalar; - - uint32_t lHuePrime = lHue / 60; - - lTemp = (int) (lHuePrime % (2 * DegreesScalar)) - (int) (DegreesScalar); - - //Calculate the "Intermediate" - a division by - //DegreesScalar is required for the Chroma to - //remain between 0 and the maximum value of a 16-bit interger - uint32_t lIntermediate = (lChroma * (DegreesScalar - abs(lTemp))) / DegreesScalar; - - uint32_t red=0, green=0, blue=0; - - //Map the hue to value to six cases - uint32_t lHueInt = lHuePrime / DegreesScalar; - switch (lHueInt) - { - case 0: - red = lChroma; - green = lIntermediate; - blue = 0.0F; - break; - - case 1: - red = lIntermediate; - green = lChroma; - blue = 0.0F; - break; - - case 2: - red = 0.0F; - green = lChroma; - blue = lIntermediate; - break; - - case 3: - red = 0.0F; - green = lIntermediate; - blue = lChroma; - break; - - case 4: - red = lIntermediate; - green = 0.0F; - blue = lChroma; - break; - - case 5: - red = lChroma; - green = 0.0F; - blue = lIntermediate; - break; - - default: - log::Logger::Log("Failed while using hue in HslToRgbConvertor."); - } - - // A value to divide the results by to map them - // between 0 and the maximum value of an unsigned - //char - static const uint32_t CharScalar = OtherScalar / std::numeric_limits::max(); - - int32_t lMatcher = (int) (lLightness) - (int) (lChroma) / 2; - - oRGBColour[0] = (unsigned char) ( (red + lMatcher) / CharScalar); - oRGBColour[1] = (unsigned char) ( (green + lMatcher) / CharScalar); - oRGBColour[2] = (unsigned char) ( (blue + lMatcher) / CharScalar); - } - } - } -} diff --git a/Code/vis/rayTracer/HSLToRGBConverter.h b/Code/vis/rayTracer/HSLToRGBConverter.h deleted file mode 100644 index b2a5c86d2..000000000 --- a/Code/vis/rayTracer/HSLToRGBConverter.h +++ /dev/null @@ -1,37 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_HSLTORGBCONVERTER_H -#define HEMELB_VIS_RAYTRACER_HSLTORGBCONVERTER_H - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - class HSLToRGBConverter - { - public: - /** - * Converts a colour in HSL coordinates to - * RGB coordinates (between 0 and 255) - * iHue must be between 0.0F and 360.0F in degrees - * and the other two between 0.0F and 1.0F - */ - static void Convert(float iHue, - float iSaturation, - float iLightness, - unsigned char oRGBColour[3]); - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_HSLTORGBCONVERTER_H diff --git a/Code/vis/rayTracer/Ray.h b/Code/vis/rayTracer/Ray.h deleted file mode 100644 index 8f8785dbc..000000000 --- a/Code/vis/rayTracer/Ray.h +++ /dev/null @@ -1,168 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_RAY_H -#define HEMELB_VIS_RAYTRACER_RAY_H - -#include -#include - -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" -#include "vis/rayTracer/SiteData.h" -#include "vis/rayTracer/RayDataNormal.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - template - class Ray - { - public: - Ray(util::Vector3D iDirection, int i, int j) : - - mDirection(iDirection.Normalise()), - mInverseDirection(1.0F / mDirection.x, 1.0F / mDirection.y, 1.0F / mDirection.z), - mInWall(false), mPassedThroughNormalFluidSite(false), mRayData(i, j) - { - } - - const util::Vector3D& GetDirection() const - { - return mDirection; - } - - const util::Vector3D& GetInverseDirection() const - { - return mInverseDirection; - } - - bool XIncreasing() const - { - return GetDirection().x > 0.0F; - } - - bool YIncreasing() const - { - return GetDirection().y > 0.0F; - } - - bool ZIncreasing() const - { - return GetDirection().z > 0.0F; - } - - void UpdateDataForWallSite(const SiteData_t& iSiteData, - const float iRayLengthInVoxel, - const float iRayUnitsInCluster, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings, - const util::Vector3D* iWallNormal) - { - //Have we just entered the wall? - if (!mInWall) - { - mRayData.UpdateDataForWallSite(iSiteData, - GetDirection(), - iRayLengthInVoxel, - iRayUnitsInCluster + mRayUnitsTraversedToCluster, - iDomainStats, - iVisSettings, - iWallNormal); - //We're in the wall - mInWall = true; - } - else - { - //We've already processed a wall site - process as a normal site - mRayData.UpdateDataForNormalFluidSite(iSiteData, - GetDirection(), - iRayLengthInVoxel, - iRayUnitsInCluster - + mRayUnitsTraversedToCluster, - iDomainStats, - iVisSettings); - } - } - - void UpdateDataForNormalFluidSite(const SiteData_t& iSiteData, - const float manhattanRayLengthThroughVoxel, - const float euclideanRayUnitsSpentInCluster, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings) - { - //Set mInWall to false in case we've just left a wall - mInWall = false; - - //We know we've passed through normal fluid - mPassedThroughNormalFluidSite = true; - - mRayData.UpdateDataForNormalFluidSite(iSiteData, - GetDirection(), - manhattanRayLengthThroughVoxel, - euclideanRayUnitsSpentInCluster - + mRayUnitsTraversedToCluster, - iDomainStats, - iVisSettings); - } - - void ProcessSolidSite() - { - //Special case - tangenting the vessel and never reaching - //normal fluid sites - if (mInWall && !mPassedThroughNormalFluidSite) - { - mRayData.ProcessTangentingVessel(); - } - - //We're out the wall - mInWall = false; - } - - RayDataType GetRayData() - { - return mRayData; - } - - bool CollectedNoData() - { - return !mRayData.ContainsRayData(); - } - - void SetRayLengthTraversedToCluster(float iRayUnitsTraversedToCluster) - { - mRayUnitsTraversedToCluster = iRayUnitsTraversedToCluster; - } - - private: - const util::Vector3D mDirection; - const util::Vector3D mInverseDirection; - - //mInWall indicates whether the ray is in a wall or not - //- if a wall site has just been processed - bool mInWall; - - //mPassedThroughNormalFluid indicates whether the ray has passed - //through normal fluid since entering a wall. If this remains false - //and the ray hits a non-fluid site, the ray has just tangented - //the surface of the vessel - bool mPassedThroughNormalFluidSite; - - float mRayUnitsTraversedToCluster; - - RayDataType mRayData; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_RAY_H diff --git a/Code/vis/rayTracer/RayData.h b/Code/vis/rayTracer/RayData.h deleted file mode 100644 index 079436b86..000000000 --- a/Code/vis/rayTracer/RayData.h +++ /dev/null @@ -1,251 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_RAYDATA_H -#define HEMELB_VIS_RAYTRACER_RAYDATA_H - -#include "constants.h" -#include "net/mpi.h" -#include "vis/BasicPixel.h" -#include "vis/DomainStats.h" -#include "vis/VisSettings.h" -#include "vis/rayTracer/SiteData.h" -#include "lb/LbmParameters.h" -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - /** - * The abstract base type for RayData, using the CRTP pattern - * Contains functionality common to all derived types - */ - - template - class RayData : public BasicPixel - { - public: - RayData(int i, int j) : - BasicPixel(i, j) - { - // A cheap way of indicating no ray data - mCumulativeLengthInFluid = 0.0F; - } - - RayData() - { - - } - - // Used to process the ray data for a normal (non-wall) fluid site - // Calls the method common to all ray data processing classes - // and then the derived class method - void UpdateDataForNormalFluidSite(const raytracer::SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const float iAbsoluteDistanceFromViewpoint, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings) - { - UpdateRayDataCommon(iSiteData, - iRayDirection, - iRayLengthInVoxel, - iAbsoluteDistanceFromViewpoint, - iDomainStats, - iVisSettings); - - ((Derived*) (this))->DoUpdateDataForNormalFluidSite(iSiteData, - iRayDirection, - iRayLengthInVoxel, - iVisSettings); - } - - // Processes the ray data for a normal (non wall) fluid site - void UpdateDataForWallSite(const raytracer::SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const float iAbsoluteDistanceFromViewpoint, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings, - const util::Vector3D* iWallNormal) - { - UpdateRayDataCommon(iSiteData, - iRayDirection, - iRayLengthInVoxel, - iAbsoluteDistanceFromViewpoint, - iDomainStats, - iVisSettings); - - ((Derived*) (this))->DoUpdateDataForWallSite(iSiteData, - iRayDirection, - iRayLengthInVoxel, - iVisSettings, - iWallNormal); - } - - void ProcessTangentingVessel() - { - ((Derived*) (this))->DoProcessTangentingVessel(); - } - - // Merges in the data from another segment of ray (from another core) - void Combine(const Derived& iOtherRayData) - { - // Carry out the merging specific to the derived class - ((Derived*) (this))->DoCombine(iOtherRayData); - - // Sum length in fluid - SetCumulativeLengthInFluid(GetCumulativeLengthInFluid() + iOtherRayData.GetCumulativeLengthInFluid()); - - // Update data relating to site nearest to viewpoint - if (iOtherRayData.GetLengthBeforeRayFirstCluster() < this->GetLengthBeforeRayFirstCluster()) - { - SetLengthBeforeRayFirstCluster(iOtherRayData.GetLengthBeforeRayFirstCluster()); - - SetNearestDensity(iOtherRayData.GetNearestDensity()); - SetNearestStress(iOtherRayData.GetNearestStress()); - } - } - - // Obtains the colour representing the velocity ray trace - void GetVelocityColour(unsigned char oColour[3], - const VisSettings& iVisSettings, - const DomainStats& iDomainStats) const - { - ((const Derived*) (this))->DoGetVelocityColour(oColour, - GetLengthBeforeRayFirstCluster() - / iVisSettings.maximumDrawDistance, - iDomainStats); - } - - // Obtains the colour representing the stress ray trace - void GetStressColour(unsigned char oColour[3], - const VisSettings& iVisSettings, - const DomainStats& iDomainStats) const - { - ((const Derived*) (this))->DoGetStressColour(oColour, - GetLengthBeforeRayFirstCluster() - / iVisSettings.maximumDrawDistance, - iDomainStats); - } - - // Whether or not the data contained in the instance has valid - // ray data, or if it has just been constructed - bool ContainsRayData() const - { - return (GetCumulativeLengthInFluid() != 0.0F); - } - - float GetNearestStress() const - { - return mStressAtNearestPoint; - } - - float GetNearestDensity() const - { - return mDensityAtNearestPoint; - } - - float GetCumulativeLengthInFluid() const - { - return mCumulativeLengthInFluid; - } - - float GetLengthBeforeRayFirstCluster() const - { - return mLengthBeforeRayFirstCluster; - } - - void LogDebuggingInformation() const - { - log::Logger::Log("Ray data at (%i,%i) with " - "(lengthToFirstCluster, lengthInFluid, nearestDensity, nearest stress) = (%f, %f, %f, %f)", - GetI(), - GetJ(), - GetLengthBeforeRayFirstCluster(), - GetCumulativeLengthInFluid(), - GetNearestDensity(), - GetNearestStress()); - } - - protected: - static const float mLongestDistanceInVoxelInverse; - - static void PickColour(float value, float colour[3]) - { - colour[0] = util::NumericalFunctions::enforceBounds(4.F * value - 2.F, 0.F, 1.F); - colour[1] - = util::NumericalFunctions::enforceBounds(2.F - 4.F * (float) fabs(value - 0.5F), 0.F, 1.F); - colour[2] = util::NumericalFunctions::enforceBounds(2.F - 4.F * value, 0.F, 1.F); - } - - float mLengthBeforeRayFirstCluster; - float mCumulativeLengthInFluid; - - float mDensityAtNearestPoint; - float mStressAtNearestPoint; - - private: - // Perform processing of the ray data common to all derived - // types, ie cumulative length in the fluid, nearest stress - // and density. - void UpdateRayDataCommon(const raytracer::SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const float iAbsoluteDistanceFromViewpoint, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings) - { - if (GetCumulativeLengthInFluid() == 0.0F) - { - SetLengthBeforeRayFirstCluster(iAbsoluteDistanceFromViewpoint); - - // Keep track of the density nearest to the viewpoint - SetNearestDensity( (iSiteData.density - (float) iDomainStats.density_threshold_min) - * (float) iDomainStats.density_threshold_minmax_inv); - - if (iVisSettings.mStressType == lb::VonMises || iVisSettings.mStressType == lb::ShearStress) - { - // Keep track of the stress nearest to the viewpoint - SetNearestStress(iSiteData.stress * (float) (iDomainStats.stress_threshold_max_inv)); - } - } - - SetCumulativeLengthInFluid(GetCumulativeLengthInFluid() + iRayLengthInVoxel); - } - - void SetNearestStress(float iStress) - { - mStressAtNearestPoint = iStress; - } - - void SetNearestDensity(float iDensity) - { - mDensityAtNearestPoint = iDensity; - } - - void SetCumulativeLengthInFluid(float iLength) - { - mCumulativeLengthInFluid = iLength; - } - - void SetLengthBeforeRayFirstCluster(float iLength) - { - mLengthBeforeRayFirstCluster = iLength; - } - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_RAYDATA_H diff --git a/Code/vis/rayTracer/RayDataEnhanced.cc b/Code/vis/rayTracer/RayDataEnhanced.cc deleted file mode 100644 index 76ae94a30..000000000 --- a/Code/vis/rayTracer/RayDataEnhanced.cc +++ /dev/null @@ -1,62 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "vis/rayTracer/RayDataEnhanced.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - namespace DepthCuing - { - const float Mist::SurfaceNormalLightnessRange = 0.3F; - const float Mist::ParallelSurfaceAttenuation = 0.5F; - const float Mist::LowestLightness = 0.3F; - const float Mist::VelocityHueMin = 240.0F; - const float Mist::VelocityHueRange = 120.0F; - const float Mist::VelocitySaturation = 1.0F; - const float Mist::StressHue = 230.0F; - const float Mist::StressSaturationRange = 0.5F; - const float Mist::StressSaturationMin = 0.5F; - - const float None::SurfaceNormalLightnessRange = 0.5F; - const float None::ParallelSurfaceAttenuation = 0.75F; - const float None::LowestLightness = 0.3F; - const float None::VelocityHueMin = 240.0F; - const float None::VelocityHueRange = 120.0F; - const float None::VelocitySaturation = 1.0F; - const float None::StressHue = 230.0F; - const float None::StressSaturationRange = 0.5F; - const float None::StressSaturationMin = 0.5F; - - const float Darkness::SurfaceNormalLightnessRange = 0.3F; - const float Darkness::ParallelSurfaceAttenuation = 0.5F; - const float Darkness::LowestLightness = 0.0F; - const float Darkness::VelocityHueMin = 240.0F; - const float Darkness::VelocityHueRange = 120.0F; - const float Darkness::VelocitySaturation = 1.0F; - const float Darkness::StressHue = 230.0F; - const float Darkness::StressSaturationRange = 0.5F; - const float Darkness::StressSaturationMin = 0.5F; - } - } - } - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() - { - MPI_Datatype ret = vis::raytracer::RayDataEnhanced::GetMpiType(); - HEMELB_MPI_CALL(MPI_Type_commit, (&ret)); - return ret; - } - } -} diff --git a/Code/vis/rayTracer/RayDataEnhanced.h b/Code/vis/rayTracer/RayDataEnhanced.h deleted file mode 100644 index f2a8e7af2..000000000 --- a/Code/vis/rayTracer/RayDataEnhanced.h +++ /dev/null @@ -1,297 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_RAYDATAENHANCED_H -#define HEMELB_VIS_RAYTRACER_RAYDATAENHANCED_H - -#include "net/mpi.h" -#include "vis/DomainStats.h" -#include "vis/rayTracer/HSLToRGBConverter.h" -#include "vis/rayTracer/RayData.h" -#include "vis/VisSettings.h" -#include "util/Vector3D.h" -#include "lb/LbmParameters.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - namespace DepthCuing - { - class Mist - { - static const float SurfaceNormalLightnessRange; - static const float ParallelSurfaceAttenuation; - static const float LowestLightness; - static const float VelocityHueMin; - static const float VelocityHueRange; - static const float VelocitySaturation; - static const float StressHue; - static const float StressSaturationRange; - static const float StressSaturationMin; - - //Obtain the lightness value for the ray, based on - //the lightness obtained through surface normals - //and optional depth cuing - static float GetLightnessValue(const float normalisedDistance, - const float surfaceNormalLightness) - { - //Set the smallest lightness value to between - //the mimimum lightness and 1.0F based on the normalised distance between - //the viewpoint and the first cluster hit - //Add onto this the surface normal lightness - float lightnessValue = LowestLightness + (1.0F - LowestLightness) - * normalisedDistance + surfaceNormalLightness * SurfaceNormalLightnessRange; - - if (lightnessValue > 1.0F) - { - return 1.0F; - } - return lightnessValue; - } - }; - - class None - { - static const float SurfaceNormalLightnessRange; - static const float ParallelSurfaceAttenuation; - static const float LowestLightness; - static const float VelocityHueMin; - static const float VelocityHueRange; - static const float VelocitySaturation; - static const float StressHue; - static const float StressSaturationRange; - static const float StressSaturationMin; - - static float GetLightnessValue(const float normalisedDistance, - const float surfaceNormalLightness) - { - return LowestLightness + surfaceNormalLightness * SurfaceNormalLightnessRange; - } - }; - - class Darkness - { - static const float SurfaceNormalLightnessRange; - static const float ParallelSurfaceAttenuation; - static const float LowestLightness; - static const float VelocityHueMin; - static const float VelocityHueRange; - static const float VelocitySaturation; - static const float StressHue; - static const float StressSaturationRange; - static const float StressSaturationMin; - - static float GetLightnessValue(const float normalisedDistance, - const float surfaceNormalLightness) - { - //Set the maximum lightness to be between 0.8F and mLowestLighness - //based on the normalised distance and take off the surface normal - //lightness - float lightnessValue = 0.8F * (1.0F - normalisedDistance) + (surfaceNormalLightness - - 1.0F) * SurfaceNormalLightnessRange; - - if (lightnessValue < LowestLightness) - { - return LowestLightness; - } - return lightnessValue; - } - }; - } - - /** - * RayDataEnhanced - sums velocity and stress data of the ray trace and provides - * with surface normal highlighting and optional depth cuing to enhance the - * 3D perception. - * NB functions prefixed Do should only be called by the base class - */ - class RayDataEnhanced : public RayData - { - public: - RayDataEnhanced(int i, int j) : - RayData (i, j), mSurfaceNormalLightness(1.0F), mVelocitySum(0.0F), - mStressSum(0.0F) - { - } - - RayDataEnhanced() - { - - } - - //Processes the ray data for a normal (non wall) fluid site - void DoUpdateDataForNormalFluidSite(const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings) - { - //Add the velocity multiplied by the ray length in each voxel, - mVelocitySum += iSiteData.velocity * iRayLengthInVoxel; - - if (iVisSettings.mStressType == lb::VonMises) - { - //Update the volume rendering of the von Mises stress flow field - mStressSum = iSiteData.stress * iRayLengthInVoxel; - } - } - - //Processes the data for wall sites - template - void DoUpdateDataForWallSite(const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings, - const util::Vector3D* iWallNormal) - { - //Do everything that would be done for a normal fluid site - DoUpdateDataForNormalFluidSite(iSiteData, - iRayDirection, - iRayLengthInVoxel, - iVisSettings); - - double lDotProduct = iRayDirection.Dot(*iWallNormal); - - // Scale the surface normal lightness between mParallelSurfaceAttenuation - // and 1.0F - // Keep a copy for the special case - mLastSurfaceNormalLightnessMultiplier = (depthCuing::ParallelSurfaceAttenuation + (1.0F - - depthCuing::ParallelSurfaceAttenuation) * fabs(lDotProduct)); - - mSurfaceNormalLightness *= mLastSurfaceNormalLightnessMultiplier; - } - - void DoProcessTangentingVessel() - { - mSurfaceNormalLightness *= mLastSurfaceNormalLightnessMultiplier; - } - - //Obtains the colour representing the velocity ray trace - - template - void DoGetVelocityColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const - { - float lVelocityHue = depthCuing::VelocityHueRange * GetAverageVelocity() - * (float) (iDomainStats.velocity_threshold_max_inv) + depthCuing::VelocityHueMin; - - if (lVelocityHue >= 360.0F) - { - lVelocityHue = 0.0F; - } - - HSLToRGBConverter::Convert(lVelocityHue, - depthCuing::VelocitySaturation, - depthCuing::GetLightnessValue(iNormalisedDistanceToFirstCluster, - GetSurfaceNormalLightness()), - oColour); - } - - //Obtains the colour representing the stress ray trace - template - void DoGetStressColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const - { - float - lStressSaturation = - util::NumericalFunctions::enforceBounds(depthCuing::StressSaturationMin - + depthCuing::StressSaturationRange - * GetAverageStress() - * (float) (iDomainStats.stress_threshold_max_inv), - 0.0F, - 1.0F); - - HSLToRGBConverter::Convert(depthCuing::StressHue, - lStressSaturation, - depthCuing::GetLightnessValue(iNormalisedDistanceToFirstCluster, - GetSurfaceNormalLightness()), - oColour); - } - - //Carries out the merging of the ray data in this - //inherited type, for different segments of the same ray - void DoCombine(const RayDataEnhanced& iOtherRayData) - { - //Add together velocities and stress sums - mVelocitySum += iOtherRayData.GetVelocitySum(); - mStressSum += iOtherRayData.GetStressSum(); - //Multiply the surface lightness - mSurfaceNormalLightness *= iOtherRayData.GetSurfaceNormalLightness(); - } - - float GetVelocitySum() const - { - return mVelocitySum; - } - - float GetStressSum() const - { - return mStressSum; - } - - float GetSurfaceNormalLightness() const - { - return mSurfaceNormalLightness; - } - - float GetAverageVelocity() const - { - return GetVelocitySum() / RayData::GetCumulativeLengthInFluid(); - } - - float GetAverageStress() const - { - return GetStressSum() / RayData::GetCumulativeLengthInFluid(); - } - - static MPI_Datatype GetMpiType() - { - HEMELB_MPI_TYPE_BEGIN(type, RayDataEnhanced, 9); - - HEMELB_MPI_TYPE_ADD_MEMBER(i); - HEMELB_MPI_TYPE_ADD_MEMBER(j); - HEMELB_MPI_TYPE_ADD_MEMBER(mLengthBeforeRayFirstCluster); - HEMELB_MPI_TYPE_ADD_MEMBER(mCumulativeLengthInFluid); - HEMELB_MPI_TYPE_ADD_MEMBER(mDensityAtNearestPoint); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressAtNearestPoint); - HEMELB_MPI_TYPE_ADD_MEMBER(mSurfaceNormalLightness); - HEMELB_MPI_TYPE_ADD_MEMBER(mVelocitySum); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressSum); - - HEMELB_MPI_TYPE_END(type, RayDataEnhanced); - - return type; - } - - private: - float mSurfaceNormalLightness; - float mVelocitySum; - float mStressSum; - - // The last surface normal lightness multiplier - // is retained for the case of tangenting a vessel - // ie only passing through walls sites - // NB: Keep this last as it isn't sent over MPI - float mLastSurfaceNormalLightnessMultiplier; - }; - } - } - - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); - } -} - -#endif // HEMELB_VIS_RAYTRACER_RAYDATAENHANCED_H diff --git a/Code/vis/rayTracer/RayDataNormal.cc b/Code/vis/rayTracer/RayDataNormal.cc deleted file mode 100644 index 7ffd33a68..000000000 --- a/Code/vis/rayTracer/RayDataNormal.cc +++ /dev/null @@ -1,165 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include - -#include "util/Vector3D.h" -#include "vis/DomainStats.h" -#include "vis/rayTracer/RayDataNormal.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - - RayDataNormal::RayDataNormal(int i, int j) : - RayData (i, j) - { - mVelR = 0.0F; - mVelG = 0.0F; - mVelB = 0.0F; - - mStressR = 0.0F; - mStressG = 0.0F; - mStressB = 0.0F; - } - - RayDataNormal::RayDataNormal() - { - - } - - void RayDataNormal::DoUpdateDataForNormalFluidSite( - const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings) - { - float lPalette[3]; - - // update the volume rendering of the velocity flow field - PickColour(iSiteData.velocity * (float) mDomainStats->velocity_threshold_max_inv, lPalette); - - UpdateVelocityColour(iRayLengthInVoxel, lPalette); - - if (iVisSettings.mStressType != lb::ShearStress) - { - // update the volume rendering of the von Mises stress flow field - float lScaledStress = iSiteData.stress * (float) mDomainStats->stress_threshold_max_inv; - - PickColour(lScaledStress, lPalette); - - UpdateStressColour(iRayLengthInVoxel, lPalette); - } - } - - void RayDataNormal::DoUpdateDataForWallSite(const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings, - const util::Vector3D* iWallNormal) - { - DoUpdateDataForNormalFluidSite(iSiteData, iRayDirection, iRayLengthInVoxel, iVisSettings); - } - - void RayDataNormal::DoGetVelocityColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const - { - MakeColourComponent(mVelR * 255.0F, oColour[0]); - MakeColourComponent(mVelG * 255.0F, oColour[1]); - MakeColourComponent(mVelB * 255.0F, oColour[2]); - } - - void RayDataNormal::DoGetStressColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const - { - MakeColourComponent(mStressR, oColour[0]); - MakeColourComponent(mStressG, oColour[1]); - MakeColourComponent(mStressB, oColour[2]); - } - - void RayDataNormal::MakeColourComponent(float value, unsigned char& colour) const - { - colour - = util::NumericalFunctions::enforceBounds((unsigned char) (value - / GetCumulativeLengthInFluid()), - 0, - 255); - } - - void RayDataNormal::DoCombine(const RayDataNormal& iOtherRayData) - { - mVelR += iOtherRayData.mVelR; - mVelG += iOtherRayData.mVelG; - mVelB += iOtherRayData.mVelB; - - mStressR += iOtherRayData.mStressR; - mStressG += iOtherRayData.mStressG; - mStressB += iOtherRayData.mStressB; - } - - void RayDataNormal::UpdateVelocityColour(float iDt, const float iPalette[3]) - { - mVelR += iDt * iPalette[0]; - mVelG += iDt * iPalette[1]; - mVelB += iDt * iPalette[2]; - } - - void RayDataNormal::UpdateStressColour(float iDt, const float iPalette[3]) - { - mStressR += iDt * iPalette[0]; - mStressG += iDt * iPalette[1]; - mStressB += iDt * iPalette[2]; - } - - void RayDataNormal::DoProcessTangentingVessel() - { - } - - MPI_Datatype RayDataNormal::GetMPIType() - { - HEMELB_MPI_TYPE_BEGIN(type, RayDataNormal, 12); - - HEMELB_MPI_TYPE_ADD_MEMBER(i); - HEMELB_MPI_TYPE_ADD_MEMBER(j); - HEMELB_MPI_TYPE_ADD_MEMBER(mLengthBeforeRayFirstCluster); - HEMELB_MPI_TYPE_ADD_MEMBER(mCumulativeLengthInFluid); - HEMELB_MPI_TYPE_ADD_MEMBER(mDensityAtNearestPoint); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressAtNearestPoint); - HEMELB_MPI_TYPE_ADD_MEMBER(mVelR); - HEMELB_MPI_TYPE_ADD_MEMBER(mVelG); - HEMELB_MPI_TYPE_ADD_MEMBER(mVelB); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressR); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressG); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressB); - - HEMELB_MPI_TYPE_END(type, RayDataNormal); - return type; - } - - const DomainStats* RayDataNormal::mDomainStats = NULL; - } - } - - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() - { - MPI_Datatype ret = vis::raytracer::RayDataNormal::GetMPIType(); - HEMELB_MPI_CALL(MPI_Type_commit, (&ret)); - return ret; - } - } -} - diff --git a/Code/vis/rayTracer/RayDataNormal.h b/Code/vis/rayTracer/RayDataNormal.h deleted file mode 100644 index 2f8a36a13..000000000 --- a/Code/vis/rayTracer/RayDataNormal.h +++ /dev/null @@ -1,86 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_RAYDATANORMAL_H -#define HEMELB_VIS_RAYTRACER_RAYDATANORMAL_H - -#include "vis/DomainStats.h" -#include "vis/rayTracer/RayData.h" -#include "vis/VisSettings.h" -#include "util/Vector3D.h" -#include "lb/LbmParameters.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - // NB functions prefixed Do should only be called by the base class - class RayDataNormal : public RayData - { - public: - RayDataNormal(int i, int j); - RayDataNormal(); - - // Used to process the ray data for a normal (non-wall) fluid site - void DoUpdateDataForNormalFluidSite(const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings); - - // Used to process the ray data for wall site - void DoUpdateDataForWallSite(const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings, - const util::Vector3D* iWallNormal); - - // Carries out the merging of the ray data in this - // inherited type, for different segments of the same ray - void DoCombine(const RayDataNormal& iOtherRayData); - - //Obtains the colour representing the velocity ray trace - void DoGetVelocityColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const; - - // Obtains the colour representing the stress ray trace - void DoGetStressColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const; - - void DoProcessTangentingVessel(); - - static MPI_Datatype GetMPIType(); - - // We need this because RayDataNormal uses it for every voxel update - static const DomainStats* mDomainStats; - - private: - void UpdateVelocityColour(float iDt, const float iPalette[3]); - - void UpdateStressColour(float iDt, const float iPalette[3]); - - void MakeColourComponent(float value, unsigned char& colour) const; - - float mVelR, mVelG, mVelB; - float mStressR, mStressG, mStressB; - }; - } - } - - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); - } -} - -#endif // HEMELB_VIS_RAYTRACER_RAYDATANORMAL_H diff --git a/Code/vis/rayTracer/RayTracer.h b/Code/vis/rayTracer/RayTracer.h deleted file mode 100644 index b420a9cd7..000000000 --- a/Code/vis/rayTracer/RayTracer.h +++ /dev/null @@ -1,103 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_RAYTRACER_H -#define HEMELB_VIS_RAYTRACER_RAYTRACER_H - -#include -#include -#include -#include -#include -#include - -#include "constants.h" -#include "debug/Debugger.h" -#include "geometry/LatticeData.h" -#include "lb/LbmParameters.h" -#include "log/Logger.h" -#include "net/IOCommunicator.h" -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" -#include "vis/DomainStats.h" -#include "vis/PixelSet.h" -#include "vis/PixelSetStore.h" -#include "vis/Screen.h" -#include "vis/Viewpoint.h" -#include "vis/VisSettings.h" -#include "vis/XYCoordinates.h" -#include "vis/rayTracer/Cluster.h" -#include "vis/rayTracer/ClusterBuilder.h" -#include "vis/rayTracer/ClusterRayTracer.h" -#include "vis/rayTracer/Ray.h" -#include "vis/rayTracer/RayTracer.h" -#include "vis/rayTracer/SiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - template - class RayTracer : public PixelSetStore > - { - public: - // Constructor and destructor do all the usual stuff. - RayTracer(const geometry::LatticeData* iLatDat, - const DomainStats* iDomainStats, - Screen* iScreen, - Viewpoint* iViewpoint, - VisSettings* iVisSettings) : - mClusterBuilder(iLatDat, iLatDat->GetLocalRank()), mLatDat(iLatDat), mDomainStats(iDomainStats), - mScreen(iScreen), mViewpoint(iViewpoint), mVisSettings(iVisSettings) - { - mClusterBuilder.BuildClusters(); - } - - ~RayTracer() - { - } - - // Render the current state into an image. - PixelSet* Render(const lb::MacroscopicPropertyCache& propertyCache) - { - PixelSet* pixels = - PixelSetStore >::GetUnusedPixelSet(); - pixels->Clear(); - - ClusterRayTracer lClusterRayTracer(*mViewpoint, - *mScreen, - *mDomainStats, - *mVisSettings, - *mLatDat, - propertyCache); - - for (unsigned int clusterId = 0; clusterId < mClusterBuilder.GetClusters().size(); clusterId++) - { - lClusterRayTracer.RenderCluster(mClusterBuilder.GetClusters()[clusterId], *pixels); - } - - return pixels; - } - - private: - ClusterBuilder mClusterBuilder; - const geometry::LatticeData* mLatDat; - - const DomainStats* mDomainStats; - Screen* mScreen; - Viewpoint* mViewpoint; - VisSettings* mVisSettings; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_RAYTRACER_H diff --git a/Code/vis/rayTracer/SiteData.h b/Code/vis/rayTracer/SiteData.h deleted file mode 100644 index 47bffd222..000000000 --- a/Code/vis/rayTracer/SiteData.h +++ /dev/null @@ -1,31 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_RAYTRACER_SITEDATA_H -#define HEMELB_VIS_RAYTRACER_SITEDATA_H - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - //Stores the data about an individual voxel - struct SiteData_t - { - public: - float density; - float velocity; - float stress; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_SITEDATA_H diff --git a/Code/vis/streaklineDrawer/NeighbouringProcessor.cc b/Code/vis/streaklineDrawer/NeighbouringProcessor.cc deleted file mode 100644 index bcd07b2e6..000000000 --- a/Code/vis/streaklineDrawer/NeighbouringProcessor.cc +++ /dev/null @@ -1,170 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include -#include - -#include "constants.h" -#include "vis/streaklineDrawer/NeighbouringProcessor.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - - NeighbouringProcessor::NeighbouringProcessor() - { - } - - NeighbouringProcessor::NeighbouringProcessor(proc_t neighbourRankIn) : - neighbourRank(neighbourRankIn) - { - } - - void NeighbouringProcessor::AddParticleToSend(const Particle& particle) - { - particlesToSend.push_back(particle); - } - - bool NeighbouringProcessor::ParticlesToBeRetrieved() - { - return (particlesToReceive.size() > 0); - } - - const Particle& NeighbouringProcessor::PopNextReceivedParticle() - { - const Particle& lParticle = particlesToReceive.back(); - particlesToReceive.pop_back(); - return lParticle; - } - - void NeighbouringProcessor::ExchangeParticleCounts(net::Net& net) - { - numberOfParticlesToSend = particlesToSend.size(); - net.RequestSendR(numberOfParticlesToSend, neighbourRank); - net.RequestReceiveR(numberOfParticlesToReceive, neighbourRank); - } - - void NeighbouringProcessor::ClearParticleSendingList() - { - particlesToSend.clear(); - } - - void NeighbouringProcessor::ExchangeParticles(net::Net& net) - { - if (numberOfParticlesToReceive > 0) - { - particlesToReceive.resize(numberOfParticlesToReceive); - - net.RequestReceiveV(particlesToReceive, - neighbourRank); - } - - if (particlesToSend.size() > 0) - { - net.RequestSendV(particlesToSend, neighbourRank); //Request - } - } - - void NeighbouringProcessor::AddSiteToRequestVelocityDataFor(site_t siteI, - site_t siteJ, - site_t siteK) - { - siteCoordsRequestedByThisCore.push_back(util::Vector3D(siteI, siteJ, siteK)); - } - - void NeighbouringProcessor::ExchangeSiteIdCounts(net::Net& net) - { - numberOfSitesRequestedByThisCore = siteCoordsRequestedByThisCore.size(); - - net.RequestReceiveR(numberOfSiteBeingRequestedByNeighbour, neighbourRank); - net.RequestSendR(numberOfSitesRequestedByThisCore, neighbourRank); - } - - void NeighbouringProcessor::ExchangeSiteIds(net::Net& net) - { - if (numberOfSiteBeingRequestedByNeighbour > 0) - { - siteCoordsRequestedByNeighbour.resize(numberOfSiteBeingRequestedByNeighbour); - velocityFieldDataForNeighbour.resize(numberOfSiteBeingRequestedByNeighbour); - - net.RequestReceive(&siteCoordsRequestedByNeighbour[0], - (int) numberOfSiteBeingRequestedByNeighbour, - neighbourRank); - } - - if (numberOfSitesRequestedByThisCore > 0) - { - velocityFieldDataFromNeighbour.resize(numberOfSitesRequestedByThisCore); - - net.RequestSendV(siteCoordsRequestedByThisCore, - neighbourRank); - } - } - - void NeighbouringProcessor::ExchangeVelocitiesForRequestedSites(net::Net& net) - { - if (numberOfSitesRequestedByThisCore > 0) - { - velocityFieldDataFromNeighbour.resize(numberOfSitesRequestedByThisCore); - - net.RequestReceiveV(velocityFieldDataFromNeighbour, - neighbourRank); - } - - if (numberOfSiteBeingRequestedByNeighbour > 0) - { - velocityFieldDataForNeighbour.resize(numberOfSiteBeingRequestedByNeighbour); - - net.RequestSendV(velocityFieldDataForNeighbour, - neighbourRank); - } - } - - site_t NeighbouringProcessor::GetNumberOfSitesRequestedByNeighbour() const - { - return siteCoordsRequestedByNeighbour.size(); - } - - const util::Vector3D& NeighbouringProcessor::GetReceivedVelocityField(const site_t receivedIndex) const - { - return velocityFieldDataFromNeighbour[receivedIndex]; - } - - const util::Vector3D& NeighbouringProcessor::GetSiteCoordsBeingRequestedByNeighbour(const site_t receivedIndex) const - { - return siteCoordsRequestedByNeighbour[receivedIndex]; - } - - void NeighbouringProcessor::SetVelocityFieldToSend(const site_t sendIndex, - const util::Vector3D& velocityFieldToSend) - { - velocityFieldDataForNeighbour[sendIndex] = velocityFieldToSend; - } - - site_t NeighbouringProcessor::GetNumberOfSitesRequestedByThisCore() const - { - return siteCoordsRequestedByThisCore.size(); - } - - const util::Vector3D& NeighbouringProcessor::GetSendingSiteCoorinates(site_t sendIndex) const - { - return siteCoordsRequestedByThisCore[sendIndex]; - } - - void NeighbouringProcessor::ClearListOfRequestedSites() - { - siteCoordsRequestedByThisCore.clear(); - } - - } - } -} diff --git a/Code/vis/streaklineDrawer/NeighbouringProcessor.h b/Code/vis/streaklineDrawer/NeighbouringProcessor.h deleted file mode 100644 index eece809a5..000000000 --- a/Code/vis/streaklineDrawer/NeighbouringProcessor.h +++ /dev/null @@ -1,80 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_NEIGHBOURINGPROCESSOR_H -#define HEMELB_VIS_STREAKLINEDRAWER_NEIGHBOURINGPROCESSOR_H - -#include - -#include "net/net.h" -#include "util/Vector3D.h" -#include "vis/streaklineDrawer/Particle.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - class NeighbouringProcessor - { - public: - // Constructors - NeighbouringProcessor(); - NeighbouringProcessor(proc_t iID); - - // Functions for communicating particles. - void AddParticleToSend(const Particle& iParticle); - bool ParticlesToBeRetrieved(); - const Particle& PopNextReceivedParticle(); - void ClearParticleSendingList(); - - void ExchangeParticleCounts(net::Net& net); - void ExchangeParticles(net::Net& net); - - // Functions for communicating velocity data. - void AddSiteToRequestVelocityDataFor(site_t, site_t, site_t); - site_t GetNumberOfSitesRequestedByNeighbour() const; - const util::Vector3D& GetReceivedVelocityField(const site_t receivedIndex) const; - const util::Vector3D - & GetSiteCoordsBeingRequestedByNeighbour(const site_t receivedIndex) const; - void SetVelocityFieldToSend(const site_t sendIndex, - const util::Vector3D& velocityFieldToSend); - - const util::Vector3D& GetSendingSiteCoorinates(site_t sendIndex) const; - site_t GetNumberOfSitesRequestedByThisCore() const; - void ClearListOfRequestedSites(); - - void ExchangeSiteIdCounts(net::Net& net); - void ExchangeSiteIds(net::Net& net); - void ExchangeVelocitiesForRequestedSites(net::Net& net); - - private: - site_t numberOfParticlesToSend; - std::vector particlesToSend; - - site_t numberOfParticlesToReceive; - std::vector particlesToReceive; - - site_t numberOfSiteBeingRequestedByNeighbour; - site_t numberOfSitesRequestedByThisCore; - - std::vector > siteCoordsRequestedByThisCore; - std::vector > siteCoordsRequestedByNeighbour; - - std::vector > velocityFieldDataForNeighbour; - std::vector > velocityFieldDataFromNeighbour; - - proc_t neighbourRank; - }; - } - } -} - -#endif // HEMELB_VIS_STREAKLINEDRAWER_NEIGHBOURINGPROCESSOR_H diff --git a/Code/vis/streaklineDrawer/Particle.cc b/Code/vis/streaklineDrawer/Particle.cc deleted file mode 100644 index 2d3c83909..000000000 --- a/Code/vis/streaklineDrawer/Particle.cc +++ /dev/null @@ -1,50 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "net/mpi.h" -#include "vis/streaklineDrawer/Particle.h" -#include "constants.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - Particle::Particle() : - position(NO_VALUE), velocity(NO_VALUE), vel(NO_VALUE), inletID() - { - - } - - Particle::Particle(float iX, float iY, float iZ, unsigned int iInletId) : - position(iX, iY, iZ), velocity(0), vel(0), inletID(iInletId) - { - } - } - } - - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() - { - HEMELB_MPI_TYPE_BEGIN(type, vis::streaklinedrawer::Particle, 3); - - HEMELB_MPI_TYPE_ADD_MEMBER(position); - HEMELB_MPI_TYPE_ADD_MEMBER(vel); - HEMELB_MPI_TYPE_ADD_MEMBER(inletID); - - HEMELB_MPI_TYPE_END(type, vis::streaklinedrawer::Particle); - - HEMELB_MPI_CALL(MPI_Type_commit, (&type)); - return type; - } - } -} diff --git a/Code/vis/streaklineDrawer/Particle.h b/Code/vis/streaklineDrawer/Particle.h deleted file mode 100644 index 95b96bfb5..000000000 --- a/Code/vis/streaklineDrawer/Particle.h +++ /dev/null @@ -1,43 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_PARTICLE_H -#define HEMELB_VIS_STREAKLINEDRAWER_PARTICLE_H - -#include "util/Vector3D.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - struct Particle - { - public: - Particle(); - - Particle(float iX, float iY, float iZ, unsigned int iInletId); - - util::Vector3D position; - util::Vector3D velocity; - float vel; - unsigned int inletID; - }; - - } - } - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); - } -} - -#endif // HEMELB_VIS_STREAKLINEDRAWER_PARTICLE_H diff --git a/Code/vis/streaklineDrawer/ParticleManager.cc b/Code/vis/streaklineDrawer/ParticleManager.cc deleted file mode 100644 index 05b710a7f..000000000 --- a/Code/vis/streaklineDrawer/ParticleManager.cc +++ /dev/null @@ -1,124 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include - -#include "vis/streaklineDrawer/ParticleManager.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - ParticleManager::ParticleManager(std::map& iNeighbouringProcessors) : - neighbouringProcessors(iNeighbouringProcessors) - { - } - - void ParticleManager::AddParticle(const Particle& iParticle) - { - particles.push_back(iParticle); - } - - std::vector& ParticleManager::GetParticles() - { - return particles; - } - - size_t ParticleManager::GetNumberOfLocalParticles() const - { - return particles.size(); - } - - void ParticleManager::DeleteParticle(site_t iIndex) - { - assert(particles.size() > static_cast (iIndex)); - - //Move the particle at the end to position - particles[iIndex] = particles.back(); - - //Delete the now duplicated particle at the end - particles.pop_back(); - } - - void ParticleManager::DeleteAll() - { - particles.clear(); - } - - void ParticleManager::ProcessParticleMovement() - { - for (unsigned int i = 0; i < GetNumberOfLocalParticles(); i++) - { - // particle coords updating (dt = 1) - particles[i].position += particles[i].velocity; - } - } - - // Communicate the particles' current state to other processors. - void ParticleManager::CommunicateParticles(net::Net& streakNet, - const geometry::LatticeData& latticeData, - VelocityField& velocityField) - { - unsigned int particles_temp = GetNumberOfLocalParticles(); - - proc_t thisRank = streakNet.Rank(); - - for (int n = (int) (particles_temp - 1); n >= 0; n--) - { - VelocitySiteData* siteVelocityData = - velocityField.GetVelocitySiteData(latticeData, - util::Vector3D(particles[n].position)); - - // TODO can we get rid of the first test? - if (siteVelocityData == NULL || siteVelocityData->proc_id == -1) - { - continue; - } - else if (thisRank == siteVelocityData->proc_id) - { - continue; - } - - neighbouringProcessors[siteVelocityData->proc_id].AddParticleToSend(particles[n]); - DeleteParticle(n); - } - - for (std::map::iterator proc = - neighbouringProcessors.begin(); proc != neighbouringProcessors.end(); ++proc) - { - (*proc).second.ExchangeParticleCounts(streakNet); - } - streakNet.Dispatch(); - - for (std::map::iterator proc = - neighbouringProcessors.begin(); proc != neighbouringProcessors.end(); ++proc) - { - (*proc).second.ExchangeParticles(streakNet); - } - - streakNet.Dispatch(); - - for (std::map::iterator proc = - neighbouringProcessors.begin(); proc != neighbouringProcessors.end(); ++proc) - { - NeighbouringProcessor& neighbourProc = (*proc).second; - neighbourProc.ClearParticleSendingList(); - - while (neighbourProc.ParticlesToBeRetrieved()) - { - AddParticle(neighbourProc.PopNextReceivedParticle()); - } - } - } - - } - } -} diff --git a/Code/vis/streaklineDrawer/ParticleManager.h b/Code/vis/streaklineDrawer/ParticleManager.h deleted file mode 100644 index 48e958f41..000000000 --- a/Code/vis/streaklineDrawer/ParticleManager.h +++ /dev/null @@ -1,63 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_PARTICLEMANAGER_H -#define HEMELB_VIS_STREAKLINEDRAWER_PARTICLEMANAGER_H - -#include -#include - -#include "constants.h" -#include "net/mpi.h" - -#include "geometry/LatticeData.h" -#include "net/IOCommunicator.h" - -#include "vis/streaklineDrawer/NeighbouringProcessor.h" -#include "vis/streaklineDrawer/Particle.h" -#include "vis/streaklineDrawer/VelocityField.h" -#include "vis/streaklineDrawer/VelocitySiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - class ParticleManager - { - public: - // Constructor - ParticleManager(std::map& iNeighbouringProcessors); - - // Functions for manipulating the particle store - void AddParticle(const Particle& iParticle); - std::vector& GetParticles(); - size_t GetNumberOfLocalParticles() const; - void DeleteParticle(site_t iIndex); - void DeleteAll(); - - // Function for updating the particles' positions. - void ProcessParticleMovement(); - - // Function for moving the particles between cores. - void CommunicateParticles(net::Net& streakNet, - const geometry::LatticeData& iLatDat, - VelocityField& iVelocityField); - - private: - std::vector particles; - std::map& neighbouringProcessors; - - }; - } - } -} - -#endif // HEMELB_VIS_STREAKLINEDRAWER_PARTICLEMANAGER_H diff --git a/Code/vis/streaklineDrawer/StreakPixel.cc b/Code/vis/streaklineDrawer/StreakPixel.cc deleted file mode 100644 index 226f79629..000000000 --- a/Code/vis/streaklineDrawer/StreakPixel.cc +++ /dev/null @@ -1,24 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "vis/streaklineDrawer/StreakPixel.h" - -namespace hemelb -{ - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() - { - MPI_Datatype ret = vis::streaklinedrawer::StreakPixel::GetMPIType(); - MPI_Type_commit(&ret); - return ret; - } - } -} diff --git a/Code/vis/streaklineDrawer/StreakPixel.h b/Code/vis/streaklineDrawer/StreakPixel.h deleted file mode 100644 index 5eb54638f..000000000 --- a/Code/vis/streaklineDrawer/StreakPixel.h +++ /dev/null @@ -1,104 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_STREAKPIXEL_H -#define HEMELB_VIS_STREAKLINEDRAWER_STREAKPIXEL_H - -#include "log/Logger.h" -#include "vis/BasicPixel.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - class StreakPixel : public BasicPixel - { - public: - StreakPixel() - { - - } - - StreakPixel(int i, int j, float particleVelocity, float particleZ, int particleInlet) : - BasicPixel(i, j), particle_vel(particleVelocity), particle_z(particleZ), particle_inlet_id(particleInlet) - { - - } - - void Combine(const StreakPixel& inPixel) - { - if (inPixel.particle_z < particle_z) - { - particle_z = inPixel.particle_z; - particle_vel = inPixel.particle_vel; - particle_inlet_id = inPixel.particle_inlet_id; - } - } - - float GetParticleVelocity() const - { - return particle_vel; - } - - /** - * Produces an MPI Datatype object but doesn't commit it or manage its memory. - * @return - */ - static MPI_Datatype GetMPIType() - { - const int typeCount = 5; - int blocklengths[typeCount] = { 1, 1, 1, 1, 1 }; - - MPI_Datatype types[typeCount] = { MPI_INT, MPI_INT, MPI_FLOAT, MPI_FLOAT, MPI_INT }; - - StreakPixel example; - - MPI_Aint displacements[typeCount]; - - MPI_Get_address(&example.i, &displacements[0]); - MPI_Get_address(&example.j, &displacements[1]); - MPI_Get_address(&example.particle_vel, &displacements[2]); - MPI_Get_address(&example.particle_z, &displacements[3]); - MPI_Get_address(&example.particle_inlet_id, &displacements[4]); - - for (int ii = typeCount - 1; ii >= 0; --ii) - { - displacements[ii] -= displacements[0]; - } - - MPI_Datatype ret; - - MPI_Type_create_struct(typeCount, blocklengths, displacements, types, &ret); - - return ret; - } - - void LogDebuggingInformation() const - { - log::Logger::Log("Streak pixel at (%i,%i) with " - "(source inlet, velocity, z) = (%d, %f, %f)", GetI(), GetJ(), particle_inlet_id, particle_vel, particle_z); - } - - private: - float particle_vel; - float particle_z; - int particle_inlet_id; - }; - } - } - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); - } -} - -#endif /* HEMELB_VIS_STREAKLINEDRAWER_STREAKPIXEL_H */ diff --git a/Code/vis/streaklineDrawer/StreaklineDrawer.cc b/Code/vis/streaklineDrawer/StreaklineDrawer.cc deleted file mode 100644 index d56ade036..000000000 --- a/Code/vis/streaklineDrawer/StreaklineDrawer.cc +++ /dev/null @@ -1,354 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include -#include -#include - -#include "geometry/BlockTraverser.h" -#include "geometry/SiteTraverser.h" -#include "util/utilityFunctions.h" -#include "vis/Control.h" -#include "vis/streaklineDrawer/StreaklineDrawer.h" -#include "vis/streaklineDrawer/StreakPixel.h" -#include "vis/XYCoordinates.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - // Constructor, populating fields from lattice data objects. - StreaklineDrawer::StreaklineDrawer(const geometry::LatticeData& iLatDat, - const Screen& iScreen, - const Viewpoint& iViewpoint, - const VisSettings& iVisSettings, - const lb::MacroscopicPropertyCache& propertyCache, - const net::MpiCommunicator& comms) : - latDat(iLatDat), screen(iScreen), viewpoint(iViewpoint), visSettings(iVisSettings), - // propertyCache(propertyCache), - particleManager(neighbouringProcessors), - velocityField(comms.Rank(), neighbouringProcessors, propertyCache), - streakNet(new net::Net(comms)) - { - velocityField.BuildVelocityField(iLatDat); - ChooseSeedParticles(); - } - - // Destructor - StreaklineDrawer::~StreaklineDrawer() - { - delete streakNet; - } - - // Reset the streakline drawer. - void StreaklineDrawer::Restart() - { - particleManager.DeleteAll(); - } - - // Create streakline particles and move them. - void StreaklineDrawer::ProgressStreaklines(unsigned long time_steps, unsigned long total_time_steps) - { - // Set the particle creation period to be every time step, unless there are >=10000 - // timesteps - unsigned int particle_creation_period = - util::NumericalFunctions::max(1, (unsigned int) (total_time_steps / 5000)); - - int timestepsBetweenStreaklinesRounded = (int) (0.5F - + (float) total_time_steps / visSettings.streaklines_per_simulation); - - if ((float) (time_steps % timestepsBetweenStreaklinesRounded) - <= (visSettings.streakline_length / 100.0F) - * ((float) total_time_steps / visSettings.streaklines_per_simulation) - && time_steps % particle_creation_period == 0) - { - CreateParticlesFromSeeds(); - } - - velocityField.InvalidateAllCalculatedVelocities(); - - // Decide which sites we will need velocity data for in order to move particles - WorkOutVelocityDataNeededForParticles(); - - // Communicate this with other processors. - CommunicateSiteIds(); - - // Recalculate velocities at the sites requested from this rank. - UpdateVelocityFieldForCommunicatedSites(); - - // Communicate the velocities back to the sites that requested them. - CommunicateVelocities(); - - // Update our local velocity field is sufficient to update all particles and prune - // those that are stationary. - UpdateVelocityFieldForAllParticlesAndPrune(); - - // Process the particles' movement. - particleManager.ProcessParticleMovement(); - - // Communicate any particles that have crossed into the territory of another rank. - particleManager.CommunicateParticles(*streakNet, latDat, velocityField); - } - - // Render the streaklines - PixelSet* StreaklineDrawer::Render() - { - int pixels_x = screen.GetPixelsX(); - int pixels_y = screen.GetPixelsY(); - - const std::vector& particles = particleManager.GetParticles(); - - PixelSet* set = GetUnusedPixelSet(); - set->Clear(); - - for (unsigned int n = 0; n < particles.size(); n++) - { - util::Vector3D p1 = particles[n].position - util::Vector3D(latDat.GetSiteDimensions() / 2); - - util::Vector3D p2 = viewpoint.Project(p1); - - XYCoordinates x = screen.TransformScreenToPixelCoordinates(XYCoordinates(p2.x, p2.y)); - - if (! (x.x < 0 || x.x >= pixels_x || x.y < 0 || x.y >= pixels_y)) - { - StreakPixel pixel(x.x, x.y, particles[n].vel, p2.z, particles[n].inletID); - set->AddPixel(pixel); - } - } - - return set; - } - - void StreaklineDrawer::ChooseSeedParticles() - { - site_t inlet_sites = 0; - - geometry::BlockTraverser blockTraverser(latDat); - do - { - const geometry::Block& block = blockTraverser.GetCurrentBlockData(); - - if (block.IsEmpty()) - { - continue; - } - - geometry::SiteTraverser siteTraverser(latDat); - do - { - if (this->streakNet->Rank() - != block.GetProcessorRankForSite(siteTraverser.GetCurrentIndex())) - { - continue; - } - - const geometry::Site site = - latDat.GetSite(block.GetLocalContiguousIndexForSite(siteTraverser.GetCurrentIndex())); - - // if the lattice site is not an inlet - if (site.GetSiteType() != geometry::INLET_TYPE) - { - continue; - } - ++inlet_sites; - - // TODO this is a problem on multiple cores. - if (inlet_sites % 50 != 0) - { - continue; - } - - particleSeeds.push_back(Particle(static_cast(blockTraverser.GetX() * blockTraverser.GetBlockSize() - + siteTraverser.GetX()), - static_cast(blockTraverser.GetY() * blockTraverser.GetBlockSize() - + siteTraverser.GetY()), - static_cast(blockTraverser.GetZ() * blockTraverser.GetBlockSize() - + siteTraverser.GetZ()), - site.GetIoletId())); - - } - while (siteTraverser.TraverseOne()); - } - while (blockTraverser.TraverseOne()); - } - - // Create seed particles to begin the streaklines. - void StreaklineDrawer::CreateParticlesFromSeeds() - { - for (unsigned int n = 0; n < particleSeeds.size(); n++) - { - particleManager.AddParticle(particleSeeds[n]); - } - } - - void StreaklineDrawer::WorkOutVelocityDataNeededForParticles() - { - std::vector &particles = particleManager.GetParticles(); - - // Note that we iterate through the array back to front because at the end of this - // loop we delete a particle. Iterating back to front ensures that we end up - // visiting each particle. - for (int n = (int) particles.size() - 1; n >= 0; --n) - { - Particle& particle = particles[n]; - - for (int unitGridI = 0; unitGridI <= 1; ++unitGridI) - { - site_t neighbourI = (site_t) particle.position.x + unitGridI; - - for (int unitGridJ = 0; unitGridJ <= 1; ++unitGridJ) - { - site_t neighbourJ = (site_t) particle.position.y + unitGridJ; - - for (int unitGridK = 0; unitGridK <= 1; ++unitGridK) - { - site_t neighbourK = (site_t) particle.position.z + unitGridK; - - proc_t sourceProcessor; - - if (velocityField.NeededFromNeighbour(util::Vector3D(neighbourI, neighbourJ, neighbourK), - latDat, - &sourceProcessor)) - { - neighbouringProcessors[sourceProcessor].AddSiteToRequestVelocityDataFor(neighbourI, - neighbourJ, - neighbourK); - } - } - } - } - } - } - - void StreaklineDrawer::UpdateVelocityFieldForAllParticlesAndPrune() - { - std::vector &particles = particleManager.GetParticles(); - - // Note that we iterate through the array back to front because at the end of this - // loop we delete a particle. Iterating back to front ensures that we end up - // visiting each particle. - for (int n = (int) particles.size() - 1; n >= 0; --n) - { - util::Vector3D localVelocityField[2][2][2]; - - velocityField.GetVelocityFieldAroundPoint(util::Vector3D(particles[n].position), - latDat, - localVelocityField); - - util::Vector3D interp_v = velocityField.InterpolateVelocityForPoint(particles[n].position, - localVelocityField); - - float vel = interp_v.Dot(interp_v); - - if (vel > 1.0F) - { - particles[n].vel = 1.0F; - particles[n].velocity = interp_v * float(1.0 / sqrtf(vel)); - } - else if (vel > 1.0e-8) - { - particles[n].vel = sqrtf(vel); - particles[n].velocity = interp_v; - } - else - { - particleManager.DeleteParticle(n); - } - } - } - - void StreaklineDrawer::UpdateVelocityFieldForCommunicatedSites() - { - for (std::map::const_iterator neighProc = neighbouringProcessors.begin(); - neighProc != neighbouringProcessors.end(); ++neighProc) - { - const NeighbouringProcessor& proc = (*neighProc).second; - - for (site_t sendingVelocityIndex = 0; sendingVelocityIndex < proc.GetNumberOfSitesRequestedByNeighbour(); - sendingVelocityIndex++) - { - const util::Vector3D& siteCoords = - proc.GetSiteCoordsBeingRequestedByNeighbour(sendingVelocityIndex); - - velocityField.UpdateLocalField(siteCoords, latDat); - } - } - } - - // Communicate site ids to other processors. - void StreaklineDrawer::CommunicateSiteIds() - { - for (std::map::iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); ++proc) - { - (*proc).second.ExchangeSiteIdCounts(*streakNet); - } - - streakNet->Dispatch(); - - for (std::map::iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); proc++) - { - (*proc).second.ExchangeSiteIds(*streakNet); - } - - streakNet->Dispatch(); - } - - // Communicate velocities to other processors. - void StreaklineDrawer::CommunicateVelocities() - { - - for (std::map::iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); ++proc) - { - (*proc).second.ExchangeVelocitiesForRequestedSites(*streakNet); - } - - for (std::map::iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); ++proc) - { - NeighbouringProcessor& neighbourProc = (*proc).second; - - for (site_t n = 0; n < neighbourProc.GetNumberOfSitesRequestedByNeighbour(); ++n) - { - const util::Vector3D siteCoords = neighbourProc.GetSiteCoordsBeingRequestedByNeighbour(n); - - const VelocitySiteData* velocityDataForSite = velocityField.GetVelocitySiteData(latDat, siteCoords); - - neighbourProc.SetVelocityFieldToSend(n, velocityDataForSite->velocity); - } - } - - streakNet->Dispatch(); - - for (std::map::const_iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); ++proc) - { - const NeighbouringProcessor& neighbourProc = (*proc).second; - - for (site_t n = 0; n < neighbourProc.GetNumberOfSitesRequestedByThisCore(); n++) - { - const util::Vector3D &coords = neighbourProc.GetSendingSiteCoorinates(n); - velocityField.GetVelocitySiteData(latDat, coords)->velocity = neighbourProc.GetReceivedVelocityField(n); - } - } - - for (std::map::iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); ++proc) - { - (*proc).second.ClearListOfRequestedSites(); - } - } - - } - } -} diff --git a/Code/vis/streaklineDrawer/StreaklineDrawer.h b/Code/vis/streaklineDrawer/StreaklineDrawer.h deleted file mode 100644 index ebdecd0eb..000000000 --- a/Code/vis/streaklineDrawer/StreaklineDrawer.h +++ /dev/null @@ -1,93 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_STREAKLINEDRAWER_H -#define HEMELB_VIS_STREAKLINEDRAWER_STREAKLINEDRAWER_H - -#include -#include - -#include "constants.h" -#include "net/mpi.h" - -#include "debug/Debugger.h" -#include "geometry/LatticeData.h" -#include "net/IOCommunicator.h" -#include "vis/PixelSet.h" -#include "vis/PixelSetStore.h" -#include "vis/streaklineDrawer/NeighbouringProcessor.h" -#include "vis/streaklineDrawer/ParticleManager.h" -#include "vis/streaklineDrawer/VelocityField.h" -#include "vis/streaklineDrawer/VelocitySiteData.h" -#include "vis/Screen.h" -#include "vis/streaklineDrawer/StreakPixel.h" -#include "vis/Viewpoint.h" -#include "vis/VisSettings.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - /** - * Class that controls the drawing of streaklines - lines that trace - * the path of an imaginary particle were it dropped into the fluid. - */ - class StreaklineDrawer : public PixelSetStore > - { - public: - // Constructor and destructor. - StreaklineDrawer(const geometry::LatticeData& iLatDat, - const Screen& iScreen, - const Viewpoint& iViewpoint, - const VisSettings& iVisSettings, - const lb::MacroscopicPropertyCache& propertyCache, - const net::MpiCommunicator& comms); - ~StreaklineDrawer(); - - // Method to reset streakline drawer - void Restart(); - - // Drawing methods. - void ProgressStreaklines(unsigned long time_steps, unsigned long total_time_steps); - PixelSet* Render(); - - private: - // Function for updating the velocity field and the particles in it. - void UpdateVelocityFieldForAllParticlesAndPrune(); - void UpdateVelocityFieldForCommunicatedSites(); - - // Private functions for the creation / deletion of particles. - void ChooseSeedParticles(); - void CreateParticlesFromSeeds(); - - // Private functions for inter-proc communication. - void CommunicateSiteIds(); - void CommunicateVelocities(); - void WorkOutVelocityDataNeededForParticles(); - - const geometry::LatticeData& latDat; - const Screen& screen; - const Viewpoint& viewpoint; - const VisSettings& visSettings; - //const lb::MacroscopicPropertyCache& propertyCache; - - std::map neighbouringProcessors; - ParticleManager particleManager; - VelocityField velocityField; - - std::vector particleSeeds; - net::Net* streakNet; - }; - } - } -} - -#endif // HEMELB_VIS_STREAKLINEDRAWER_STREAKLINEDRAWER_H diff --git a/Code/vis/streaklineDrawer/VelocityField.cc b/Code/vis/streaklineDrawer/VelocityField.cc deleted file mode 100644 index 59eb4c0ce..000000000 --- a/Code/vis/streaklineDrawer/VelocityField.cc +++ /dev/null @@ -1,350 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include -#include -#include - -#include "debug/Debugger.h" -#include "geometry/BlockTraverser.h" -#include "geometry/SiteTraverser.h" -#include "vis/streaklineDrawer/VelocityField.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - VelocityField::VelocityField(proc_t localRank_, - std::map& neighbouringProcessorsIn, - const lb::MacroscopicPropertyCache& propertyCache) : - counter(0), localRank(localRank_), neighbouringProcessors(neighbouringProcessorsIn), propertyCache(propertyCache) - { - } - - void VelocityField::BuildVelocityField(const geometry::LatticeData& latDat) - { - velocityField.resize(latDat.GetBlockCount()); - - // Iterate over each block with some sites on this rank. - geometry::BlockTraverser blockTraverser(latDat); - do - { - const geometry::Block& block = blockTraverser.GetCurrentBlockData(); - - if (block.IsEmpty()) - { - continue; - } - - geometry::SiteTraverser siteTraverser(latDat); - do - { - // Only interested if the site lives on this rank. - if (localRank != block.GetProcessorRankForSite(siteTraverser.GetCurrentIndex())) - { - continue; - } - - // Calculate the bounds of the unit cube around the current site (within the lattice) - const site_t startI = util::NumericalFunctions::max(0, - blockTraverser.GetX() - * blockTraverser.GetBlockSize() - + siteTraverser.GetX() - 1); - - const site_t startJ = util::NumericalFunctions::max(0, - blockTraverser.GetY() - * blockTraverser.GetBlockSize() - + siteTraverser.GetY() - 1); - - const site_t startK = util::NumericalFunctions::max(0, - blockTraverser.GetZ() - * blockTraverser.GetBlockSize() - + siteTraverser.GetZ() - 1); - - const site_t endI = util::NumericalFunctions::min(latDat.GetSiteDimensions().x - 1, - blockTraverser.GetX() - * blockTraverser.GetBlockSize() - + siteTraverser.GetX() + 1); - - const site_t endJ = util::NumericalFunctions::min(latDat.GetSiteDimensions().y - 1, - blockTraverser.GetY() - * blockTraverser.GetBlockSize() - + siteTraverser.GetY() + 1); - - const site_t endK = util::NumericalFunctions::min(latDat.GetSiteDimensions().z - 1, - blockTraverser.GetZ() - * blockTraverser.GetBlockSize() - + siteTraverser.GetZ() + 1); - - // Iterate over the sites in the unit cube. - for (site_t neighbourI = startI; neighbourI <= endI; neighbourI++) - { - for (site_t neighbourJ = startJ; neighbourJ <= endJ; neighbourJ++) - { - for (site_t neighbourK = startK; neighbourK <= endK; neighbourK++) - { - // Get the rank that the neighbour lives on. - const proc_t neigh_proc_id = latDat.GetProcIdFromGlobalCoords(util::Vector3D(neighbourI, - neighbourJ, - neighbourK)); - - // If we have data for it, we should initialise a block in the velocity field - // for the neighbour site. - if (neigh_proc_id == BIG_NUMBER2) - { - continue; - } - - InitializeVelocityFieldBlock(latDat, - util::Vector3D(neighbourI, neighbourJ, neighbourK), - neigh_proc_id); - - // If the neighbour is on this rank, ignore it. - if (localRank == neigh_proc_id) - { - continue; - } - - if (neighbouringProcessors.count(neigh_proc_id) == 0) - { - NeighbouringProcessor newProc(neigh_proc_id); - neighbouringProcessors[neigh_proc_id] = newProc; - } - } - } - } - } - while (siteTraverser.TraverseOne()); - } - while (blockTraverser.TraverseOne()); - - // Iterate over the blocks, updating the value of the counter variable wherever - // there is velocity field data. - for (site_t block = 0; block < latDat.GetBlockCount(); block++) - { - if (velocityField[block].empty()) - { - continue; - } - - if (latDat.GetBlock(block).IsEmpty()) - { - continue; - } - - // Update the site id on each velocity field unit as required. - for (site_t localSiteId = 0; localSiteId < latDat.GetSitesPerBlockVolumeUnit(); localSiteId++) - { - velocityField[block][localSiteId].site_id = - latDat.GetBlock(block).GetLocalContiguousIndexForSite(localSiteId); - } - } - } - - bool VelocityField::BlockContainsData(size_t blockNumber) const - { - return !velocityField[blockNumber].empty(); - } - - VelocitySiteData& VelocityField::GetSiteData(site_t blockNumber, site_t siteNumber) - { - return velocityField[blockNumber][siteNumber]; - } - - // Returns the velocity site data for a given index, or NULL if the index isn't valid / has - // no data. - VelocitySiteData* VelocityField::GetVelocitySiteData(const geometry::LatticeData& latDat, - const util::Vector3D& location) - { - if (!latDat.IsValidLatticeSite(location)) - { - return NULL; - } - - util::Vector3D blockCoords, siteCoords; - latDat.GetBlockAndLocalSiteCoords(location, blockCoords, siteCoords); - - site_t block_id = latDat.GetBlockIdFromBlockCoords(blockCoords); - - if (!BlockContainsData(static_cast(block_id))) - { - return NULL; - } - - site_t site_id = latDat.GetLocalSiteIdFromLocalSiteCoords(siteCoords); - - return &GetSiteData(block_id, site_id); - } - - // Function to initialise the velocity field at given coordinates. - void VelocityField::InitializeVelocityFieldBlock(const geometry::LatticeData& latDat, - const util::Vector3D location, - const proc_t proc_id) - { - util::Vector3D blockCoords, siteCoords; - latDat.GetBlockAndLocalSiteCoords(location, blockCoords, siteCoords); - - site_t blockId = latDat.GetBlockIdFromBlockCoords(blockCoords); - - if (!BlockContainsData(blockId)) - { - for (site_t localSiteId = 0; localSiteId < latDat.GetSitesPerBlockVolumeUnit(); ++localSiteId) - { - velocityField[blockId].push_back(VelocitySiteData()); - } - } - - site_t localSiteId = latDat.GetLocalSiteIdFromLocalSiteCoords(siteCoords); - velocityField[blockId][localSiteId].proc_id = proc_id; - } - - // Populate the matrix v with all the velocity field data at each index. - // Returns true if this the area resides entirely on this core. - void VelocityField::GetVelocityFieldAroundPoint(const util::Vector3D location, - const geometry::LatticeData& latDat, - util::Vector3D localVelocityField[2][2][2]) - { - for (int unitGridI = 0; unitGridI <= 1; ++unitGridI) - { - site_t neighbourI = location.x + unitGridI; - - for (int unitGridJ = 0; unitGridJ <= 1; ++unitGridJ) - { - site_t neighbourJ = location.y + unitGridJ; - - for (int unitGridK = 0; unitGridK <= 1; ++unitGridK) - { - site_t neighbourK = location.z + unitGridK; - - util::Vector3D neighbour(neighbourI, neighbourJ, neighbourK); - - if (!latDat.IsValidLatticeSite(neighbour)) - { - // it is a solid site and the velocity is - // assumed to be zero - localVelocityField[unitGridI][unitGridJ][unitGridK] = util::Vector3D::Zero(); - continue; - } - - VelocitySiteData *vel_site_data_p = GetVelocitySiteData(latDat, - util::Vector3D(neighbourI, - neighbourJ, - neighbourK)); - - if (vel_site_data_p == NULL || vel_site_data_p->proc_id == -1) - { - // it is a solid site and the velocity is - // assumed to be zero - localVelocityField[unitGridI][unitGridJ][unitGridK] = util::Vector3D::Zero(); - continue; - } - - if (vel_site_data_p->counter != counter) - { - UpdateLocalField(vel_site_data_p, latDat); - } - - localVelocityField[unitGridI][unitGridJ][unitGridK] = vel_site_data_p->velocity; - } - } - } - } - - bool VelocityField::NeededFromNeighbour(const util::Vector3D location, - const geometry::LatticeData& latDat, - proc_t* sourceProcessor) - { - if (!latDat.IsValidLatticeSite(location)) - { - return false; - } - - VelocitySiteData *vel_site_data_p = GetVelocitySiteData(latDat, location); - - if (vel_site_data_p == NULL || vel_site_data_p->proc_id == -1 || vel_site_data_p->proc_id == localRank - || vel_site_data_p->counter == counter) - { - return false; - } - - vel_site_data_p->counter = counter; - *sourceProcessor = vel_site_data_p->proc_id; - - return true; - } - - void VelocityField::UpdateLocalField(const util::Vector3D& position, const geometry::LatticeData& latDat) - { - VelocitySiteData *localVelocitySiteData = GetVelocitySiteData(latDat, position); - - if (log::Logger::ShouldDisplay()) - { - if (localRank != localVelocitySiteData->proc_id) - { - log::Logger::Log("Got a request for velocity data " - "that actually seems to be on rank %i", - localVelocitySiteData->proc_id); - } - } - - UpdateLocalField(localVelocitySiteData, latDat); - } - - void VelocityField::UpdateLocalField(VelocitySiteData* localVelocitySiteData, const geometry::LatticeData& latDat) - { - // the local counter is set equal to the global one - // and the local velocity is calculated - localVelocitySiteData->counter = counter; - - const util::Vector3D& velocity = propertyCache.velocityCache.Get(localVelocitySiteData->site_id); - localVelocitySiteData->velocity.x = (float) velocity.x; - localVelocitySiteData->velocity.y = (float) velocity.y; - localVelocitySiteData->velocity.z = (float) velocity.z; - } - - // Interpolates a velocity field to get the velocity at the position of a particle. - util::Vector3D VelocityField::InterpolateVelocityForPoint(const util::Vector3D point, - const util::Vector3D localVelocityField[2][2][2]) const - { - float dummy; - - // Get the fractional parts of each of x, y, z - float dx = modff(point.x, &dummy); - float dy = modff(point.y, &dummy); - float dz = modff(point.z, &dummy); - - util::Vector3D yInterpolatedVelocity[2]; - - for (int unitX = 0; unitX <= 1; unitX++) - { - util::Vector3D zInterpolatedVelocityForThisX[2]; - - for (int unitY = 0; unitY <= 1; unitY++) - { - zInterpolatedVelocityForThisX[unitY] = localVelocityField[unitX][unitY][0] * (1.F - dz) - + localVelocityField[unitX][unitY][1] * dz; - } - - yInterpolatedVelocity[unitX] = zInterpolatedVelocityForThisX[0] * (1.F - dy) - + zInterpolatedVelocityForThisX[1] * dy; - } - - return yInterpolatedVelocity[0] * (1.F - dx) + yInterpolatedVelocity[1] * dx; - } - - void VelocityField::InvalidateAllCalculatedVelocities() - { - ++counter; - } - - } - } -} diff --git a/Code/vis/streaklineDrawer/VelocityField.h b/Code/vis/streaklineDrawer/VelocityField.h deleted file mode 100644 index 6f1a60398..000000000 --- a/Code/vis/streaklineDrawer/VelocityField.h +++ /dev/null @@ -1,84 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_VELOCITYFIELD_H -#define HEMELB_VIS_STREAKLINEDRAWER_VELOCITYFIELD_H - -#include -#include - -#include "constants.h" -#include "net/mpi.h" - -#include "geometry/LatticeData.h" -#include "lb/MacroscopicPropertyCache.h" -#include "net/IOCommunicator.h" - -#include "vis/streaklineDrawer/NeighbouringProcessor.h" -#include "vis/streaklineDrawer/VelocitySiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - class VelocityField - { - public: - VelocityField(proc_t localRank, - std::map& iNeighbouringProcessors, - const lb::MacroscopicPropertyCache& propertyCache); - - void BuildVelocityField(const geometry::LatticeData& latDat); - - bool BlockContainsData(size_t iBlockNumber) const; - - VelocitySiteData* GetVelocitySiteData(const geometry::LatticeData& latDat, - const util::Vector3D& location); - - void GetVelocityFieldAroundPoint(const util::Vector3D location, - const geometry::LatticeData& latDat, - util::Vector3D localVelocityField[2][2][2]); - - util::Vector3D - InterpolateVelocityForPoint(const util::Vector3D position, - const util::Vector3D localVelocityField[2][2][2]) const; - - void InvalidateAllCalculatedVelocities(); - - void UpdateLocalField(const util::Vector3D& position, const geometry::LatticeData& latDat); - - bool NeededFromNeighbour(const util::Vector3D location, - const geometry::LatticeData& latDat, - proc_t* sourceProcessor); - - private: - void UpdateLocalField(VelocitySiteData* localVelocitySiteData, const geometry::LatticeData& latDat); - - // Counter to make sure the velocity field blocks are correct for the current iteration. - site_t counter; - - VelocitySiteData& GetSiteData(site_t iBlockNumber, site_t iSiteNumber); - - void InitializeVelocityFieldBlock(const geometry::LatticeData& latDat, - const util::Vector3D location, - const proc_t proc_id); - - const proc_t localRank; - // Vector containing VelocityFields - std::vector > velocityField; - std::map& neighbouringProcessors; - const lb::MacroscopicPropertyCache& propertyCache; - }; - } - } -} - -#endif // HEMELB_VIS_STREAKLINEDRAWER_VELOCITYFIELD_H diff --git a/Code/vis/streaklineDrawer/VelocitySiteData.h b/Code/vis/streaklineDrawer/VelocitySiteData.h deleted file mode 100644 index 9fb4c9cc6..000000000 --- a/Code/vis/streaklineDrawer/VelocitySiteData.h +++ /dev/null @@ -1,56 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_VELOCITYSITEDATA_H -#define HEMELB_VIS_STREAKLINEDRAWER_VELOCITYSITEDATA_H - -#include "util/Vector3D.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - // Class to hold information about the velocity field at some point. - class VelocitySiteData - { - public: - VelocitySiteData() : - counter(-1), proc_id(-1), site_id(-1), velocity(NO_VALUE) - { - } - - /** - * Counter describes how many iterations have passed since the objects creation. - * It allows for quickly checking whether the local velocity field has been - * calculated this iteration. - **/ - site_t counter; - - /** - * The rank this volume of the geometry lives on. - */ - proc_t proc_id; - - /** - * The site id of the volume represented. - */ - site_t site_id; - - /** - * The velocity calculated for that site. - */ - util::Vector3D velocity; - }; - } - } -} - -#endif //HEMELB_VIS_STREAKLINEDRAWER_VELOCITYSITEDATA_H From 876cab32bdca95ce8c8cb33cf98b892638e9430b Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 3 Dec 2015 16:29:51 +0000 Subject: [PATCH 26/99] Remove the DoRayTracing template parameter from the LB --- Code/lb/lb.h | 5 +- Code/lb/streamers/BaseStreamer.h | 49 ++- Code/lb/streamers/JunkYangFactory.h | 22 +- Code/lb/streamers/SimpleCollideAndStream.h | 23 +- Code/lb/streamers/StreamerTypeFactory.h | 79 ++-- Code/lb/streamers/VirtualSiteIolet.h | 70 ++-- Code/unittests/lbtests/StreamerTests.h | 356 +++++++++--------- .../lbtests/VirtualSiteIoletStreamerTests.h | 128 +++---- 8 files changed, 347 insertions(+), 385 deletions(-) diff --git a/Code/lb/lb.h b/Code/lb/lb.h index fca4de283..c356c55ce 100644 --- a/Code/lb/lb.h +++ b/Code/lb/lb.h @@ -121,14 +121,13 @@ namespace hemelb template void StreamAndCollide(Collision* collision, const site_t iFirstIndex, const site_t iSiteCount) { - // TODO: factor out template bool "do rendering" - collision->template StreamAndCollide (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); + collision->StreamAndCollide(iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); } template void PostStep(Collision* collision, const site_t iFirstIndex, const site_t iSiteCount) { - collision->template DoPostStep (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); + collision->DoPostStep(iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); } unsigned int inletCount; diff --git a/Code/lb/streamers/BaseStreamer.h b/Code/lb/streamers/BaseStreamer.h index 1bd40b540..345afb7d4 100644 --- a/Code/lb/streamers/BaseStreamer.h +++ b/Code/lb/streamers/BaseStreamer.h @@ -28,8 +28,8 @@ namespace hemelb * BaseStreamer: inheritable base class for the streaming operator. The public interface * here defines the complete interface usable by external code. * - Constructor(InitParams&) - * - StreamAndCollide(const site_t, const site_t, const LbmParameters*, - * geometry::LatticeData*, hemelb::vis::Control*) + * - StreamAndCollide(const site_t, const site_t, const LbmParameters*, + * geometry::LatticeData*, lb::MacroscopicPropertyCache&) * - PostStep(const site_t, const site_t, const LbmParameters*, * geometry::LatticeData*, hemelb::vis::Control*) * - Reset(kernels::InitParams* init) @@ -56,43 +56,37 @@ namespace hemelb class BaseStreamer { public: - // TODO: remove tDoRayTracing - template - inline void StreamAndCollide(const site_t firstIndex, - const site_t siteCount, + inline void StreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) { - static_cast (this)->template DoStreamAndCollide (firstIndex, - siteCount, - lbmParams, - latDat, - propertyCache); + static_cast(this)->DoStreamAndCollide(firstIndex, + siteCount, + lbmParams, + latDat, + propertyCache); } - template - inline void PostStep(const site_t firstIndex, - const site_t siteCount, - const LbmParameters* lbmParams, - geometry::LatticeData* latDat, + inline void PostStep(const site_t firstIndex, const site_t siteCount, + const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) { // The template parameter is required because we're using the CRTP to call a // metaprogrammed method of the implementation class. - static_cast (this)->template DoPostStep (firstIndex, - siteCount, - lbmParams, - latDat, - propertyCache); + static_cast(this)->DoPostStep(firstIndex, + siteCount, + lbmParams, + latDat, + propertyCache); } protected: - template - inline static void UpdateMinsAndMaxes(const geometry::Site& site, - const kernels::HydroVarsBase& hydroVars, - const LbmParameters* lbmParams, - lb::MacroscopicPropertyCache& propertyCache) + template + inline static void UpdateMinsAndMaxes( + const geometry::Site& site, + const kernels::HydroVarsBase& hydroVars, const LbmParameters* lbmParams, + lb::MacroscopicPropertyCache& propertyCache) { if (propertyCache.densityCache.RequiresRefresh()) { @@ -194,7 +188,8 @@ namespace hemelb tangentialProjectionTractionOnAPoint); } - propertyCache.tangentialProjectionTractionCache.Put(site.GetIndex(), tangentialProjectionTractionOnAPoint); + propertyCache.tangentialProjectionTractionCache.Put(site.GetIndex(), + tangentialProjectionTractionOnAPoint); } } diff --git a/Code/lb/streamers/JunkYangFactory.h b/Code/lb/streamers/JunkYangFactory.h index 774cb4904..d893188ec 100644 --- a/Code/lb/streamers/JunkYangFactory.h +++ b/Code/lb/streamers/JunkYangFactory.h @@ -80,7 +80,6 @@ namespace hemelb } } - template inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latticeData, @@ -148,15 +147,14 @@ namespace hemelb fOld[siteIndex](index) = site.GetFOld()[*outgoingVelocityIter]; } - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template inline void DoPostStep(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latticeData, lb::MacroscopicPropertyCache& propertyCache) @@ -327,8 +325,10 @@ namespace hemelb + LatticeType::CZ[*rowIndexIncomingVelocity] * LatticeType::CZ[*columnIndexIncomingVelocity]; - assert(site.template GetWallDistance (LatticeType::INVERSEDIRECTIONS[(Direction) *rowIndexIncomingVelocity]) >= 0); - assert(site.template GetWallDistance (LatticeType::INVERSEDIRECTIONS[(Direction) *rowIndexIncomingVelocity]) < 1); + assert(site.template GetWallDistance(LatticeType::INVERSEDIRECTIONS[(Direction ) *rowIndexIncomingVelocity]) + >= 0); + assert(site.template GetWallDistance(LatticeType::INVERSEDIRECTIONS[(Direction ) *rowIndexIncomingVelocity]) + < 1); kMatrices[contiguousSiteIndex](rowIndex, columnIndex) = (-3.0 / 2.0) @@ -369,8 +369,10 @@ namespace hemelb + LatticeType::CZ[*rowIndexIncomingVelocity] * LatticeType::CZ[*columnIndexOutgoingVelocity]; - assert(site.template GetWallDistance (LatticeType::INVERSEDIRECTIONS[(Direction) *rowIndexIncomingVelocity]) >= 0); - assert(site.template GetWallDistance (LatticeType::INVERSEDIRECTIONS[(Direction) *rowIndexIncomingVelocity]) < 1); + assert(site.template GetWallDistance(LatticeType::INVERSEDIRECTIONS[(Direction ) *rowIndexIncomingVelocity]) + >= 0); + assert(site.template GetWallDistance(LatticeType::INVERSEDIRECTIONS[(Direction ) *rowIndexIncomingVelocity]) + < 1); kMatrices[contiguousSiteIndex](rowIndex, columnIndex) = (-3.0 / 2.0) diff --git a/Code/lb/streamers/SimpleCollideAndStream.h b/Code/lb/streamers/SimpleCollideAndStream.h index 27e694907..76f146a8b 100644 --- a/Code/lb/streamers/SimpleCollideAndStream.h +++ b/Code/lb/streamers/SimpleCollideAndStream.h @@ -35,14 +35,12 @@ namespace hemelb public: SimpleCollideAndStream(kernels::InitParams& initParams) : - collider(initParams), bulkLinkDelegate(collider, initParams) + collider(initParams), bulkLinkDelegate(collider, initParams) { } - template - inline void DoStreamAndCollide(const site_t firstIndex, - const site_t siteCount, + inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) @@ -51,7 +49,7 @@ namespace hemelb { geometry::Site site = latDat->GetSite(siteIndex); - const distribn_t* lFOld = site.GetFOld (); + const distribn_t* lFOld = site.GetFOld(); kernels::HydroVars hydroVars(lFOld); @@ -67,18 +65,15 @@ namespace hemelb bulkLinkDelegate.StreamLink(lbmParams, latDat, site, hydroVars, ii); } - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template - inline void DoPostStep(const site_t iFirstIndex, - const site_t iSiteCount, - const LbmParameters* iLbmParams, - geometry::LatticeData* bLatDat, + inline void DoPostStep(const site_t iFirstIndex, const site_t iSiteCount, + const LbmParameters* iLbmParams, geometry::LatticeData* bLatDat, lb::MacroscopicPropertyCache& propertyCache) { diff --git a/Code/lb/streamers/StreamerTypeFactory.h b/Code/lb/streamers/StreamerTypeFactory.h index d990e1c59..d92badc5c 100644 --- a/Code/lb/streamers/StreamerTypeFactory.h +++ b/Code/lb/streamers/StreamerTypeFactory.h @@ -29,7 +29,8 @@ namespace hemelb * template on WallLinkImpl. */ template - class WallStreamerTypeFactory : public BaseStreamer > + class WallStreamerTypeFactory : public BaseStreamer< + WallStreamerTypeFactory > { public: typedef CollisionImpl CollisionType; @@ -43,14 +44,13 @@ namespace hemelb public: WallStreamerTypeFactory(kernels::InitParams& initParams) : - collider(initParams), bulkLinkDelegate(collider, initParams), wallLinkDelegate(collider, initParams) + collider(initParams), bulkLinkDelegate(collider, initParams), + wallLinkDelegate(collider, initParams) { } - template - inline void DoStreamAndCollide(const site_t firstIndex, - const site_t siteCount, + inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) @@ -59,7 +59,7 @@ namespace hemelb { geometry::Site site = latDat->GetSite(siteIndex); - const distribn_t* fOld = site.GetFOld (); + const distribn_t* fOld = site.GetFOld(); kernels::HydroVars hydroVars(fOld); @@ -83,15 +83,14 @@ namespace hemelb } //TODO: Necessary to specify sub-class? - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template - inline void DoPostStep(const site_t firstIndex, - const site_t siteCount, + + inline void DoPostStep(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParameters, geometry::LatticeData* latticeData, lb::MacroscopicPropertyCache& propertyCache) @@ -120,7 +119,8 @@ namespace hemelb * template on IoletLinkImpl. */ template - class IoletStreamerTypeFactory : public BaseStreamer > + class IoletStreamerTypeFactory : public BaseStreamer< + IoletStreamerTypeFactory > { public: typedef CollisionImpl CollisionType; @@ -134,14 +134,13 @@ namespace hemelb public: IoletStreamerTypeFactory(kernels::InitParams& initParams) : - collider(initParams), bulkLinkDelegate(collider, initParams), ioletLinkDelegate(collider, initParams) + collider(initParams), bulkLinkDelegate(collider, initParams), + ioletLinkDelegate(collider, initParams) { } - template - inline void DoStreamAndCollide(const site_t firstIndex, - const site_t siteCount, + inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) @@ -150,7 +149,7 @@ namespace hemelb { geometry::Site site = latDat->GetSite(siteIndex); - const distribn_t* fOld = site.GetFOld (); + const distribn_t* fOld = site.GetFOld(); kernels::HydroVars hydroVars(fOld); @@ -174,15 +173,14 @@ namespace hemelb } //TODO: Necessary to specify sub-class? - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template - inline void DoPostStep(const site_t firstIndex, - const site_t siteCount, + + inline void DoPostStep(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParameters, geometry::LatticeData* latticeData, lb::MacroscopicPropertyCache& propertyCache) @@ -212,8 +210,8 @@ namespace hemelb * template on WallLinkImpl and IoletLinkImpl. */ template - class WallIoletStreamerTypeFactory : public BaseStreamer > + class WallIoletStreamerTypeFactory : public BaseStreamer< + WallIoletStreamerTypeFactory > { public: typedef CollisionImpl CollisionType; @@ -227,15 +225,13 @@ namespace hemelb public: WallIoletStreamerTypeFactory(kernels::InitParams& initParams) : - collider(initParams), bulkLinkDelegate(collider, initParams), wallLinkDelegate(collider, initParams), - ioletLinkDelegate(collider, initParams) + collider(initParams), bulkLinkDelegate(collider, initParams), + wallLinkDelegate(collider, initParams), ioletLinkDelegate(collider, initParams) { } - template - inline void DoStreamAndCollide(const site_t firstIndex, - const site_t siteCount, + inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) @@ -244,7 +240,7 @@ namespace hemelb { geometry::Site site = latDat->GetSite(siteIndex); - const distribn_t* fOld = site.GetFOld (); + const distribn_t* fOld = site.GetFOld(); kernels::HydroVars hydroVars(fOld); @@ -272,18 +268,15 @@ namespace hemelb } //TODO: Necessary to specify sub-class? - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template - inline void DoPostStep(const site_t firstIndex, - const site_t siteCount, - const LbmParameters* lbmParams, - geometry::LatticeData* latticeData, + inline void DoPostStep(const site_t firstIndex, const site_t siteCount, + const LbmParameters* lbmParams, geometry::LatticeData* latticeData, lb::MacroscopicPropertyCache& propertyCache) { for (site_t siteIndex = firstIndex; siteIndex < (firstIndex + siteCount); siteIndex++) diff --git a/Code/lb/streamers/VirtualSiteIolet.h b/Code/lb/streamers/VirtualSiteIolet.h index 161737586..fb8801aec 100644 --- a/Code/lb/streamers/VirtualSiteIolet.h +++ b/Code/lb/streamers/VirtualSiteIolet.h @@ -50,22 +50,21 @@ namespace hemelb struct IoletVSiteDirection { IoletVSiteDirection(InOutLet*iolet_, VirtualSite* vsite_, Direction i_) : - iolet(iolet_), vsite(vsite_), direction(i_) + iolet(iolet_), vsite(vsite_), direction(i_) { } InOutLet* iolet; VirtualSite* vsite; Direction direction; }; - typedef typename util::FlatMultiMap::Type - VSiteByLocalIdxMultiMap; + typedef typename util::FlatMultiMap::Type VSiteByLocalIdxMultiMap; VSiteByLocalIdxMultiMap vsByLocalIdx; public: VirtualSiteIolet(kernels::InitParams& initParams) : - collider(initParams), bulkLinkDelegate(collider, initParams), - wallLinkDelegate(collider, initParams), bValues(initParams.boundaryObject), - neighbouringLatticeData(initParams.latDat->GetNeighbouringData()) + collider(initParams), bulkLinkDelegate(collider, initParams), + wallLinkDelegate(collider, initParams), bValues(initParams.boundaryObject), + neighbouringLatticeData(initParams.latDat->GetNeighbouringData()) { // Loop over the local in/outlets, creating the extra data objects. unsigned nIolets = bValues->GetLocalIoletCount(); @@ -73,7 +72,7 @@ namespace hemelb { InOutLet& iolet = *bValues->GetLocalIolet(iIolet); if (iolet.GetExtraData() == NULL) - iolet.SetExtraData(new VSExtra (iolet)); + iolet.SetExtraData(new VSExtra(iolet)); } lattices::LatticeInfo& lattice = LatticeType::GetLatticeInfo(); @@ -108,21 +107,19 @@ namespace hemelb continue; const LatticeVector neighbourLocation = siteLocation + lattice.GetVector(i); - site_t - neighbourGlobalIdx = - initParams.latDat->GetGlobalNoncontiguousSiteIdFromGlobalCoords(neighbourLocation); + site_t neighbourGlobalIdx = + initParams.latDat->GetGlobalNoncontiguousSiteIdFromGlobalCoords(neighbourLocation); typename VSiteType::Map::iterator vNeigh = extra->vSites.find(neighbourGlobalIdx); // find() returns end() if key not present if (vNeigh == extra->vSites.end()) { // Create a vSite - std::pair - inserted = - extra->vSites.insert(typename VSiteType::Map::value_type(neighbourGlobalIdx, - VSiteType(initParams, - *extra, - neighbourLocation))); + std::pair inserted = + extra->vSites.insert(typename VSiteType::Map::value_type(neighbourGlobalIdx, + VSiteType(initParams, + *extra, + neighbourLocation))); // inserted.first is an iterator, pointing to a pair vNeigh = inserted.first; } @@ -143,7 +140,6 @@ namespace hemelb * links will be done in the post-step as we must ensure that all * the data is available to construct virtual sites. */ - template inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, @@ -153,7 +149,7 @@ namespace hemelb { geometry::Site site = latDat->GetSite(siteIndex); - const distribn_t* fOld = site.GetFOld (); + const distribn_t* fOld = site.GetFOld(); kernels::HydroVars hydroVars(fOld); @@ -195,14 +191,13 @@ namespace hemelb cachedHV.u = hydroVars.velocity; // TODO: Necessary to specify sub-class? - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template inline void DoPostStep(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) @@ -212,8 +207,8 @@ namespace hemelb vsByLocalIdx.lower_bound(firstIndex), endVSites = vsByLocalIdx.lower_bound(firstIndex + siteCount); - for (typename VSiteByLocalIdxMultiMap::iterator vSiteIt = beginVSites; vSiteIt - != endVSites; ++vSiteIt) + for (typename VSiteByLocalIdxMultiMap::iterator vSiteIt = beginVSites; + vSiteIt != endVSites; ++vSiteIt) { site_t siteIdx = vSiteIt->first; // vSiteIt->second == (Iolet*, VirtualSite*, Direction) @@ -241,8 +236,8 @@ namespace hemelb std::ofstream hvCache("hvCache"); hvCache << "# local global x y z" << std::endl; - for (RSHV::Map::const_iterator hvIt = extra->hydroVarsCache.begin(); hvIt - != extra->hydroVarsCache.end(); ++hvIt) + for (RSHV::Map::const_iterator hvIt = extra->hydroVarsCache.begin(); + hvIt != extra->hydroVarsCache.end(); ++hvIt) { site_t global = hvIt->first; LatticeVector pos; @@ -275,7 +270,8 @@ namespace hemelb std::ofstream outletMap("outletMap"); outletMap << "# local global x y z vSitePtr direction" << std::endl; for (typename VSiteByLocalIdxMultiMap::const_iterator entry = - ioletStreamer->vsByLocalIdx.begin(); entry != ioletStreamer->vsByLocalIdx.end(); ++entry) + ioletStreamer->vsByLocalIdx.begin(); entry != ioletStreamer->vsByLocalIdx.end(); + ++entry) { site_t local = entry->first; geometry::Site site = latDat->GetSite(local); @@ -289,8 +285,8 @@ namespace hemelb std::ofstream outletWallMap("outletWallMap"); outletWallMap << "# local global x y z vSitePtr direction" << std::endl; for (typename VSiteByLocalIdxMultiMap::const_iterator entry = - ioletWallStreamer->vsByLocalIdx.begin(); entry - != ioletWallStreamer->vsByLocalIdx.end(); ++entry) + ioletWallStreamer->vsByLocalIdx.begin(); + entry != ioletWallStreamer->vsByLocalIdx.end(); ++entry) { site_t local = entry->first; geometry::Site site = latDat->GetSite(local); @@ -306,7 +302,7 @@ namespace hemelb static VSExtra* GetExtra(InOutLet* iolet) { // Get the extra data for this iolet - VSExtra* ans = dynamic_cast*> (iolet->GetExtraData()); + VSExtra* ans = dynamic_cast*>(iolet->GetExtraData()); if (ans == NULL) { // panic @@ -354,7 +350,8 @@ namespace hemelb LatticeDensity CalculateVirtualSiteDensity(const geometry::LatticeData& latDat, const InOutLet& iolet, RSHV::Map& hydroVarsCache, - const VSiteType& vSite, const LatticeTimeStep t) + const VSiteType& vSite, + const LatticeTimeStep t) { LatticeDensity rho = 0.; LatticeDensity rho_iolet = iolet.GetDensity(t); @@ -387,7 +384,8 @@ namespace hemelb LatticeVelocity CalculateVirtualSiteVelocity(const geometry::LatticeData& latDat, const InOutLet& iolet, RSHV::Map& hydroVarsCache, - const VSiteType& vSite, const LatticeTimeStep t) + const VSiteType& vSite, + const LatticeTimeStep t) { /* * Anstaz is u(x,y) = Ax + By + C @@ -440,8 +438,8 @@ namespace hemelb coeffs[i] += vSite.velocityMatrixInv[i][j] * sums[j]; // Compute the magnitude of the velocity. - LatticeSpeed ansNorm = coeffs[0] * vSite.hv.posIolet.x + coeffs[1] - * vSite.hv.posIolet.y + coeffs[2]; + LatticeSpeed ansNorm = coeffs[0] * vSite.hv.posIolet.x + coeffs[1] * vSite.hv.posIolet.y + + coeffs[2]; // multiply by the iolet normal and we're done! return iolet.GetNormal() * ansNorm; @@ -462,7 +460,7 @@ namespace hemelb geometry::neighbouring::ConstNeighbouringSite neigh = latDat.GetNeighbouringData().GetSite(globalIdx); - const distribn_t* fOld = neigh.GetFOld (); + const distribn_t* fOld = neigh.GetFOld(); LatticeType::CalculateDensityAndMomentum(fOld, ans.rho, ans.u.x, ans.u.y, ans.u.z); if (LatticeType::IsLatticeCompressible()) { diff --git a/Code/unittests/lbtests/StreamerTests.h b/Code/unittests/lbtests/StreamerTests.h index 64c9d2df0..044e72ec1 100644 --- a/Code/unittests/lbtests/StreamerTests.h +++ b/Code/unittests/lbtests/StreamerTests.h @@ -36,14 +36,14 @@ namespace hemelb */ class StreamerTests : public helpers::FourCubeBasedTestFixture { - CPPUNIT_TEST_SUITE ( StreamerTests); - CPPUNIT_TEST ( TestSimpleCollideAndStream); - CPPUNIT_TEST ( TestBouzidiFirdaousLallemand); - CPPUNIT_TEST ( TestSimpleBounceBack); - CPPUNIT_TEST ( TestGuoZhengShi); - CPPUNIT_TEST ( TestNashZerothOrderPressureIolet); - CPPUNIT_TEST ( TestNashZerothOrderPressureBB); - CPPUNIT_TEST ( TestJunkYangEquivalentToBounceBack);CPPUNIT_TEST_SUITE_END(); + CPPUNIT_TEST_SUITE (StreamerTests); + CPPUNIT_TEST (TestSimpleCollideAndStream); + CPPUNIT_TEST (TestBouzidiFirdaousLallemand); + CPPUNIT_TEST (TestSimpleBounceBack); + CPPUNIT_TEST (TestGuoZhengShi); + CPPUNIT_TEST (TestNashZerothOrderPressureIolet); + CPPUNIT_TEST (TestNashZerothOrderPressureBB); + CPPUNIT_TEST (TestJunkYangEquivalentToBounceBack);CPPUNIT_TEST_SUITE_END(); public: void setUp() @@ -51,8 +51,8 @@ namespace hemelb FourCubeBasedTestFixture::setUp(); propertyCache = new lb::MacroscopicPropertyCache(*simState, *latDat); - normalCollision - = new lb::collisions::Normal >(initParams); + normalCollision = + new lb::collisions::Normal >(initParams); } void tearDown() @@ -65,8 +65,8 @@ namespace hemelb void TestSimpleCollideAndStream() { - lb::streamers::SimpleCollideAndStream > > simpleCollideAndStream(initParams); + lb::streamers::SimpleCollideAndStream< + lb::collisions::Normal > > simpleCollideAndStream(initParams); // Initialise fOld in the lattice data. We choose values so that each site has // an anisotropic distribution function, and that each site's function is @@ -74,32 +74,32 @@ namespace hemelb LbTestsHelper::InitialiseAnisotropicTestData(latDat); // Use the streaming operator on the entire lattice. - simpleCollideAndStream.StreamAndCollide (0, - latDat->GetLocalFluidSiteCount(), - lbmParams, - latDat, - *propertyCache); + simpleCollideAndStream.StreamAndCollide(0, + latDat->GetLocalFluidSiteCount(), + lbmParams, + latDat, + *propertyCache); // Now, go over each lattice site and check each value in f_new is correct. - for (site_t streamedToSite = 0; streamedToSite < latDat->GetLocalFluidSiteCount(); ++streamedToSite) + for (site_t streamedToSite = 0; streamedToSite < latDat->GetLocalFluidSiteCount(); + ++streamedToSite) { - geometry::Site < geometry::LatticeData > streamedSite - = latDat->GetSite(streamedToSite); + geometry::Site streamedSite = latDat->GetSite(streamedToSite); distribn_t* streamedToFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * streamedToSite); - for (unsigned int streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (unsigned int streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { - site_t - streamerIndex = - streamedSite.GetStreamedIndex (lb::lattices::D3Q15::INVERSEDIRECTIONS[streamedDirection]); + site_t streamerIndex = + streamedSite.GetStreamedIndex(lb::lattices::D3Q15::INVERSEDIRECTIONS[streamedDirection]); // If this site streamed somewhere sensible, it must have been streamed to. - if (streamerIndex >= 0 && streamerIndex < (lb::lattices::D3Q15::NUMVECTORS - * latDat->GetLocalFluidSiteCount())) + if (streamerIndex >= 0 + && streamerIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { site_t streamerSiteId = streamerIndex / lb::lattices::D3Q15::NUMVECTORS; @@ -109,8 +109,7 @@ namespace hemelb streamerFOld); // Calculate what the value streamed to site streamedToSite should be. - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamedSite); normalCollision->Collide(lbmParams, streamerHydroVars); @@ -132,22 +131,19 @@ namespace hemelb // an anisotropic distribution function, and that each site's function is // distinguishable. LbTestsHelper::InitialiseAnisotropicTestData(latDat); - lb::streamers::BouzidiFirdaousLallemand > >::Type bfl(initParams); - - bfl.StreamAndCollide (0, - latDat->GetLocalFluidSiteCount(), - lbmParams, - latDat, - *propertyCache); - bfl.PostStep (0, + lb::streamers::BouzidiFirdaousLallemand< + lb::collisions::Normal > >::Type bfl(initParams); + + bfl.StreamAndCollide(0, latDat->GetLocalFluidSiteCount(), lbmParams, latDat, *propertyCache); + bfl.PostStep(0, latDat->GetLocalFluidSiteCount(), lbmParams, latDat, *propertyCache); // Now, go over each lattice site and check each value in f_new is correct. - for (site_t streamedToSite = 0; streamedToSite < latDat->GetLocalFluidSiteCount(); ++streamedToSite) + for (site_t streamedToSite = 0; streamedToSite < latDat->GetLocalFluidSiteCount(); + ++streamedToSite) { const geometry::Site streamedSite = latDat->GetSite(streamedToSite); @@ -155,21 +151,21 @@ namespace hemelb distribn_t* streamedToFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * streamedToSite); - for (unsigned int streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (unsigned int streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { unsigned int oppDirection = lb::lattices::D3Q15::INVERSEDIRECTIONS[streamedDirection]; site_t streamerIndex = - streamedSite.GetStreamedIndex (oppDirection); + streamedSite.GetStreamedIndex(oppDirection); - geometry::Site < geometry::LatticeData > streamerSite - = latDat->GetSite(streamerIndex); + geometry::Site streamerSite = latDat->GetSite(streamerIndex); // If this site streamed somewhere sensible, it must have been streamed to. - if (streamerIndex >= 0 && streamerIndex < (lb::lattices::D3Q15::NUMVECTORS - * latDat->GetLocalFluidSiteCount())) + if (streamerIndex >= 0 + && streamerIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { site_t streamerSiteId = streamerIndex / lb::lattices::D3Q15::NUMVECTORS; @@ -179,8 +175,7 @@ namespace hemelb streamerFOld); // Calculate what the value streamed to site streamedToSite should be. - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamerSite); normalCollision->Collide(lbmParams, streamerHydroVars); @@ -205,18 +200,18 @@ namespace hemelb << " Data: " << streamedSite.GetSiteData().GetWallIntersectionData() << std::flush; CPPUNIT_ASSERT_MESSAGE("Expected to find a boundary " - "opposite an unstreamed-to direction " + message.str(), + "opposite an unstreamed-to direction " + message.str(), streamedSite.HasWall(oppDirection)); // Test disabled due to RegressionTests issue, see discussion in #87 CPPUNIT_ASSERT_MESSAGE("Expect defined cut distance opposite an unstreamed-to direction " + message.str(), - streamedSite.GetWallDistance (oppDirection) + streamedSite.GetWallDistance(oppDirection) != -1.0); // To verify the operation of the BFL boundary condition, we'll need: // - the distance to the wall * 2 distribn_t twoQ = 2.0 - * streamedSite.GetWallDistance (oppDirection); + * streamedSite.GetWallDistance(oppDirection); // - the post-collision distribution at the current site. distribn_t streamedToSiteFOld[lb::lattices::D3Q15::NUMVECTORS]; @@ -225,8 +220,7 @@ namespace hemelb LbTestsHelper::InitialiseAnisotropicTestData(streamedToSite, streamedToSiteFOld); - lb::kernels::HydroVars > - hydroVars(streamedToSiteFOld); + lb::kernels::HydroVars > hydroVars(streamedToSiteFOld); normalCollision->CalculatePreCollision(hydroVars, streamedSite); @@ -238,12 +232,12 @@ namespace hemelb distribn_t awayFromWallFOld[lb::lattices::D3Q15::NUMVECTORS]; site_t awayFromWallIndex = - streamedSite.GetStreamedIndex (streamedDirection) + streamedSite.GetStreamedIndex(streamedDirection) / lb::lattices::D3Q15::NUMVECTORS; // If there's a valid index in that direction, use BFL - if (awayFromWallIndex >= 0 && awayFromWallIndex - < latDat->GetLocalFluidSiteCount()) + if (awayFromWallIndex >= 0 + && awayFromWallIndex < latDat->GetLocalFluidSiteCount()) { const geometry::Site awayFromWallSite = latDat->GetSite(awayFromWallIndex); @@ -252,10 +246,10 @@ namespace hemelb LbTestsHelper::InitialiseAnisotropicTestData(awayFromWallIndex, awayFromWallFOld); - lb::kernels::HydroVars > - awayFromWallsHydroVars(awayFromWallFOld); + lb::kernels::HydroVars > awayFromWallsHydroVars(awayFromWallFOld); - normalCollision->CalculatePreCollision(awayFromWallsHydroVars, awayFromWallSite); + normalCollision->CalculatePreCollision(awayFromWallsHydroVars, + awayFromWallSite); // (find post-collision values using the collision operator). normalCollision->Collide(lbmParams, awayFromWallsHydroVars); @@ -267,9 +261,9 @@ namespace hemelb distribn_t oppWallOld = hydroVars.GetFPostCollision()[streamedDirection]; // The streamed value should be as given below. - distribn_t streamed = (twoQ < 1.0) - ? (toWallNew + twoQ * (toWallOld - toWallNew)) - : (oppWallOld + (1. / twoQ) * (toWallOld - oppWallOld)); + distribn_t streamed = (twoQ < 1.0) ? + (toWallNew + twoQ * (toWallOld - toWallNew)) : + (oppWallOld + (1. / twoQ) * (toWallOld - oppWallOld)); std::stringstream msg(std::stringstream::in); msg << "BouzidiFirdaousLallemand, PostStep: site " << streamedToSite @@ -321,34 +315,33 @@ namespace hemelb site_t offset = 0; // Mid-Fluid sites use simple collide and stream - lb::streamers::SimpleCollideAndStream > > simpleCollideAndStream(initParams); - - simpleCollideAndStream.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(0), - lbmParams, - latDat, - *propertyCache); + lb::streamers::SimpleCollideAndStream< + lb::collisions::Normal > > simpleCollideAndStream(initParams); + + simpleCollideAndStream.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(0), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(0); // Wall sites use simple bounce back - lb::streamers::SimpleBounceBack > >::Type simpleBounceBack(initParams); - - simpleBounceBack.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(1), - lbmParams, - latDat, - *propertyCache); + lb::streamers::SimpleBounceBack< + lb::collisions::Normal > >::Type simpleBounceBack(initParams); + + simpleBounceBack.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(1), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(1); // Consider inlet/outlets and their walls as mid-fluid sites - simpleCollideAndStream.StreamAndCollide (offset, - latDat->GetLocalFluidSiteCount() - - offset, - lbmParams, - latDat, - *propertyCache); + simpleCollideAndStream.StreamAndCollide(offset, + latDat->GetLocalFluidSiteCount() - offset, + lbmParams, + latDat, + *propertyCache); offset += latDat->GetLocalFluidSiteCount() - offset; // Sanity check @@ -361,7 +354,8 @@ namespace hemelb * depending on where they sit relative to the wall. We ignore mid-Fluid sites since * StreamAndCollide was tested before. */ - for (site_t wallSiteLocalIndex = 0; wallSiteLocalIndex < wallSitesCount; wallSiteLocalIndex++) + for (site_t wallSiteLocalIndex = 0; wallSiteLocalIndex < wallSitesCount; + wallSiteLocalIndex++) { site_t streamedToSite = firstWallSite + wallSiteLocalIndex; const geometry::Site streamedSite = @@ -369,18 +363,19 @@ namespace hemelb distribn_t* streamedToFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * streamedToSite); - for (unsigned int streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (unsigned int streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { unsigned oppDirection = lb::lattices::D3Q15::INVERSEDIRECTIONS[streamedDirection]; // Index of the site streaming to streamedToSite via direction streamedDirection site_t streamerIndex = - streamedSite.GetStreamedIndex (oppDirection); + streamedSite.GetStreamedIndex(oppDirection); // Is streamerIndex a valid index? - if (streamerIndex >= 0 && streamerIndex < (lb::lattices::D3Q15::NUMVECTORS - * latDat->GetLocalFluidSiteCount())) + if (streamerIndex >= 0 + && streamerIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { // The streamer index is a valid index in the domain, therefore stream and collide has happened site_t streamerSiteId = streamerIndex / lb::lattices::D3Q15::NUMVECTORS; @@ -391,8 +386,7 @@ namespace hemelb streamerFOld); // Calculate what the value streamed to site streamedToSite should be. - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamedSite); normalCollision->Collide(lbmParams, streamerHydroVars); @@ -413,8 +407,7 @@ namespace hemelb distribn_t streamerToSiteFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(streamedToSite, streamerToSiteFOld); - lb::kernels::HydroVars > - hydroVars(streamerToSiteFOld); + lb::kernels::HydroVars > hydroVars(streamerToSiteFOld); normalCollision->CalculatePreCollision(hydroVars, streamedSite); // Simulate post-collision using the collision operator. @@ -436,11 +429,11 @@ namespace hemelb void TestGuoZhengShi() { - lb::streamers::GuoZhengShi > >::Type guoZhengShi(initParams); + lb::streamers::GuoZhengShi< + lb::collisions::Normal > >::Type guoZhengShi(initParams); - for (double assignedWallDistance = 0.4; assignedWallDistance < 1.0; assignedWallDistance - += 0.5) + for (double assignedWallDistance = 0.4; assignedWallDistance < 1.0; + assignedWallDistance += 0.5) { // Initialise fOld in the lattice data. We choose values so that each site has // an anisotropic distribution function, and that each site's function is @@ -470,20 +463,19 @@ namespace hemelb assignedWallDistance); // Perform the collision and streaming. - guoZhengShi.StreamAndCollide (chosenSite, 1, lbmParams, latDat, *propertyCache); + guoZhengShi.StreamAndCollide(chosenSite, 1, lbmParams, latDat, *propertyCache); // Calculate the distributions at the chosen site up to post-collision. distribn_t streamerFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(chosenSite, streamerFOld); - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamer); normalCollision->Collide(lbmParams, streamerHydroVars); // Check each streamed direction. - for (Direction streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (Direction streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { switch (streamedDirection) { @@ -497,8 +489,8 @@ namespace hemelb // This is the first means of estimating from the source paper: only // use the nearest fluid site. - LatticeVelocity velocityEstimate1 = streamerHydroVars.momentum * (1. - 1. - / assignedWallDistance) / streamerHydroVars.density; + LatticeVelocity velocityEstimate1 = streamerHydroVars.momentum + * (1. - 1. / assignedWallDistance) / streamerHydroVars.density; distribn_t fNeqEstimate1 = streamerHydroVars.GetFNeq()[streamedDirection]; @@ -507,30 +499,28 @@ namespace hemelb // This is the second method for estimating: using the next fluid site // away from the wall. const site_t nextSiteAwayFromWall = streamer.GetStreamedIndex< - lb::lattices::D3Q15> (streamedDirection) - / lb::lattices::D3Q15::NUMVECTORS; + lb::lattices::D3Q15>(streamedDirection) / lb::lattices::D3Q15::NUMVECTORS; const geometry::Site& nextSiteAway = latDat->GetSite(nextSiteAwayFromWall); distribn_t nextSiteOutFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(nextSiteAwayFromWall, nextSiteOutFOld); - lb::kernels::HydroVars > - nextSiteOutHydroVars(nextSiteOutFOld); + lb::kernels::HydroVars > nextSiteOutHydroVars(nextSiteOutFOld); normalCollision->CalculatePreCollision(nextSiteOutHydroVars, nextSiteAway); LatticeVelocity velocityEstimate2 = nextSiteOutHydroVars.momentum - * (assignedWallDistance - 1.) / ( (1. + assignedWallDistance) - * nextSiteOutHydroVars.density); + * (assignedWallDistance - 1.) + / ( (1. + assignedWallDistance) * nextSiteOutHydroVars.density); distribn_t fNeqEstimate2 = nextSiteOutHydroVars.GetFNeq()[streamedDirection]; // The actual value is taken to be an interpolation between the two // estimates. - velocityWall = velocityEstimate1 * assignedWallDistance + velocityEstimate2 - * (1. - assignedWallDistance); + velocityWall = velocityEstimate1 * assignedWallDistance + + velocityEstimate2 * (1. - assignedWallDistance); - fNeqWall = assignedWallDistance * fNeqEstimate1 + (1. - assignedWallDistance) - * fNeqEstimate2; + fNeqWall = assignedWallDistance * fNeqEstimate1 + + (1. - assignedWallDistance) * fNeqEstimate2; } else { @@ -550,8 +540,8 @@ namespace hemelb momentumWall.z, fEqm); // Perform collision on the wall f's - distribn_t prediction = fEqm[streamedDirection] + (1.0 + lbmParams->GetOmega()) - * fNeqWall; + distribn_t prediction = fEqm[streamedDirection] + + (1.0 + lbmParams->GetOmega()) * fNeqWall; // This is the answer from the code we're testing distribn_t streamedFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * chosenSite)[streamedDirection]; @@ -579,8 +569,8 @@ namespace hemelb // This is the first means of estimating from the source paper: only // use the nearest fluid site. - LatticeVelocity velocityWall = streamerHydroVars.momentum * (1. - 1. - / assignedWallDistance) / streamerHydroVars.density; + LatticeVelocity velocityWall = streamerHydroVars.momentum + * (1. - 1. / assignedWallDistance) / streamerHydroVars.density; distribn_t fNeqWall = streamerHydroVars.GetFNeq()[streamedDirection]; @@ -596,8 +586,8 @@ namespace hemelb fEqm); // Perform collision on the wall f's - distribn_t prediction = fEqm[streamedDirection] + (1.0 - + lbmParams->GetOmega()) * fNeqWall; + distribn_t prediction = fEqm[streamedDirection] + + (1.0 + lbmParams->GetOmega()) * fNeqWall; // This is the answer from the code we're testing distribn_t streamedFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * chosenSite)[streamedDirection]; @@ -613,7 +603,7 @@ namespace hemelb default: // We have nothing to do with a wall so simple streaming const site_t streamedIndex = - streamer.GetStreamedIndex (streamedDirection); + streamer.GetStreamedIndex(streamedDirection); distribn_t streamedToFNew = *latDat->GetFNew(streamedIndex); // F_new should be equal to the value that was streamed from this other site @@ -650,44 +640,42 @@ namespace hemelb site_t offset = 0; // Mid-Fluid sites use simple collide and stream - lb::streamers::SimpleCollideAndStream > > simpleCollideAndStream(initParams); - - simpleCollideAndStream.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(0), - lbmParams, - latDat, - *propertyCache); + lb::streamers::SimpleCollideAndStream< + lb::collisions::Normal > > simpleCollideAndStream(initParams); + + simpleCollideAndStream.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(0), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(0); // Wall sites use junk and yang initParams.siteRanges.push_back(std::pair(offset, offset + latDat->GetMidDomainCollisionCount(1))); - lb::streamers::JunkYang > >::Type - junkYang(initParams); + lb::streamers::JunkYang > >::Type junkYang(initParams); - junkYang.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(1), - lbmParams, - latDat, - *propertyCache); - - junkYang.PostStep (offset, + junkYang.StreamAndCollide(offset, latDat->GetMidDomainCollisionCount(1), lbmParams, latDat, *propertyCache); + junkYang.PostStep(offset, + latDat->GetMidDomainCollisionCount(1), + lbmParams, + latDat, + *propertyCache); + offset += latDat->GetMidDomainCollisionCount(1); // Consider inlet/outlets and their walls as mid-fluid sites - simpleCollideAndStream.StreamAndCollide (offset, - latDat->GetLocalFluidSiteCount() - - offset, - lbmParams, - latDat, - *propertyCache); + simpleCollideAndStream.StreamAndCollide(offset, + latDat->GetLocalFluidSiteCount() - offset, + lbmParams, + latDat, + *propertyCache); offset += latDat->GetLocalFluidSiteCount() - offset; // Sanity check @@ -700,7 +688,8 @@ namespace hemelb * depending on where they sit relative to the wall. We ignore mid-Fluid sites since * StreamAndCollide was tested before. */ - for (site_t wallSiteLocalIndex = 0; wallSiteLocalIndex < wallSitesCount; wallSiteLocalIndex++) + for (site_t wallSiteLocalIndex = 0; wallSiteLocalIndex < wallSitesCount; + wallSiteLocalIndex++) { site_t streamedToSite = firstWallSite + wallSiteLocalIndex; const geometry::Site streamedSite = @@ -708,18 +697,19 @@ namespace hemelb distribn_t* streamedToFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * streamedToSite); - for (unsigned int streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (unsigned int streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { unsigned oppDirection = lb::lattices::D3Q15::INVERSEDIRECTIONS[streamedDirection]; // Index of the site streaming to streamedToSite via direction streamedDirection site_t streamerIndex = - streamedSite.GetStreamedIndex (oppDirection); + streamedSite.GetStreamedIndex(oppDirection); // Is streamerIndex a valid index? - if (streamerIndex >= 0 && streamerIndex < (lb::lattices::D3Q15::NUMVECTORS - * latDat->GetLocalFluidSiteCount())) + if (streamerIndex >= 0 + && streamerIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { // The streamer index is a valid index in the domain, therefore stream and collide has happened site_t streamerSiteId = streamerIndex / lb::lattices::D3Q15::NUMVECTORS; @@ -730,8 +720,7 @@ namespace hemelb streamerFOld); // Calculate what the value streamed to site streamedToSite should be. - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamedSite); normalCollision->Collide(lbmParams, streamerHydroVars); @@ -752,8 +741,7 @@ namespace hemelb distribn_t streamerToSiteFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(streamedToSite, streamerToSiteFOld); - lb::kernels::HydroVars > - hydroVars(streamerToSiteFOld); + lb::kernels::HydroVars > hydroVars(streamerToSiteFOld); normalCollision->CalculatePreCollision(hydroVars, streamedSite); // Simulate post-collision using the collision operator. @@ -784,11 +772,11 @@ namespace hemelb initParams.boundaryObject = &inletBoundary; - lb::streamers::NashZerothOrderPressureIolet > >::Type ioletCollider(initParams); + lb::streamers::NashZerothOrderPressureIolet< + lb::collisions::Normal > >::Type ioletCollider(initParams); - for (double assignedWallDistance = 0.4; assignedWallDistance < 1.0; assignedWallDistance - += 0.5) + for (double assignedWallDistance = 0.4; assignedWallDistance < 1.0; + assignedWallDistance += 0.5) { // Initialise fOld in the lattice data. We choose values so that each site has // an anisotropic distribution function, and that each site's function is @@ -813,35 +801,31 @@ namespace hemelb latDat->SetIoletId(chosenSite, chosenBoundaryId); // Perform the collision and streaming. - ioletCollider.StreamAndCollide (chosenSite, - 1, - lbmParams, - latDat, - *propertyCache); + ioletCollider.StreamAndCollide(chosenSite, 1, lbmParams, latDat, *propertyCache); // Check each streamed direction. - for (Direction streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (Direction streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { // Calculate the distributions at the chosen site up to post-collision. distribn_t streamerFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(chosenSite, streamerFOld); - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamer); normalCollision->Collide(lbmParams, streamerHydroVars); // Calculate the streamed-to index. const site_t streamedIndex = - streamer.GetStreamedIndex (streamedDirection); + streamer.GetStreamedIndex(streamedDirection); // Check that simple collide and stream has happened when appropriate. // Is streamerIndex a valid index? (And is it not in one of the directions // that has been meddled with for the test)? - if (!streamer.HasIolet(streamedDirection) && streamedIndex >= 0 && streamedIndex - < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) + if (!streamer.HasIolet(streamedDirection) && streamedIndex >= 0 + && streamedIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { distribn_t streamedToFNew = *latDat->GetFNew(streamedIndex); @@ -898,11 +882,11 @@ namespace hemelb initParams.boundaryObject = &inletBoundary; - lb::streamers::NashZerothOrderPressureIoletSBB > >::Type ioletCollider(initParams); + lb::streamers::NashZerothOrderPressureIoletSBB< + lb::collisions::Normal > >::Type ioletCollider(initParams); - for (double assignedIoletDistance = 0.4; assignedIoletDistance < 1.0; assignedIoletDistance - += 0.5) + for (double assignedIoletDistance = 0.4; assignedIoletDistance < 1.0; + assignedIoletDistance += 0.5) { // Initialise fOld in the lattice data. We choose values so that each site has // an anisotropic distribution function, and that each site's function is @@ -929,36 +913,32 @@ namespace hemelb latDat->SetIoletId(chosenSite, chosenBoundaryId); // Perform the collision and streaming. - ioletCollider.StreamAndCollide (chosenSite, - 1, - lbmParams, - latDat, - *propertyCache); + ioletCollider.StreamAndCollide(chosenSite, 1, lbmParams, latDat, *propertyCache); // Check each streamed direction. - for (Direction streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (Direction streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { // Calculate the distributions at the chosen site up to post-collision. distribn_t streamerFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(chosenSite, streamerFOld); - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamer); normalCollision->Collide(lbmParams, streamerHydroVars); // Calculate the streamed-to index. const site_t streamedIndex = - streamer.GetStreamedIndex (streamedDirection); + streamer.GetStreamedIndex(streamedDirection); // Check that simple collide and stream has happened when appropriate. // Is streamerIndex a valid index? (And is it not in one of the directions // that has been meddled with for the test)? if (!streamer.HasIolet(streamedDirection) && !streamer.HasWall(streamedDirection) - && streamedIndex >= 0 && streamedIndex < (lb::lattices::D3Q15::NUMVECTORS - * latDat->GetLocalFluidSiteCount())) + && streamedIndex >= 0 + && streamedIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { distribn_t streamedToFNew = *latDat->GetFNew(streamedIndex); @@ -1022,7 +1002,7 @@ namespace hemelb lb::MacroscopicPropertyCache* propertyCache; lb::collisions::Normal >* normalCollision; }; - CPPUNIT_TEST_SUITE_REGISTRATION ( StreamerTests); + CPPUNIT_TEST_SUITE_REGISTRATION (StreamerTests); } } } diff --git a/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h b/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h index bc897741c..f96106153 100644 --- a/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h +++ b/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h @@ -35,7 +35,7 @@ namespace hemelb public: //typedef hemelb::lb::streamers::VSExtra::UnitVec UnitVec; LocalVSExtra(lb::iolets::InOutLet& iolet) : - VSExtra (iolet) + VSExtra(iolet) { } const UnitVec& GetE1() const @@ -61,11 +61,11 @@ namespace hemelb */ class VirtualSiteIoletStreamerTests : public helpers::FourCubeBasedTestFixture { - CPPUNIT_TEST_SUITE( VirtualSiteIoletStreamerTests); - CPPUNIT_TEST( TestVirtualSiteMatrixInverse); - CPPUNIT_TEST( TestVirtualSiteConstruction); - CPPUNIT_TEST( TestStreamerInitialisation); - CPPUNIT_TEST( TestStep);CPPUNIT_TEST_SUITE_END(); + CPPUNIT_TEST_SUITE (VirtualSiteIoletStreamerTests); + CPPUNIT_TEST (TestVirtualSiteMatrixInverse); + CPPUNIT_TEST (TestVirtualSiteConstruction); + CPPUNIT_TEST (TestStreamerInitialisation); + CPPUNIT_TEST (TestStep);CPPUNIT_TEST_SUITE_END(); public: typedef lb::lattices::D3Q15 Lattice; @@ -78,7 +78,7 @@ namespace hemelb InOutLetCosine* GetIolet(lb::iolets::BoundaryValues* iolets) { InOutLetCosine* ans = - dynamic_cast (iolets->GetLocalIolet(0)); + dynamic_cast(iolets->GetLocalIolet(0)); CPPUNIT_ASSERT(ans != NULL); return ans; } @@ -113,7 +113,7 @@ namespace hemelb for (unsigned j = 1; j <= 4; ++j) { site_t siteId = latDat->GetContiguousSiteId(LatticeVector(i, j, 1)); - geometry::Site < geometry::LatticeData > site = latDat->GetSite(siteId); + geometry::Site site = latDat->GetSite(siteId); for (Direction p = 0; p < lattice->GetNumVectors(); ++p) { if (lattice->GetVector(p).z < 0.) @@ -149,7 +149,7 @@ namespace hemelb for (unsigned j = 1; j <= 4; ++j) { site_t siteId = latDat->GetContiguousSiteId(LatticeVector(i, j, 4)); - geometry::Site < geometry::LatticeData > site = latDat->GetSite(siteId); + geometry::Site site = latDat->GetSite(siteId); for (Direction p = 0; p < lattice->GetNumVectors(); ++p) { if (lattice->GetVector(p).z > 0.) @@ -246,8 +246,8 @@ namespace hemelb // { 2.5, -2.5, 5.0 } }; // It's inverse is distribn_t velMatInv[3][3] = { { 0.25, 0., -0.125 }, { 0., 0.25, 0.125 }, { -0.125, - 0.125, - 0.325 } }; + 0.125, + 0.325 } }; for (unsigned i = 0; i < 3; ++i) { for (unsigned j = 0; j < 3; ++j) @@ -283,7 +283,7 @@ namespace hemelb // All the sites at the outlet plane (x, y, 3) should be in the cache. InOutLetCosine& outlet = *GetIolet(outletBoundary); - VSExtra* extra = dynamic_cast*> (outlet.GetExtraData()); + VSExtra* extra = dynamic_cast*>(outlet.GetExtraData()); CPPUNIT_ASSERT(extra != NULL); for (unsigned i = 1; i <= 4; ++i) @@ -301,8 +301,8 @@ namespace hemelb } // And the reverse is true: every cache entry should be a site at the outlet plane - for (RSHV::Map::iterator hvPtr = extra->hydroVarsCache.begin(); hvPtr - != extra->hydroVarsCache.end(); ++hvPtr) + for (RSHV::Map::iterator hvPtr = extra->hydroVarsCache.begin(); + hvPtr != extra->hydroVarsCache.end(); ++hvPtr) { site_t globalIdx = hvPtr->first; LatticeVector pos; @@ -330,32 +330,32 @@ namespace hemelb site_t offset = 0; offset += latDat->GetMidDomainCollisionCount(0); offset += latDat->GetMidDomainCollisionCount(1); - inletStreamer.DoStreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(2), - lbmParams, - static_cast (latDat), - *propertyCache); + inletStreamer.DoStreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(2), + lbmParams, + static_cast(latDat), + *propertyCache); offset += latDat->GetMidDomainCollisionCount(2); - outletStreamer.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(3), - lbmParams, - latDat, - *propertyCache); + outletStreamer.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(3), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(3); - inletStreamer.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(4), - lbmParams, - latDat, - *propertyCache); + inletStreamer.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(4), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(4); - outletStreamer.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(5), - lbmParams, - latDat, - *propertyCache); + outletStreamer.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(5), + lbmParams, + latDat, + *propertyCache); // Now every entry in the RSHV cache should have been updated CheckAllHVUpdated(inletBoundary, 1); @@ -365,39 +365,39 @@ namespace hemelb offset = 0; offset += latDat->GetMidDomainCollisionCount(0); offset += latDat->GetMidDomainCollisionCount(1); - inletStreamer.DoPostStep (offset, - latDat->GetMidDomainCollisionCount(2), - lbmParams, - static_cast (latDat), - *propertyCache); + inletStreamer.DoPostStep(offset, + latDat->GetMidDomainCollisionCount(2), + lbmParams, + static_cast(latDat), + *propertyCache); offset += latDat->GetMidDomainCollisionCount(2); - outletStreamer.DoPostStep (offset, - latDat->GetMidDomainCollisionCount(3), - lbmParams, - latDat, - *propertyCache); + outletStreamer.DoPostStep(offset, + latDat->GetMidDomainCollisionCount(3), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(3); - inletStreamer.DoPostStep (offset, - latDat->GetMidDomainCollisionCount(4), - lbmParams, - latDat, - *propertyCache); + inletStreamer.DoPostStep(offset, + latDat->GetMidDomainCollisionCount(4), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(4); - outletStreamer.DoPostStep (offset, - latDat->GetMidDomainCollisionCount(5), - lbmParams, - latDat, - *propertyCache); + outletStreamer.DoPostStep(offset, + latDat->GetMidDomainCollisionCount(5), + lbmParams, + latDat, + *propertyCache); // Check that all the vsites have sensible hydro values InOutLetCosine* inlet = GetIolet(inletBoundary); - VSExtra * inExtra = dynamic_cast*> (inlet->GetExtraData()); + VSExtra * inExtra = dynamic_cast*>(inlet->GetExtraData()); - for (VirtualSite::Map::iterator vsIt = inExtra->vSites.begin(); vsIt - != inExtra->vSites.end(); ++vsIt) + for (VirtualSite::Map::iterator vsIt = inExtra->vSites.begin(); + vsIt != inExtra->vSites.end(); ++vsIt) { site_t vSiteGlobalIdx = vsIt->first; VirtualSite& vSite = vsIt->second; @@ -414,10 +414,10 @@ namespace hemelb } InOutLetCosine* outlet = GetIolet(outletBoundary); - VSExtra * outExtra = dynamic_cast*> (outlet->GetExtraData()); + VSExtra * outExtra = dynamic_cast*>(outlet->GetExtraData()); - for (VirtualSite::Map::iterator vsIt = outExtra->vSites.begin(); vsIt - != outExtra->vSites.end(); ++vsIt) + for (VirtualSite::Map::iterator vsIt = outExtra->vSites.begin(); + vsIt != outExtra->vSites.end(); ++vsIt) { site_t vSiteGlobalIdx = vsIt->first; VirtualSite& vSite = vsIt->second; @@ -442,9 +442,9 @@ namespace hemelb void CheckAllHVUpdated(lb::iolets::BoundaryValues* iolets, LatticeTimeStep expectedT) { VSExtra * extra = - dynamic_cast*> (iolets->GetLocalIolet(0)->GetExtraData()); - for (RSHV::Map::iterator hvPtr = extra->hydroVarsCache.begin(); hvPtr - != extra->hydroVarsCache.end(); ++hvPtr) + dynamic_cast*>(iolets->GetLocalIolet(0)->GetExtraData()); + for (RSHV::Map::iterator hvPtr = extra->hydroVarsCache.begin(); + hvPtr != extra->hydroVarsCache.end(); ++hvPtr) { site_t siteGlobalIdx = hvPtr->first; LatticeVector sitePos; @@ -508,7 +508,7 @@ namespace hemelb latDat->SwapOldAndNew(); } }; - CPPUNIT_TEST_SUITE_REGISTRATION( VirtualSiteIoletStreamerTests); + CPPUNIT_TEST_SUITE_REGISTRATION (VirtualSiteIoletStreamerTests); } } } From e2fbbd431b08c7d569ab82487b1ca67c7c39cdb0 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 3 Dec 2015 16:33:32 +0000 Subject: [PATCH 27/99] tidy doc strings --- Code/lb/streamers/BaseStreamer.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Code/lb/streamers/BaseStreamer.h b/Code/lb/streamers/BaseStreamer.h index 345afb7d4..6d3218603 100644 --- a/Code/lb/streamers/BaseStreamer.h +++ b/Code/lb/streamers/BaseStreamer.h @@ -30,18 +30,18 @@ namespace hemelb * - Constructor(InitParams&) * - StreamAndCollide(const site_t, const site_t, const LbmParameters*, * geometry::LatticeData*, lb::MacroscopicPropertyCache&) - * - PostStep(const site_t, const site_t, const LbmParameters*, - * geometry::LatticeData*, hemelb::vis::Control*) + * - PostStep(const site_t, const site_t, const LbmParameters*, + * geometry::LatticeData*, lb::MacroscopicPropertyCache&) * - Reset(kernels::InitParams* init) * * The following must be implemented by concrete streamers (which derive from this class * using the CRTP). * - typedef for CollisionType, the type of the collider operation. * - Constructor(InitParams&) - * - DoStreamAndCollide(const site_t, const site_t, const LbmParameters*, - * geometry::LatticeData*, hemelb::vis::Control*) - * - DoPostStep(const site_t, const site_t, const LbmParameters*, - * geometry::LatticeData*, hemelb::vis::Control*) + * - DoStreamAndCollide(const site_t, const site_t, const LbmParameters*, + * geometry::LatticeData*, lb::MacroscopicPropertyCache&) + * - DoPostStep(const site_t, const site_t, const LbmParameters*, + * geometry::LatticeData*, lb::MacroscopicPropertyCache&) * - DoReset(kernels::InitParams* init) * * The design is to for the streamers to be pretty dumb and for them to From 4e9d6d48cb7f784f347d20e39bbfd0d0c37b5d7d Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 3 Dec 2015 16:37:58 +0000 Subject: [PATCH 28/99] remove vis section from xml --- .../pythontests/resources/poiseuille_flow_test_master.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Code/functionaltests/pythontests/resources/poiseuille_flow_test_master.xml b/Code/functionaltests/pythontests/resources/poiseuille_flow_test_master.xml index 3442aabfb..1fe5eb01c 100644 --- a/Code/functionaltests/pythontests/resources/poiseuille_flow_test_master.xml +++ b/Code/functionaltests/pythontests/resources/poiseuille_flow_test_master.xml @@ -18,12 +18,6 @@ - - - - - - From cc8655b5175ce29eee799936995b318e3d73c89e Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 4 Dec 2015 09:19:42 +0000 Subject: [PATCH 29/99] Remove images from Fabric et al --- deploy/fab.py | 13 +++++-------- deploy/templates/hemelb | 2 +- deploy/templates/multiscale_hemelb | 2 +- deploy/templates/regression | 3 +-- deploy/test/fixtures/templates/hemelb | 2 +- 5 files changed, 9 insertions(+), 13 deletions(-) diff --git a/deploy/fab.py b/deploy/fab.py index 62fe6b3ed..1da3b3ee1 100644 --- a/deploy/fab.py +++ b/deploy/fab.py @@ -498,7 +498,6 @@ def hemelb(config, **args): config : config directory to use to define geometry, e.g. config=cylinder Keyword arguments: cores : number of compute cores to request - images : number of images to take steering : steering session i.d. wall_time : wall-time job limit memory : memory per node @@ -506,7 +505,7 @@ def hemelb(config, **args): with_config(config) execute(put_configs, config) job(dict(script='hemelb', - cores=4, images=10, steering=1111, wall_time='0:15:0', memory='2G'), args) + cores=4, steering=1111, wall_time='0:15:0', memory='2G'), args) if args.get('steer', False): execute(steer, env.name, retry=True, framerate=args.get('framerate'), orbit=args.get('orbit')) @@ -518,7 +517,6 @@ def multiscale_hemelb(config,**args): config : config directory to use to define geometry, e.g. config=cylinder Keyword arguments: cores : number of compute cores to request - images : number of images to take steering : steering session i.d. wall_time : wall-time job limit memory : memory per node @@ -526,7 +524,7 @@ def multiscale_hemelb(config,**args): with_config(config) execute(put_configs,config) job(dict(script='multiscale_hemelb', - cores=4,images=10, steering=1111, wall_time='0:15:0',memory='2G'),args) + cores=4, steering=1111, wall_time='0:15:0',memory='2G'),args) if args.get('steer',False): execute(steer,env.name,retry=True,framerate=args.get('framerate'),orbit=args.get('orbit')) @@ -546,7 +544,7 @@ def multijob(*names, **args): env.jobstorun = "\n".join(jobscriptpaths) # And then, submit it job(dict(script='multijob', job_name_template='multijob', - cores=4, images=10, steering=1111, wall_time='0:15:0', memory='2G'), args) + cores=4, steering=1111, wall_time='0:15:0', memory='2G'), args) @task def hemelbs(config, **args): @@ -557,7 +555,6 @@ def hemelbs(config, **args): config : config directory to use to define geometry, e.g. config=cylinder Keyword arguments: cores : number of compute cores to request - images : number of images to take steering : steering session i.d. wall_time : wall-time job limit memory : memory per node @@ -578,7 +575,7 @@ def hemelb_benchmark(config, min_cores, max_cores, **args): with_config(config) execute(put_configs, config) job(dict(script='hemelb', - cores=cores_used, images=10, steering=1111, wall_time='0:15:0', memory='2G'), args) + cores=cores_used, steering=1111, wall_time='0:15:0', memory='2G'), args) cores_used *= 2 @task(alias='regress') @@ -588,7 +585,7 @@ def regression_test(**args): execute(copy_regression_tests) execute(build_python_tools) job(dict(job_name_template='regression_${build_number}_${machine_name}', cores=3, - wall_time='0:20:0', memory='2G', images=0, steering=1111, script='regression'), args) + wall_time='0:20:0', memory='2G', steering=1111, script='regression'), args) def calc_nodes(): # If we're not reserving whole nodes, then if we request less than one node's worth of cores, need to keep N<=n diff --git a/deploy/templates/hemelb b/deploy/templates/hemelb index 0ea44ac95..826d5b4fe 100644 --- a/deploy/templates/hemelb +++ b/deploy/templates/hemelb @@ -12,4 +12,4 @@ cd $job_results $run_prefix rm -rf results cp $job_config_path/* . -$run_command $install_path/bin/$executable -in config.xml -i $images +$run_command $install_path/bin/$executable -in config.xml diff --git a/deploy/templates/multiscale_hemelb b/deploy/templates/multiscale_hemelb index fafa1b703..c6afec67a 100644 --- a/deploy/templates/multiscale_hemelb +++ b/deploy/templates/multiscale_hemelb @@ -10,4 +10,4 @@ cd $job_results $run_prefix rm -rf results cp $job_config_path/* . -$run_command $install_path/bin/multiscale_hemelb -in config.xml -i $images -s $snapshots -ss $steering +$run_command $install_path/bin/multiscale_hemelb -in config.xml -s $snapshots -ss $steering diff --git a/deploy/templates/regression b/deploy/templates/regression index e90dd25f8..70f45b303 100644 --- a/deploy/templates/regression +++ b/deploy/templates/regression @@ -9,6 +9,5 @@ cd $job_results $run_prefix rm -rf results -$run_command $install_path/bin/hemelb -in $regression_test_path/config.xml -out $job_results/results -i $images -ss $steering -$run_command_one_proc $regression_test_path/ImageComparison $regression_test_path/CleanImages results/Images +$run_command $install_path/bin/hemelb -in $regression_test_path/config.xml -out $job_results/results -ss $steering $run_command_one_proc $regression_test_path/NumericalComparison $regression_test_path/CleanExtracted results/Extracted \ No newline at end of file diff --git a/deploy/test/fixtures/templates/hemelb b/deploy/test/fixtures/templates/hemelb index 2de0d1421..2998a4beb 100644 --- a/deploy/test/fixtures/templates/hemelb +++ b/deploy/test/fixtures/templates/hemelb @@ -2,4 +2,4 @@ cd $job_results $run_prefix rm -rf results cp $job_config_path/* . -mpirun -np $cores $install_path/bin/hemelb -in config.xml -i $images -ss $steering +mpirun -np $cores $install_path/bin/hemelb -in config.xml -ss $steering From 6a934cc1853517b4a7debff480f8cb3593a7e377 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 4 Dec 2015 09:30:04 +0000 Subject: [PATCH 30/99] Remove visim and python image reader --- Tools/hemeTools/image.py | 161 ----------- Tools/visim/.cproject | 304 -------------------- Tools/visim/.project | 79 ------ Tools/visim/Makefile | 20 -- Tools/visim/visualize_images.cc | 474 -------------------------------- 5 files changed, 1038 deletions(-) delete mode 100644 Tools/hemeTools/image.py delete mode 100644 Tools/visim/.cproject delete mode 100644 Tools/visim/.project delete mode 100644 Tools/visim/Makefile delete mode 100644 Tools/visim/visualize_images.cc diff --git a/Tools/hemeTools/image.py b/Tools/hemeTools/image.py deleted file mode 100644 index de7f5df13..000000000 --- a/Tools/hemeTools/image.py +++ /dev/null @@ -1,161 +0,0 @@ -# -# Copyright (C) University College London, 2007-2012, all rights reserved. -# -# This file is part of HemeLB and is CONFIDENTIAL. You may not work -# with, install, use, duplicate, modify, redistribute or share this -# file, or any part thereof, other than as allowed by any agreement -# specifically made by you with University College London. -# - -import pdb -import xdrlib -import numpy as N - -class Image(object): - - def __init__(self, filename): - self.filename = filename - - f = file(filename) - reader = xdrlib.Unpacker(f.read()) - self.mode = reader.unpack_uint() - self.pressure_threshold = (reader.unpack_float(), reader.unpack_float()) - self.velocity_max = reader.unpack_float() - self.stress_max = reader.unpack_float() - self.screen = (reader.unpack_uint(), reader.unpack_uint()) - self.nPixels = reader.unpack_uint() - - tempPixels = N.zeros((self.nPixels,4), dtype=N.uint32) - - for i in xrange(self.nPixels): - for j in xrange(4): - tempPixels[i,j] = reader.unpack_uint() - continue - continue - sortedIndices = N.argsort(tempPixels[:,0]) - self.pixels = tempPixels[sortedIndices].view(dtype=[('index', N.uint16, 2), - ('r', N.uint8, 4), - ('g', N.uint8, 4), - ('b', N.uint8, 4)]).view(N.recarray) - - # The above statement puts the pixel indices in the wrong order (i.e. j,i rather than i,j). - # This is corrected here. - self.pixels.index[:,0] = self.pixels.index[:, 0, [1,0]] - - return - - def old__init__(self, filename): - self.filename = filename - - f = file(filename) - reader = xdrlib.Unpacker(f.read()) - self.mode = reader.unpack_uint() - self.pressure_threshold = (reader.unpack_float(), reader.unpack_float()) - self.velocity_max = reader.unpack_float() - self.stress_max = reader.unpack_float() - self.screen = (reader.unpack_uint(), reader.unpack_uint()) - self.nPixels = reader.unpack_uint() - - tempPixels = N.zeros(self.nPixels, dtype=type(self).RowType) - - # Masks and shifts to get the bits for each vis mode - masks = [(2**32-1) ^ (2**24-1), - (2**24-1) ^ (2**16-1), - (2**16-1) ^ (2**8-1), - (2**8-1)] - shifts = [24, 16, 8, 0] - - conversions = zip(masks, shifts) - for i in xrange(self.nPixels): - ind = reader.unpack_uint() - tempPixels[i]['index'][0] = (ind & ((2**32-1) ^ (2**16-1))) >> 16 - tempPixels[i]['index'][1] = ind & (2**16-1) - r = reader.unpack_uint() - g = reader.unpack_uint() - b = reader.unpack_uint() - - for j, (mask, shift) in enumerate(conversions): - tempPixels[i]['r'][j] = (r & mask) >> shift - tempPixels[i]['g'][j] = (g & mask) >> shift - tempPixels[i]['b'][j] = (b & mask) >> shift - continue - - # assert r == ( - # (int(tempPixels[i]['r'][0]) << 24) + - # (int(tempPixels[i]['r'][1]) << 16) + - # (int(tempPixels[i]['r'][2]) << 8) + - # (int(tempPixels[i]['r'][3]) << 0) - # ) - # assert g == ( - # (int(tempPixels[i]['g'][0]) << 24) + - # (int(tempPixels[i]['g'][1]) << 16) + - # (int(tempPixels[i]['g'][2]) << 8) + - # (int(tempPixels[i]['g'][3]) << 0) - # ) - # assert b == ( - # (int(tempPixels[i]['b'][0]) << 24) + - # (int(tempPixels[i]['b'][1]) << 16) + - # (int(tempPixels[i]['b'][2]) << 8) + - # (int(tempPixels[i]['b'][3]) << 0) - # ) - - continue - sortedIndices = N.argsort(tempPixels['index'][:, 0]<<16 + tempPixels['index'][:, 1]) - self.pixels = tempPixels[sortedIndices] - - return - - def __eq__(self, other): - """Return True if the Images are identical. - """ - attrs = ('mode', 'pressure_threshold', 'velocity_max', 'stress_max', 'screen') - for at in attrs: - if not getattr(self, at) == getattr(other, at): - return False - continue - - if not N.alltrue(self.pixels.view(dtype=N.uint32) == other.pixels.view(dtype=N.uint32)): - return False - - return True - - def almost_eq(self, other, tol=1): - """Same as __eq__, except allow data array to differ by up to - tol (default 1). - """ - - attrs = ('mode', 'pressure_threshold', 'velocity_max', 'stress_max', 'screen') - for at in attrs: - if not getattr(self, at) == getattr(other, at): - print 'differed on ' + at + ': ' + str(getattr(self, at)) + ' and ' + str(getattr(other, at)) - return False - continue - - if not N.alltrue(self.pixels.index == other.pixels.index): - for i in range(self.pixels.index.shape[0]): - if not N.alltrue(self.pixels.index[i] == other.pixels.index[i]): - print 'Images differed on indices, first at index: ' + str(i) - print 'left array had ' + str(self.pixels.index[i]) - print 'right array had ' + str(other.pixels.index[i]) - return False - - for attr in ('r', 'g', 'b'): - d = self.pixels[attr] - other.pixels[attr] + 128 - if N.min(d) < (128-tol) or N.max(d) > (128+tol): - minArg = N.argmin(d, 0)[0][0] - maxArg = N.argmax(d, 0)[0][0] - print 'Differed on ' + attr + ' channel with delta range:' - print str(N.min(d) - 128) + ' between ' + str(self.pixels[minArg]) + ' and ' + str(other.pixels[minArg]) - print 'and' - print str(N.max(d) - 128) + ' between ' + str(self.pixels[maxArg]) + ' and ' + str(other.pixels[maxArg]) - return False - continue - - return True - - pass - -if __name__ == "__main__": - import sys - images = [Image(arg) for arg in sys.argv[1:]] - diff --git a/Tools/visim/.cproject b/Tools/visim/.cproject deleted file mode 100644 index 9150f6308..000000000 --- a/Tools/visim/.cproject +++ /dev/null @@ -1,304 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/visim/.project b/Tools/visim/.project deleted file mode 100644 index 8cc45ebb6..000000000 --- a/Tools/visim/.project +++ /dev/null @@ -1,79 +0,0 @@ - - - visim - - - - - - org.eclipse.cdt.managedbuilder.core.genmakebuilder - clean,full,incremental, - - - ?name? - - - - org.eclipse.cdt.make.core.append_environment - true - - - org.eclipse.cdt.make.core.autoBuildTarget - all - - - org.eclipse.cdt.make.core.buildArguments - - - - org.eclipse.cdt.make.core.buildCommand - make - - - org.eclipse.cdt.make.core.cleanBuildTarget - clean - - - org.eclipse.cdt.make.core.contents - org.eclipse.cdt.make.core.activeConfigSettings - - - org.eclipse.cdt.make.core.enableAutoBuild - false - - - org.eclipse.cdt.make.core.enableCleanBuild - true - - - org.eclipse.cdt.make.core.enableFullBuild - true - - - org.eclipse.cdt.make.core.fullBuildTarget - - - - org.eclipse.cdt.make.core.stopOnError - true - - - org.eclipse.cdt.make.core.useDefaultBuildCmd - true - - - - - org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder - full,incremental, - - - - - - org.eclipse.cdt.core.cnature - org.eclipse.cdt.core.ccnature - org.eclipse.cdt.managedbuilder.core.managedBuildNature - org.eclipse.cdt.managedbuilder.core.ScannerConfigNature - - diff --git a/Tools/visim/Makefile b/Tools/visim/Makefile deleted file mode 100644 index 32255d24e..000000000 --- a/Tools/visim/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -target = visim - -CC = g++ -ifeq ($(HEMELB_MACHINE),OSX) -CFLAGS = -framework OpenGL -framework GLUT -framework Foundation -O4 -DHEMELB_CFG_ON_OSX -else -CFLAGS = -O4 -L/usr/include/GT/ -lglut -lGLU -lGL -endif - -objects = visualize_images.o - -src = visualize_images.cc -$(target) : $(src) - $(CC) $(CFLAGS) $< -o $@ - - -.PHONY : clean - -clean : - -rm $(target) $(objects) diff --git a/Tools/visim/visualize_images.cc b/Tools/visim/visualize_images.cc deleted file mode 100644 index 9614de742..000000000 --- a/Tools/visim/visualize_images.cc +++ /dev/null @@ -1,474 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include -#include -#include -#ifdef HEMELB_CFG_ON_OSX -#include -#else -#include -#endif// HEMELB_CFG_OSX -#include -#include -#include -#include - -// g++ -framework OpenGL -framework GLUT -framework Foundation visualize_images.c -o visualize_images - -#define RAINBOW 0 -#define HALF_RAINBOW 1 -#define GREY 2 - -char *input_path; -char *output_path; - -unsigned char *image_data, *image_buffer; - -int pixels_x, pixels_y; -int images; -int image_count = 0; -int cycle_id = 1; - -int min(int a, int b) { - if (a < b) { - return a; - } else { - return b; - } -} - -int max(int a, int b) { - if (a > b) { - return a; - } else { - return b; - } -} - -void visRainbowPalette(int id, float col[3]) { - if (id == 0) { - col[0] = 0.F; - col[1] = 0.F; - col[2] = 1.F; - } else if (id == 1) { - col[0] = 0.F; - col[1] = 1.F; - col[2] = 1.F; - } else if (id == 2) { - col[0] = 0.F; - col[1] = 1.F; - col[2] = 0.F; - } else if (id == 3) { - col[0] = 1.F; - col[1] = 1.F; - col[2] = 0.F; - } else if (id == 4) { - col[0] = 1.F; - col[1] = 0.F; - col[2] = 0.F; - } -} - -void OpenWindow(int pixels_x, int pixels_y) { - glutInitDisplayMode(GLUT_RGB | GLUT_DOUBLE); - glutInitWindowPosition(0, 0); - glutInitWindowSize(pixels_x - 1, pixels_y - 1); - - glutCreateWindow(" "); - - glDisable (GL_DEPTH_TEST); - glDisable (GL_BLEND); - glShadeModel (GL_SMOOTH); - glDisable (GL_DITHER); - - glClear (GL_COLOR_BUFFER_BIT); -} - -void Projection(int pixels_x, int pixels_y) { - glLoadIdentity(); - gluOrtho2D(0.F, (float) (pixels_x - 1), - 0.F, (float)(pixels_y - 1)); - -glClearColor (1.0F, 1.0F, 1.0F, 0.0F); -} - -void DisplayFrame(int pixels_x, int pixels_y) { - for (int j = 0; j < pixels_y; j++) { - for (int i = 0; i < pixels_x; i++) { - if (i > 3 && i <= pixels_x - 3 && j > 3 && j <= pixels_y - 3) - continue; - - image_data[(j * pixels_x + i) * 3 + 0] = 0; - image_data[(j * pixels_x + i) * 3 + 1] = 0; - image_data[(j * pixels_x + i) * 3 + 2] = 0; - } - } -} - -void visVisualiseString(float r, float g, float b, float x, float y, - char *string, void *font) { - glColor3f(r, g, b); - glRasterPos2f(x, y); - - for (int i = 0; i < (int) strlen(string); i++) { - glutBitmapCharacter(font, string[i]); - } - glEnd(); -} - -void visVisualiseBarPars(char *unit, float threshold_min, float threshold_max, - float x_min, float y_min, float x_max) { - float dx = 0.25F * (x_max - x_min); - - char string[256]; - - for (int i = 0; i <= 4; i++) { - sprintf(string, "%.2f", - threshold_min + 0.25 * i * (threshold_max - threshold_min)); - - visVisualiseString(0.F, 0.F, 0.F, x_min + i * dx - 20.0F, y_min - 16.0F, - string, GLUT_BITMAP_HELVETICA_18); - } - visVisualiseString(0.F, 0.F, 0.F, x_max + 45, y_min - 16.0F, unit, - GLUT_BITMAP_HELVETICA_18); -} - -void visVisualiseColourBar(int type, float x_min, float y_min, float x_max, - float y_max) { - float dx = 0.25F * (x_max - x_min); - float col[3], grey; - - int edge = 0; - - glBegin (GL_QUAD_STRIP); - - for (int i = 0; i <= 4; i++) { - if (type == RAINBOW) - { - visRainbowPalette(edge++, col); - glColor3fv(col); - } else if (type == HALF_RAINBOW) - { - visRainbowPalette(edge++, col); - - for (int l = 0; l < 3; l++) - col[l] = 0.5F * (1.0F + col[l]); - glColor3fv(col); - } else { - grey = edge++ / (float) 8; - glColor3f(grey, grey, grey); - } - glVertex2f(x_min + i * dx, y_min); - glVertex2f(x_min + i * dx, y_max); - } - glEnd(); - - glBegin (GL_LINES); - - for (int i = 0; i <= 4; i++) { - if (type == GREY) - { - glColor3f(1.0F, 1.0F, 1.0F); - } else { - glColor3f(0.0F, 0.0F, 0.0F); - } - glVertex2f(x_min + i * dx, y_min); - glVertex2f(x_min + i * dx, y_max); - } - glEnd(); -} - -void SaveWindowImage(int pixels_x, int pixels_y, char *file_name) { - FILE *ppm_image_file_ptr = fopen(file_name, "wb"); - - int i, j; - - unsigned char *data_p = NULL; - - glReadBuffer (GL_FRONT); - - data_p = image_data; - - for (j = 0; j < pixels_y; j++) { - glReadPixels(0, j, pixels_x, 1, GL_RGB, GL_UNSIGNED_BYTE, image_buffer); - - for (i = 0; i < pixels_x; i++) { - *data_p = image_buffer[i * 3]; - data_p++; - *data_p = image_buffer[i * 3 + 1]; - data_p++; - *data_p = image_buffer[i * 3 + 2]; - data_p++; - } - } - fprintf(ppm_image_file_ptr, "P6\n%i %i\n255\n", pixels_x, pixels_y); - - for (j = pixels_y - 1; j >= 0; j--) { - fwrite(image_data + j * pixels_x * 3, 1, pixels_x * 3, - ppm_image_file_ptr); - } - fclose(ppm_image_file_ptr); -} - -void Display(void) { - FILE *input_image_file; - FILE *temp_file_ptr; - XDR xdr_image_file; - - float pressure_threshold_min, pressure_threshold_max; - float velocity_threshold_max, stress_threshold_max; - - int mode; - int bits_per_char = sizeof(char) * 8; - int colour_mask = (1 << (sizeof(char) * 8)) - 1; - int pixel_mask = (1 << (sizeof(char) * 16)) - 1; - int col_pixels; - int pixel_i, pixel_j; - int palette_type; - int dummy; - int i, n; - - unsigned int pixel_id; - unsigned int col_data[3]; - - unsigned char pixel_r[4], pixel_g[4], pixel_b[4]; - - char partial_image_name[256], input_image_name[256], output_image_name[256]; - - char *pressure_unit = "(mmHg)"; - char *velocity_unit = "(m/s)"; - char *stress_unit = "(Pa)"; - - glClear (GL_COLOR_BUFFER_BIT); - - glPointSize(1.F); - glBegin (GL_POINTS); - - temp_file_ptr = fopen("temp_file.txt", "r"); - - for (i = -1; i < image_count; i++) { - fscanf(temp_file_ptr, "%s\n", partial_image_name); - } - fclose(temp_file_ptr); - - sprintf(input_image_name, "%s/%s", input_path, partial_image_name); - - std::string lImageNameNoExtension = std::string(partial_image_name).substr( - 0, std::string(partial_image_name).rfind('.')); - - sprintf(output_image_name, "%s/%s.ppm", output_path, - lImageNameNoExtension.c_str()); - - input_image_file = fopen(input_image_name, "r"); - xdrstdio_create(&xdr_image_file, input_image_file, XDR_DECODE); - - xdr_int(&xdr_image_file, &mode); - xdr_float(&xdr_image_file, &pressure_threshold_min); - xdr_float(&xdr_image_file, &pressure_threshold_max); - xdr_float(&xdr_image_file, &velocity_threshold_max); - xdr_float(&xdr_image_file, &stress_threshold_max); - - xdr_int(&xdr_image_file, &dummy); - xdr_int(&xdr_image_file, &dummy); - xdr_int(&xdr_image_file, &col_pixels); - - for (n = 0; n < col_pixels; n++) { - xdr_u_int(&xdr_image_file, &pixel_id); - - xdr_u_int(&xdr_image_file, &col_data[0]); - xdr_u_int(&xdr_image_file, &col_data[1]); - xdr_u_int(&xdr_image_file, &col_data[2]); - - pixel_i = (pixel_id >> (2 * bits_per_char)) & pixel_mask; - pixel_j = (pixel_id) & pixel_mask; - - pixel_r[0] = (col_data[0] >> (3 * bits_per_char)) & colour_mask; - pixel_g[0] = (col_data[0] >> (2 * bits_per_char)) & colour_mask; - pixel_b[0] = (col_data[0] >> (1 * bits_per_char)) & colour_mask; - pixel_r[1] = (col_data[0] >> (0 * bits_per_char)) & colour_mask; - pixel_g[1] = (col_data[1] >> (3 * bits_per_char)) & colour_mask; - pixel_b[1] = (col_data[1] >> (2 * bits_per_char)) & colour_mask; - pixel_r[2] = (col_data[1] >> (1 * bits_per_char)) & colour_mask; - pixel_g[2] = (col_data[1] >> (0 * bits_per_char)) & colour_mask; - pixel_b[2] = (col_data[2] >> (3 * bits_per_char)) & colour_mask; - pixel_r[3] = (col_data[2] >> (2 * bits_per_char)) & colour_mask; - pixel_g[3] = (col_data[2] >> (1 * bits_per_char)) & colour_mask; - pixel_b[3] = (col_data[2] >> (0 * bits_per_char)) & colour_mask; - - glColor3f(pixel_r[0] * (1.F / 255.F), pixel_g[0] * (1.F / 255.F), - pixel_b[0] * (1.F / 255.F)); - glVertex2f(pixel_i, pixel_j + (pixels_y >> 1)); - - glColor3f(pixel_r[1] * (1.F / 255.F), pixel_g[1] * (1.F / 255.F), - pixel_b[1] * (1.F / 255.F)); - glVertex2f(pixel_i + (pixels_x >> 1), pixel_j + (pixels_y >> 1)); - - glColor3f(pixel_r[2] * (1.F / 255.F), pixel_g[2] * (1.F / 255.F), - pixel_b[2] * (1.F / 255.F)); - glVertex2f(pixel_i, pixel_j); - - glColor3f(pixel_r[3] * (1.F / 255.F), pixel_g[3] * (1.F / 255.F), - pixel_b[3] * (1.F / 255.F)); - - glVertex2f(pixel_i + (pixels_x >> 1), pixel_j); - } - glEnd(); - - xdr_destroy(&xdr_image_file); - fclose(input_image_file); - - visVisualiseColourBar(RAINBOW, pixels_x * 0.1F, pixels_y * 0.5F + 20.F, - pixels_x * 0.4F, pixels_y * 0.5F + 40.F); - visVisualiseBarPars(velocity_unit, 0.0F, velocity_threshold_max, - pixels_x * 0.1F, pixels_y * 0.5F + 20.F, pixels_x * 0.4F); - - visVisualiseColourBar(RAINBOW, pixels_x * 0.6F, pixels_y * 0.5F + 20.F, - pixels_x * 0.9F, pixels_y * 0.5F + 40.F); - visVisualiseBarPars(stress_unit, 0.0F, stress_threshold_max, - pixels_x * 0.6F, pixels_y * 0.5F + 20.F, pixels_x * 0.9F); - - if (mode == 0) { - palette_type = RAINBOW; - } else if (mode == 1) { - palette_type = HALF_RAINBOW; - } else { - palette_type = GREY; - } - visVisualiseColourBar(palette_type, pixels_x * 0.1F, 20.F, pixels_x * 0.4F, - 40.F); - visVisualiseBarPars(pressure_unit, pressure_threshold_min, - pressure_threshold_max, pixels_x * 0.1F, 20.F, pixels_x * 0.4F); - - visVisualiseColourBar(palette_type, pixels_x * 0.6F, 20.F, pixels_x * 0.9F, - 40.F); - visVisualiseBarPars(stress_unit, 0.0F, stress_threshold_max, - pixels_x * 0.6F, 20.F, pixels_x * 0.9F); - - glutSwapBuffers(); - - if (cycle_id <= 2) { - SaveWindowImage(pixels_x, pixels_y, output_image_name); - } - if (++image_count == images - 1) { - ++cycle_id; - image_count = 0; - } -} - -void KeybordFunction(unsigned char key, int x, int y) { - if (key == 'q') { - free(image_buffer); - free(image_data); - - exit(0); - } -} - -void Reshape(GLsizei w, GLsizei h) { - //Projection (pixels_x, pixels_y); -} - -void usage(char *progname) { - printf( - "Usage: %s <'x' if you want to view images> \n", - progname); -} - -int main(int argc, char *argv[]) { - int required_args = 3; - - if (argc < required_args) { - usage(argv[0]); - exit(1); - } - - input_path = argv[1]; - output_path = argv[2]; - - FILE *image_file; - FILE *temp_file_ptr; - XDR xdr_image_file; - - int mode; - - float pressure_threshold_min, pressure_threshold_max; - float velocity_threshold_max, stress_threshold_max; - - char partial_image_name[256], image_name[256]; - char *first_command_part, *last_command_part; - char my_command[256]; - - first_command_part = "ls -l"; - last_command_part = "| wc > temp_file.txt"; - - sprintf(my_command, "%s %s %s", first_command_part, input_path, - last_command_part); - system(my_command); - - temp_file_ptr = fopen("temp_file.txt", "r"); - fscanf(temp_file_ptr, "%i ", &images); - fclose(temp_file_ptr); - - temp_file_ptr = fopen("temp_file.txt", "r"); - - first_command_part = "ls -1"; - last_command_part = "> temp_file.txt"; - - sprintf(my_command, "%s %s %s", first_command_part, input_path, - last_command_part); - system(my_command); - - rewind(temp_file_ptr); - - fscanf(temp_file_ptr, "%s\n", partial_image_name); - sprintf(image_name, "%s/%s", input_path, partial_image_name); - - fclose(temp_file_ptr); - - image_file = fopen(image_name, "r"); - xdrstdio_create(&xdr_image_file, image_file, XDR_DECODE); - - xdr_int(&xdr_image_file, &mode); - xdr_float(&xdr_image_file, &pressure_threshold_min); - xdr_float(&xdr_image_file, &pressure_threshold_max); - xdr_float(&xdr_image_file, &velocity_threshold_max); - xdr_float(&xdr_image_file, &stress_threshold_max); - - xdr_int(&xdr_image_file, &pixels_x); - xdr_int(&xdr_image_file, &pixels_y); - - pixels_x *= 2; - pixels_y *= 2; - - xdr_destroy(&xdr_image_file); - fclose(image_file); - - if(argc == 4 && argv[3][0] == 'x' && argv[3][1] == '\0') - { - glutInit(&argc, argv); - OpenWindow(pixels_x, pixels_y); - - Projection(pixels_x, pixels_y); - - image_data = (unsigned char *) malloc( - sizeof(unsigned char) * pixels_x * pixels_y * 3); - - image_buffer = (unsigned char *) malloc( - sizeof(unsigned char) * pixels_x * 3); - - //glutReshapeFunc (Reshape); - glutIdleFunc(Display); - glutDisplayFunc(Display); - glutKeyboardFunc(KeybordFunction); - glutMainLoop(); - } - - return (0); -} From 16409a6fc04216431c0a529eaa911b111dd23d25 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 4 Dec 2015 10:13:58 +0000 Subject: [PATCH 31/99] remove dangling .project file --- Tools/visclients/.project | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 Tools/visclients/.project diff --git a/Tools/visclients/.project b/Tools/visclients/.project deleted file mode 100644 index 1ce174429..000000000 --- a/Tools/visclients/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - CLINICAL-GUI - - - - - - org.eclipse.jdt.core.javabuilder - - - - - - org.eclipse.jdt.core.javanature - - From e9bd7b09d158d0387c809392440c43731e6b627d Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 4 Dec 2015 10:34:01 +0000 Subject: [PATCH 32/99] Remove the min and max density and pressure members from iolets etc --- Code/lb/iolets/BoundaryValues.cc | 10 ------- Code/lb/iolets/BoundaryValues.h | 3 -- Code/lb/iolets/InOutLet.h | 30 ------------------- Code/lb/iolets/InOutLetCosine.h | 9 ------ Code/lb/iolets/InOutLetFile.cc | 9 ------ Code/lb/iolets/InOutLetFile.h | 10 ------- Code/lb/iolets/InOutLetFileVelocity.cc | 5 ---- Code/lb/iolets/InOutLetMultiscale.cc | 8 ----- Code/lb/iolets/InOutLetMultiscale.h | 2 -- Code/lb/iolets/InOutLetVelocity.cc | 8 ----- Code/lb/iolets/InOutLetVelocity.h | 2 -- Code/lb/lb.h | 2 ++ Code/lb/lb.hpp | 26 +++++++--------- Code/multiscale/MultiscaleSimulationMaster.h | 26 +++++----------- Code/unittests/lbtests/iolets/InOutLetTests.h | 24 +++++---------- 15 files changed, 28 insertions(+), 146 deletions(-) diff --git a/Code/lb/iolets/BoundaryValues.cc b/Code/lb/iolets/BoundaryValues.cc index f7e50b72e..04b8a4f96 100644 --- a/Code/lb/iolets/BoundaryValues.cc +++ b/Code/lb/iolets/BoundaryValues.cc @@ -187,16 +187,6 @@ namespace hemelb return iolets[index]->GetDensity(state->Get0IndexedTimeStep()); } - LatticeDensity BoundaryValues::GetDensityMin(int iBoundaryId) - { - return iolets[iBoundaryId]->GetDensityMin(); - } - - LatticeDensity BoundaryValues::GetDensityMax(int iBoundaryId) - { - return iolets[iBoundaryId]->GetDensityMax(); - } - } } } diff --git a/Code/lb/iolets/BoundaryValues.h b/Code/lb/iolets/BoundaryValues.h index 07881c313..570be5533 100644 --- a/Code/lb/iolets/BoundaryValues.h +++ b/Code/lb/iolets/BoundaryValues.h @@ -42,9 +42,6 @@ namespace hemelb LatticeDensity GetBoundaryDensity(const int index); - LatticeDensity GetDensityMin(int boundaryId); - LatticeDensity GetDensityMax(int boundaryId); - static proc_t GetBCProcRank(); iolets::InOutLet* GetLocalIolet(unsigned int index) { diff --git a/Code/lb/iolets/InOutLet.h b/Code/lb/iolets/InOutLet.h index 42d1d534c..36472fe17 100644 --- a/Code/lb/iolets/InOutLet.h +++ b/Code/lb/iolets/InOutLet.h @@ -114,36 +114,6 @@ namespace hemelb { } - /*** - * Get the minimum density, in lattice units - * @return minimum density, in lattice units - */ - virtual LatticeDensity GetDensityMin() const = 0; - - /*** - * Get the maximum density, in lattice units - * @return maximum density, in lattice units - */ - virtual LatticeDensity GetDensityMax() const = 0; - - /*** - * Get the minimum pressure, in lattice units - * @return - */ - LatticePressure GetPressureMin() const - { - return GetDensityMin() * Cs2; - } - - /*** - * Get the maximum pressure, in lattice units - * @return - */ - LatticePressure GetPressureMax() const - { - return GetDensityMax() * Cs2; - } - /// @todo: #632 This method must be moved to InOutletPressure virtual LatticeDensity GetDensity(LatticeTimeStep time_step) const = 0; diff --git a/Code/lb/iolets/InOutLetCosine.h b/Code/lb/iolets/InOutLetCosine.h index a6f6e1e5c..47ef57221 100644 --- a/Code/lb/iolets/InOutLetCosine.h +++ b/Code/lb/iolets/InOutLetCosine.h @@ -55,15 +55,6 @@ namespace hemelb densityAmp = rho; } - LatticeDensity GetDensityMin() const - { - return (densityMean - densityAmp); - } - LatticeDensity GetDensityMax() const - { - return (densityMean + densityAmp); - } - LatticePressure GetPressureMean() const { return densityMean * Cs2; diff --git a/Code/lb/iolets/InOutLetFile.cc b/Code/lb/iolets/InOutLetFile.cc index 286e649d1..981a4df51 100644 --- a/Code/lb/iolets/InOutLetFile.cc +++ b/Code/lb/iolets/InOutLetFile.cc @@ -73,9 +73,6 @@ namespace hemelb std::vector values(0); // Must convert into vectors since LinearInterpolate works on a pair of vectors - // Determine min and max pressure on the way - PhysicalPressure pMin = timeValuePairs.begin()->second; - PhysicalPressure pMax = timeValuePairs.begin()->second; for (std::map::iterator entry = timeValuePairs.begin(); entry != timeValuePairs.end(); entry++) { @@ -90,19 +87,13 @@ namespace hemelb PhysicalSpeed final_pressure = values.back() + time_diff_ratio * pres_diff; times.push_back(totalTimeSteps*timeStepLength); - pMin = util::NumericalFunctions::min(pMin, final_pressure); - pMax = util::NumericalFunctions::max(pMax, final_pressure); values.push_back(final_pressure); break; } - pMin = util::NumericalFunctions::min(pMin, entry->second); - pMax = util::NumericalFunctions::max(pMax, entry->second); times.push_back(entry->first); values.push_back(entry->second); } - densityMin = units->ConvertPressureToLatticeUnits(pMin) / Cs2; - densityMax = units->ConvertPressureToLatticeUnits(pMax) / Cs2; // Check if last point's value matches the first if (values.back() != values.front()) diff --git a/Code/lb/iolets/InOutLetFile.h b/Code/lb/iolets/InOutLetFile.h index 8efa75320..d9c08fa8f 100644 --- a/Code/lb/iolets/InOutLetFile.h +++ b/Code/lb/iolets/InOutLetFile.h @@ -48,14 +48,6 @@ namespace hemelb pressureFilePath = path; } - LatticeDensity GetDensityMin() const - { - return densityMin; - } - LatticeDensity GetDensityMax() const - { - return densityMax; - } LatticeDensity GetDensity(LatticeTimeStep timeStep) const { return densityTable[timeStep]; @@ -64,8 +56,6 @@ namespace hemelb private: void CalculateTable(LatticeTimeStep totalTimeSteps, PhysicalTime timeStepLength); std::vector densityTable; - LatticeDensity densityMin; - LatticeDensity densityMax; std::string pressureFilePath; const util::UnitConverter* units; }; diff --git a/Code/lb/iolets/InOutLetFileVelocity.cc b/Code/lb/iolets/InOutLetFileVelocity.cc index f6c8917d8..f18b61c91 100644 --- a/Code/lb/iolets/InOutLetFileVelocity.cc +++ b/Code/lb/iolets/InOutLetFileVelocity.cc @@ -60,9 +60,6 @@ namespace hemelb std::vector values(0); // Must convert into vectors since LinearInterpolate works on a pair of vectors - // Determine min and max pressure on the way -// PhysicalPressure pMin = timeValuePairs.begin()->second; -// PhysicalPressure pMax = timeValuePairs.begin()->second; for (std::map::iterator entry = timeValuePairs.begin(); entry != timeValuePairs.end(); entry++) { @@ -85,8 +82,6 @@ namespace hemelb times.push_back(entry->first); values.push_back(entry->second); } -// densityMin = units->ConvertPressureToLatticeUnits(pMin) / Cs2; -// densityMax = units->ConvertPressureToLatticeUnits(pMax) / Cs2; /* If the time values in the input file end BEFORE the planned end of the simulation, then loop the profile afterwards (using %TimeStepsInInletVelocityProfile). */ int TimeStepsInInletVelocityProfile = times.back() / timeStepLength; diff --git a/Code/lb/iolets/InOutLetMultiscale.cc b/Code/lb/iolets/InOutLetMultiscale.cc index 325a09376..76489d48f 100644 --- a/Code/lb/iolets/InOutLetMultiscale.cc +++ b/Code/lb/iolets/InOutLetMultiscale.cc @@ -74,14 +74,6 @@ namespace hemelb /* TODO: Fix pressure and GetPressure values (using PressureMax() for now). */ return units->ConvertPressureToLatticeUnits(maxPressure.GetPayload()) / Cs2; } - LatticeDensity InOutLetMultiscale::GetDensityMin() const - { - return units->ConvertPressureToLatticeUnits(minPressure.GetPayload()) / Cs2; - } - LatticeDensity InOutLetMultiscale::GetDensityMax() const - { - return units->ConvertPressureToLatticeUnits(maxPressure.GetPayload()) / Cs2; - } PhysicalVelocity InOutLetMultiscale::GetVelocity() const { return velocity; diff --git a/Code/lb/iolets/InOutLetMultiscale.h b/Code/lb/iolets/InOutLetMultiscale.h index c26607792..0b22a928f 100644 --- a/Code/lb/iolets/InOutLetMultiscale.h +++ b/Code/lb/iolets/InOutLetMultiscale.h @@ -58,8 +58,6 @@ namespace hemelb // field points will map one-to-one to HemeLB lattice sites initially; we will // rely on external tools to perform any interpolation tasks. LatticeDensity GetDensity(unsigned long timeStep) const; - virtual LatticeDensity GetDensityMin() const; - virtual LatticeDensity GetDensityMax() const; PhysicalVelocity GetVelocity() const; LatticePressure GetPressure() const; //std::vector void LBM::PrepareBoundaryObjects() { - // First, iterate through all of the inlet and outlet objects, finding out the minimum density seen in the simulation. - distribn_t minDensity = std::numeric_limits::max(); - + LatticeDensity initialDensity = GetInitialDensity(); + // Now go through iolets, informing them of the minimum density. for (unsigned inlet = 0; inlet < mInletValues->GetLocalIoletCount(); ++inlet) { - minDensity = std::min(minDensity, mInletValues->GetLocalIolet(inlet)->GetDensityMin()); + mInletValues->GetLocalIolet(inlet)->SetMinimumSimulationDensity(initialDensity); } for (unsigned outlet = 0; outlet < mOutletValues->GetLocalIoletCount(); ++outlet) { - minDensity = std::min(minDensity, mOutletValues->GetLocalIolet(outlet)->GetDensityMin()); - } - - // Now go through them again, informing them of the minimum density. - for (unsigned inlet = 0; inlet < mInletValues->GetLocalIoletCount(); ++inlet) - { - mInletValues->GetLocalIolet(inlet)->SetMinimumSimulationDensity(minDensity); + mOutletValues->GetLocalIolet(outlet)->SetMinimumSimulationDensity(initialDensity); } + } - for (unsigned outlet = 0; outlet < mOutletValues->GetLocalIoletCount(); ++outlet) - { - mOutletValues->GetLocalIolet(outlet)->SetMinimumSimulationDensity(minDensity); - } + template + LatticeDensity LBM::GetInitialDensity() const + { + return mUnits->ConvertPressureToLatticeUnits(mSimConfig->GetInitialPressure()) / Cs2; } template void LBM::SetInitialConditions() { - distribn_t density = mUnits->ConvertPressureToLatticeUnits(mSimConfig->GetInitialPressure()) / Cs2; + distribn_t density = GetInitialDensity(); for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) { diff --git a/Code/multiscale/MultiscaleSimulationMaster.h b/Code/multiscale/MultiscaleSimulationMaster.h index bdb2d666e..d9540c1ef 100644 --- a/Code/multiscale/MultiscaleSimulationMaster.h +++ b/Code/multiscale/MultiscaleSimulationMaster.h @@ -38,13 +38,9 @@ namespace hemelb inletValues->GetLocalIoletCount(), outletValues->GetLocalIoletCount()); hemelb::log::Logger::Log("inlets: %d", - inletValues->GetLocalIolet(0)->IsCommsRequired(), - inletValues->GetLocalIolet(0)->GetDensityMax(), - inletValues->GetLocalIolet(0)->GetPressureMax()); + inletValues->GetLocalIolet(0)->IsCommsRequired()); hemelb::log::Logger::Log("outlets: %d", - outletValues->GetLocalIolet(0)->IsCommsRequired(), - outletValues->GetLocalIolet(0)->GetDensityMax(), - outletValues->GetLocalIolet(0)->GetPressureMax()); + outletValues->GetLocalIolet(0)->IsCommsRequired()); // we only want to register those iolets which are needed on this process. // Fortunately, the BoundaryValues instance has worked this out for us. @@ -236,13 +232,9 @@ namespace hemelb inletValues->GetLocalIoletCount(), outletValues->GetLocalIoletCount()); hemelb::log::Logger::Log("inlets: %d", - inletValues->GetLocalIolet(0)->IsCommsRequired(), - inletValues->GetLocalIolet(0)->GetDensityMax(), - inletValues->GetLocalIolet(0)->GetPressureMax()); + inletValues->GetLocalIolet(0)->IsCommsRequired()); hemelb::log::Logger::Log("outlets: %d", - outletValues->GetLocalIolet(0)->IsCommsRequired(), - outletValues->GetLocalIolet(0)->GetDensityMax(), - outletValues->GetLocalIolet(0)->GetPressureMax()); + outletValues->GetLocalIolet(0)->IsCommsRequired()); SetCommsRequired(inletValues, true); SetCommsRequired(outletValues, true); @@ -254,17 +246,15 @@ namespace hemelb for (unsigned int i = 0; i < inletValues->GetLocalIoletCount(); i++) { - hemelb::log::Logger::Log("Inlet[%i]: Measured Density is %f. Pressure is %f.", + hemelb::log::Logger::Log("Inlet[%i]: Measured Density is %f.", i, - inletValues->GetLocalIolet(i)->GetDensity(GetState()->GetTimeStep()), - inletValues->GetLocalIolet(i)->GetPressureMax()); + inletValues->GetLocalIolet(i)->GetDensity(GetState()->GetTimeStep())); } for (unsigned int i = 0; i < outletValues->GetLocalIoletCount(); i++) { - hemelb::log::Logger::Log("Outlet[%i]: Measured Density is %f. Pressure is %f.", + hemelb::log::Logger::Log("Outlet[%i]: Measured Density is %f.", i, - outletValues->GetLocalIolet(i)->GetDensity(GetState()->GetTimeStep()), - outletValues->GetLocalIolet(i)->GetPressureMax()); + outletValues->GetLocalIolet(i)->GetDensity(GetState()->GetTimeStep())); } /* Temporary Orchestration hardcode for testing 1/100 step ratio diff --git a/Code/unittests/lbtests/iolets/InOutLetTests.h b/Code/unittests/lbtests/iolets/InOutLetTests.h index ce55c664c..d19c9672b 100644 --- a/Code/unittests/lbtests/iolets/InOutLetTests.h +++ b/Code/unittests/lbtests/iolets/InOutLetTests.h @@ -140,12 +140,12 @@ namespace hemelb // Ok, now we have an inlet, check the values are right. CPPUNIT_ASSERT_EQUAL(std::string("./iolet.txt"), file->GetFilePath()); - CPPUNIT_ASSERT_DOUBLES_EQUAL(78.0, - converter.ConvertPressureToPhysicalUnits(file->GetPressureMin()), - 1e-6); - CPPUNIT_ASSERT_DOUBLES_EQUAL(82.0, - converter.ConvertPressureToPhysicalUnits(file->GetPressureMax()), - 1e-6); +// CPPUNIT_ASSERT_DOUBLES_EQUAL(78.0, +// converter.ConvertPressureToPhysicalUnits(file->GetPressureMin()), +// 1e-6); +// CPPUNIT_ASSERT_DOUBLES_EQUAL(82.0, +// converter.ConvertPressureToPhysicalUnits(file->GetPressureMax()), +// 1e-6); PhysicalPosition expected(-1.66017717834e-05, -4.58437586355e-05, -0.05); PhysicalPosition actual = converter.ConvertPositionToPhysicalUnits(file->GetPosition()); @@ -161,7 +161,7 @@ namespace hemelb + (82.0 - REFERENCE_PRESSURE_mmHg) * mmHg_TO_PASCAL * temp * temp / (Cs2 * BLOOD_DENSITY_Kg_per_m3); - CPPUNIT_ASSERT_DOUBLES_EQUAL(targetStartDensity, file->GetDensityMin(), 1e-6); +// CPPUNIT_ASSERT_DOUBLES_EQUAL(targetStartDensity, file->GetDensityMin(), 1e-6); CPPUNIT_ASSERT_DOUBLES_EQUAL(targetStartDensity, file->GetDensity(0), 1e-6); CPPUNIT_ASSERT_DOUBLES_EQUAL(targetMidDensity, file->GetDensity(state.GetTotalTimeSteps() / 2), @@ -289,7 +289,7 @@ namespace hemelb UncheckedSimConfig config(Resource("config_file_velocity_inlet.xml").Path()); lb::SimulationState state = lb::SimulationState(config.GetTimeStepLength(), config.GetTotalTimeSteps()); - double voxelSize = config.GetVoxelSize(); + const util::UnitConverter& converter = config.GetUnitConverter(); fileVel = static_cast(config.GetInlets()[0]); // at this stage, Initialise() has not been called, so the unit converter will be invalid, so we will not be able to convert to physical units. @@ -344,14 +344,6 @@ namespace hemelb ConcreteIolet* copy = new ConcreteIolet(*this); return copy; } - virtual LatticeDensity GetDensityMin() const - { - return 1.0; - } - virtual LatticeDensity GetDensityMax() const - { - return 1.0; - } virtual LatticeDensity GetDensity(hemelb::LatticeTimeStep) const { return 1.0; From ee9820a22ce8e60bf26c3d6dc44a88b765439863 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 4 Dec 2015 11:00:00 +0000 Subject: [PATCH 33/99] Remove phased broadcast --- Code/net/PhasedBroadcast.h | 410 ---------------------------- Code/net/PhasedBroadcastIrregular.h | 396 --------------------------- 2 files changed, 806 deletions(-) delete mode 100644 Code/net/PhasedBroadcast.h delete mode 100644 Code/net/PhasedBroadcastIrregular.h diff --git a/Code/net/PhasedBroadcast.h b/Code/net/PhasedBroadcast.h deleted file mode 100644 index 0a2fef6ce..000000000 --- a/Code/net/PhasedBroadcast.h +++ /dev/null @@ -1,410 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_NET_PHASEDBROADCAST_H -#define HEMELB_NET_PHASEDBROADCAST_H - -#include - -#include "debug/Debugger.h" -#include "net/IteratedAction.h" -#include "net/net.h" -#include "lb/SimulationState.h" -#include "net/IOCommunicator.h" - -namespace hemelb -{ - namespace net - { - /* - * PROGRAMME - * --------- - * - * For 0-indexed cycle counts and 0-indexed depths, tree depth = N. - * - * Iterations 0 to (N-1) are nodes at depth (it) passing down to nodes at depth (it + 1). - * Action happens on all nodes at the end of iteration (N-1). - * Iterations N to (2N-1) are nodes at depth (2N - it) passing up to nodes at depth (2N - it - 1). - */ - - /** - * PhasedBroadcast - a class to control the general process of communication between - * all nodes over multiple iterations. By using a common interface, we can ensure that - * communication happens asynchronously at a single point in each iteration - only one - * communication from any node to any other node will be performed, giving efficient - * performance. - * - * Communication uses a tree structure with a single top-level node, and a constant number of - * children per node down the tree. All nodes at the same depth communicate on the same - * iteration with nodes either above or below them (depending on the position through the - * communication programme defined above). The class supports actions that have to be performed - * before any communication begins, followed by potentially multiple, overlapping communication - * stages between related nodes in each consecutive pair of depths in the tree. Communication - * can be up the tree towards the topmost node, down the tree (beginning at the topmost node) - * or both (down the tree then up). An action to be performed on all nodes, when communication - * from top to bottom of the tree has been completed, is also supported. - * - * The class is called via the IteratedAction interface. Classes that use this interface should - * derive from either PhasedBroadcastRegular (for communication that is to happen at regular - * intervals) or PhasedBroadcastIrregular (for communication that is to happen at irregular - * intervals). The derived class should override one or more virtual functions in the chosen - * base class, and should perform communication using the templated - * [SendTo|ReceiveFrom][Children|Parent] functions defined in this class. - * - * This class is made general using template parameters: - * - * initialAction = if true, an extra iteration occurs at the start of each broadcast cycle - * splay = the number of consecutive iterations communication between a pair of nodes needs to - * go on for. Useful if the passed data is an array of variable length; one node can spend an - * iteration telling the other how many elements will be passed then the next iteration - * sending them. - * ovrlp = the number of iterations where a node both receives and sends. overlap <= splay. - * down = true if parents communicate to their child nodes in the pattern. - * up = true if children communicate to their parent nodes in the pattern. - */ - template - class PhasedBroadcast : public IteratedAction - { - public: - PhasedBroadcast(Net * iNet, - const lb::SimulationState * iSimState, - unsigned int spreadFactor) : - mSimState(iSimState), mMyDepth(0), mTreeDepth(0), mNet(iNet) - { - // Calculate the correct values for the depth variables. - proc_t noSeenToThisDepth = 1; - proc_t noAtCurrentDepth = 1; - - const MpiCommunicator& netTop = mNet->GetCommunicator(); - - while (noSeenToThisDepth < netTop.Size()) - { - // Go down a level. I.e. increase the depth of the tree, to a new level which has M times - // as many nodes on it. - ++mTreeDepth; - noAtCurrentDepth *= spreadFactor; - noSeenToThisDepth += noAtCurrentDepth; - - // If this node is at the current depth, it must have a rank lower than or equal to the highest - // rank at the current depth but greater than the highest rank at the previous depth. - if (noSeenToThisDepth > netTop.Rank() && ( (noSeenToThisDepth - - noAtCurrentDepth) <= netTop.Rank())) - { - mMyDepth = mTreeDepth; - } - } - - // In a M-tree, with a root of 0, each node N's parent has rank floor((N-1) / M) - if (netTop.Rank() == 0) - { - mParent = NOPARENT; - } - else - { - mParent = (netTop.Rank() - 1) / spreadFactor; - } - - // The children of a node N in a M-tree with root 0 are those in the range (M*N)+1,...,(M*N) + M - for (unsigned int child = (spreadFactor * netTop.Rank()) + 1; child - <= spreadFactor * (1 + netTop.Rank()); ++child) - { - if (child < (unsigned int) netTop.Size()) - { - mChildren.push_back(child); - } - } - } - - protected: - - const lb::SimulationState * mSimState; - - /** - * Helper function for sending data to child nodes. - */ - template - void SendToChildren(T* data, int count) - { - for (std::vector::iterator it = mChildren.begin(); it != mChildren.end(); ++it) - { - mNet->RequestSend (data, count, *it); - } - } - - /** - * Helper function for receiving data from parent nodes. - */ - template - void ReceiveFromParent(T* data, int count) - { - if (mParent != NOPARENT) - { - mNet ->RequestReceive (data, count, mParent); - } - } - - /** - * Receives data from each child. This is a set length per child, and each child's - * data is inserted contiguously into the provided array. - * - * The user must handle the case where the number of children is smaller than the - * spread factor and the array pointed by dataStart is not completely filled in. - * - * @param dataStart Pointer to the start of the array. - * @param countPerChild Number of elements to receive per child. - */ - template - void ReceiveFromChildren(T* dataStart, int countPerChild) - { - T* data = dataStart; - for (std::vector::iterator it = mChildren.begin(); it != mChildren.end(); ++it) - { - mNet->RequestReceive (data, countPerChild, *it); - data += countPerChild; - } - } - - /** - * Receives data from each child. This is a variable length per child, and each child's - * data is inserted into the appropriate array. - * - * The user must handle the case where the number of children is smaller than the - * spread factor and the array pointed by dataStart is not completely filled in. - * - * @param dataStart Pointer to the start of the array. - * @param countPerChild Number of elements to receive per child. - */ - template - void ReceiveFromChildren(T** dataStart, unsigned int* countPerChild) - { - unsigned int childNum = 0; - for (std::vector::iterator it = mChildren.begin(); it != mChildren.end(); ++it) - { - mNet->RequestReceive (dataStart[childNum], countPerChild[childNum], *it); - ++childNum; - } - } - - /** - * Helper function for sending data to parent nodes. - */ - template - void SendToParent(T* data, int count) - { - if (mParent != NOPARENT) - { - mNet->RequestSend (data, count, mParent); - } - } - - /** - * Returns the total number of iterations spent doing a complete traversal -- all the way - * down the tree and back up again. - * - * @return - */ - unsigned long GetRoundTripLength() const - { - unsigned long delayTime = initialAction - ? 1 - : 0; - - unsigned long multiplier = (down - ? 1 - : 0) + (up - ? 1 - : 0); - - return delayTime + multiplier * GetTraverseTime(); - } - - /** - * Get the index of the iteration when we first start descending the tree. - * - * @return - */ - unsigned long GetFirstDescending() const - { - return (initialAction - ? 1 - : 0); - } - - /** - * Get the index of the iteration when we first start ascending the tree. - * - * @return - */ - unsigned long GetFirstAscending() const - { - return GetFirstDescending() + (down - ? GetTraverseTime() - : 0); - } - - /** - * Get the 0-indexed depth of this rank. - * - * @return - */ - unsigned long GetMyDepth() const - { - return mMyDepth; - } - - /** - * Get the 0-indexed depth of the whole tree. - * - * @return - */ - unsigned long GetTreeDepth() const - { - return mTreeDepth; - } - - /** - * Get the number of iterations required for a half-cycle (messages going either all the - * way up the tree or all the way down) - * - * @return - */ - unsigned long GetTraverseTime() const - { - return mTreeDepth * (splay - ovrlp) + ovrlp; - } - - /** - * Gets the overlap value for a send to a parent node given the number of iterations - * through the 'upwards' phase. Returns true if this node should send to is parent. - */ - bool GetSendParentOverlap(unsigned long subCycleNumber, unsigned long* sendOverlap) - { - // The first cycle we're sending to parents. - unsigned long firstSendCycle = (mTreeDepth - mMyDepth) * (splay - ovrlp); - - // If we're either not far enough or too far through the cycle, don't send. - if (subCycleNumber < firstSendCycle || subCycleNumber >= (firstSendCycle + splay)) - { - return false; - } - // Otherwise calculate the overlap value and return true. - else - { - *sendOverlap = subCycleNumber - firstSendCycle; - return true; - } - } - - /** - * Gets the overlap value for a send to child nodes given the number of iterations - * through the 'downwards' phase. Returns true if this node should send to is children. - */ - bool GetSendChildrenOverlap(unsigned long subCycleNumber, unsigned long* sendOverlap) - { - // The first cycle we're sending to children. - unsigned long firstSendCycle = mMyDepth * (splay - ovrlp); - - // If we're either not far enough or too far through the cycle, don't send. - if (subCycleNumber < firstSendCycle || subCycleNumber >= (firstSendCycle + splay)) - { - return false; - } - // Otherwise calculate the overlap value and return true. - else - { - *sendOverlap = subCycleNumber - firstSendCycle; - return true; - } - } - - /** - * Gets the overlap value for a receive from a parent node given the number of iterations - * through the 'downwards' phase. Returns true if this node should receive from its parent. - */ - bool GetReceiveParentOverlap(unsigned long subCycleNumber, unsigned long* receiveOverlap) - { - // The first cycle we're receiving from parents. - unsigned long firstReceiveCycle = (mMyDepth - 1) * (splay - ovrlp); - - // If we're either not far enough or too far through the cycle, don't receive. - if (subCycleNumber < firstReceiveCycle || subCycleNumber >= (firstReceiveCycle + splay)) - { - return false; - } - // Otherwise calculate the overlap value and return true. - else - { - *receiveOverlap = subCycleNumber - firstReceiveCycle; - return true; - } - } - - /** - * Gets the overlap value for a receive from child node given the number of iterations - * through the 'upwards' phase. Returns true if this node should receive from its children. - */ - bool GetReceiveChildrenOverlap(unsigned long subCycleNumber, unsigned long* receiveOverlap) - { - // The first cycle we're receiving from parents. - unsigned long firstReceiveCycle = (mTreeDepth - (mMyDepth + 1)) * (splay - ovrlp); - - // If we're either not far enough or too far through the cycle, don't receive. - if (subCycleNumber < firstReceiveCycle || subCycleNumber >= (firstReceiveCycle + splay)) - { - return false; - } - // Otherwise calculate the overlap value and return true. - else - { - *receiveOverlap = subCycleNumber - firstReceiveCycle; - return true; - } - } - - /** - * Returns the number of the parent node. - * @return - */ - int GetParent() const - { - return mParent; - } - - const std::vector& GetChildren() const - { - return mChildren; - } - - protected: - static const int NOPARENT = -1; - - private: - /** - * Note that depths are 0-indexed. I.e. a tree with a single node has - * a depth of zero, and the node itself is considered to be at depth 0. - */ - unsigned int mMyDepth; - unsigned int mTreeDepth; - - /** - * This node's parent rank. - */ - int mParent; - /** - * This node's child ranks. - */ - std::vector mChildren; - protected: - Net * mNet; - }; - } -} - -#endif /* HEMELB_NET_PHASEDBROADCAST_H */ diff --git a/Code/net/PhasedBroadcastIrregular.h b/Code/net/PhasedBroadcastIrregular.h deleted file mode 100644 index c6c657b7f..000000000 --- a/Code/net/PhasedBroadcastIrregular.h +++ /dev/null @@ -1,396 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_NET_PHASEDBROADCASTIRREGULAR_H -#define HEMELB_NET_PHASEDBROADCASTIRREGULAR_H - -#include - -#include "net/PhasedBroadcast.h" - -namespace hemelb -{ - namespace net - { - /** - * PhasedBroadcastIrregular - a class for performing phased broadcasts that are not restricted - * to starting at regular intervals. A longer description is given in PhasedBroadcast.h. - */ - template - class PhasedBroadcastIrregular : public PhasedBroadcast - { - // Typedef for the base class's type - typedef PhasedBroadcast base; - typedef std::list storeType; - - public: - /** - * Constructor that initialises the base class. - * - * @param iNet - * @param iSimState - * @param spreadFactor - * @return - */ - PhasedBroadcastIrregular(Net * iNet, - const lb::SimulationState * iSimState, - unsigned int spreadFactor) : - base(iNet, iSimState, spreadFactor) - { - performInstantBroadcast = false; - } - - /** - * Request the broadcasting to start on this iteration. Returns the number of the iteration - * on which it will complete. - * - * @return - */ - unsigned long Start() - { - unsigned long currentTimeStep = base::mSimState->GetTimeStep(); - unsigned long finishTime = currentTimeStep + base::GetRoundTripLength() - 1; - - // If it's going to take longer than is available for the simulation, assume the - // implementing class will do its thing instantaneously this iteration. - if (finishTime > base::mSimState->GetTotalTimeSteps()) - { - performInstantBroadcast = true; - return currentTimeStep; - } - else - { - if (startIterations.empty() || startIterations.back() != currentTimeStep) - { - startIterations.push_back(currentTimeStep); - } - - return finishTime; - } - } - - /** - * Function that requests all the communications from the Net object. - */ - void RequestComms() - { - const unsigned long currentIt = base::mSimState->GetTimeStep(); - const unsigned long firstAscent = base::GetFirstAscending(); - const unsigned long firstDescent = base::GetFirstDescending(); - - for (storeType::const_iterator it = startIterations.begin(); it != startIterations.end(); it++) - { - unsigned long progress = currentIt - *it; - - // Next, deal with the case of a cycle with an initial pass down the tree. - if (down) - { - if (progress >= firstDescent && progress < firstAscent) - { - unsigned long sendOverlap; - unsigned long receiveOverlap; - - if (base::GetSendChildrenOverlap(progress - firstDescent, &sendOverlap) - && !base::GetChildren().empty()) - { - ProgressToChildren(*it, sendOverlap); - } - - if (base::GetReceiveParentOverlap(progress - firstDescent, &receiveOverlap) - && base::GetParent() >= 0) - { - ProgressFromParent(*it, receiveOverlap); - } - } - } - - // Now deal with the case of a pass up the tree. - if (up) - { - if (progress >= firstAscent) - { - unsigned long sendOverlap; - unsigned long receiveOverlap; - - if (base::GetSendParentOverlap(progress - firstAscent, &sendOverlap) - && base::GetParent() >= 0) - { - ProgressToParent(*it, sendOverlap); - } - - if (base::GetReceiveChildrenOverlap(progress - firstAscent, &receiveOverlap) - && !base::GetChildren().empty()) - { - ProgressFromChildren(*it, receiveOverlap); - } - } - } - } - - unsigned long searchValue = currentIt - (base::GetRoundTripLength() + 1); - - // Use this time to clear out the array. We do this once every iteration so it - // suffices to get rid of one value. - for (storeType::iterator it = startIterations.begin(); it != startIterations.end(); it++) - { - if (*it == searchValue) - { - ClearOut(searchValue); - startIterations.erase(it); - break; - } - } - } - - /** - * Function that acts while waiting for data to be received. This performs the initial - * action on the first iteration only. - */ - void PreReceive() - { - // The only thing to do while waiting is the initial action. - if (initAction) - { - if (IsInitialAction()) - { - InitialAction(base::mSimState->GetTimeStep()); - } - } - } - - /** - * Function to be called after the Receives have completed, where the - * data is used. - */ - void PostReceive() - { - const unsigned long firstAscent = base::GetFirstAscending(); - const unsigned long traversalLength = base::GetTraverseTime(); - const unsigned long cycleLength = base::GetRoundTripLength(); - const unsigned long currentIt = base::mSimState->GetTimeStep(); - - for (storeType::const_iterator it = startIterations.begin(); it != startIterations.end(); it++) - { - const unsigned long progress = currentIt - *it; - - // Deal with the case of a cycle with an initial pass down the tree. - if (down) - { - const unsigned long firstDescent = base::GetFirstDescending(); - - if (progress >= firstDescent && progress < firstAscent) - { - unsigned long receiveOverlap; - - if (base::GetReceiveParentOverlap(progress - firstDescent, &receiveOverlap)) - { - PostReceiveFromParent(*it, receiveOverlap); - } - - // If we're halfway through the programme, all top-down changes have occurred and - // can be applied on all nodes at once safely. - if (progress == (traversalLength - 1)) - { - Effect(currentIt); - } - } - } - - // Deal with the case of a pass up the tree. - if (up) - { - if (progress >= firstAscent && progress < cycleLength) - { - unsigned long receiveOverlap; - - if (base::GetReceiveChildrenOverlap(progress - firstAscent, &receiveOverlap) - && !base::GetChildren().empty()) - { - PostReceiveFromChildren(*it, receiveOverlap); - } - - unsigned long sendOverlap; - - if (base::GetSendParentOverlap(progress - firstAscent, &sendOverlap) - && base::GetParent() >= 0) - { - PostSendToParent(*it, sendOverlap); - } - } - } - - // If this node is the root of the tree and we've just finished the upwards half, it - // must act. - if (progress == (base::GetRoundTripLength() - 1) - && this->mNet->Rank() == 0) - { - TopNodeAction(*it); - } - } - - if (performInstantBroadcast) - { - InstantBroadcast(currentIt); - performInstantBroadcast = false; - } - } - - protected: - /** - * Returns true if we are performing an instant broadcast on the current iteration. - * - * @return - */ - bool IsInstantBroadcast() const - { - return performInstantBroadcast; - } - - /** - * Returns true if we are performing the initial action for a phased broadcast on - * the current iteration. - * - * @return - */ - bool IsInitialAction() const - { - for (storeType::const_iterator it = startIterations.begin(); it != startIterations.end(); it++) - { - if (*it == base::mSimState->GetTimeStep()) - { - return true; - } - } - - return false; - } - - /** - * Overridable function for the initial action performed by a node at the beginning of the - * cycle. Only has an effect if the template paramter initialAction is true. - */ - virtual void InitialAction(unsigned long startIteration) - { - - } - - /** - * Overridable function for when a node has to receive from its children in the tree. - * - * Use ReceiveFromChildren to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressFromChildren(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to receive from its parent in the tree. - * - * Use ReceiveFromParent to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressFromParent(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to send to its children in the tree. - * - * Use SendToChildren to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressToChildren(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to send to its parent in the tree. - * - * Use SendToParent to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressToParent(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Overridable function, called by a node after data has been received from its children. - * The parameter splayNumber is 0 indexed and less than splay. - */ - virtual void PostReceiveFromChildren(unsigned long startIteration, - unsigned long splayNumber) - { - - } - - /** - * Overridable function, called by a node after data has been received from its parent. The - * parameter splayNumber is 0 indexed and less than splay. - */ - virtual void PostReceiveFromParent(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Overridable function, called by a node after data has been sent to its parent. - * @param startIteration The iteration on which this phased send began. - * @param splayNumber The number of steps we have passed through this phase of the sending. - */ - virtual void PostSendToParent(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Action taken when upwards-travelling data reaches the top node. - */ - virtual void TopNodeAction(unsigned long startIteration) - { - - } - - /** - * Action taken by all nodes when downwards-travelling data has been sent to every node. - */ - virtual void Effect(unsigned long startIteration) - { - - } - - /** - * For the case where we don't have enough iterations for a phased broadcast to complete. - */ - virtual void InstantBroadcast(unsigned long startIteration) - { - - } - - /** - * For clearing out the results of an iteration after completion. - */ - virtual void ClearOut(unsigned long startIteration) - { - - } - - private: - storeType startIterations; - bool performInstantBroadcast; - }; - } -} - -#endif /* HEMELB_NET_PHASEDBROADCASTIRREGULAR_H */ From b6cbeef8e4e2239d9fa1095ae6189b93c6ac624d Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 4 Dec 2015 11:34:37 +0000 Subject: [PATCH 34/99] remove some missed references to streaklines --- CMakeLists.txt | 2 -- Code/CMakeLists.txt | 5 ----- Code/reporting/BuildInfo.h.in | 2 -- Code/resources/report.txt.ctp | 1 - Code/resources/report.xml.ctp | 1 - deploy/compile_options.yml | 2 -- 6 files changed, 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f01179c23..441af3d6a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -19,7 +19,6 @@ set(HEMELB_STEERING_LIB basic CACHE STRING "Steering library, choose 'basic' or 'none'" ) option(HEMELB_USE_MULTIMACHINE "Use multi-level parallelism support" OFF) option(HEMELB_BUILD_UNITTESTS "Build the unit-tests" ON) -option(HEMELB_USE_STREAKLINES "Calculate streakline images" OFF) option(HEMELB_USE_ALL_WARNINGS_GNU "Show all compiler warnings on development builds (gnu-style-compilers)" ON) option(HEMELB_USE_BOOST "Use Boost" ON) option(HEMELB_STATIC_ASSERT "Use simple compile-time assertions" ON) @@ -61,7 +60,6 @@ ExternalProject_Add( -DHEMELB_STEERING_LIB=${HEMELB_STEERING_LIB} -DHEMELB_USE_MULTIMACHINE=${HEMELB_USE_MULTIMACHINE} -DHEMELB_BUILD_UNITTESTS=${HEMELB_BUILD_UNITTESTS} - -DHEMELB_USE_STREAKLINES=${HEMELB_USE_STREAKLINES} -DHEMELB_OPTIMISATION=${HEMELB_OPTIMISATION} -DHEMELB_READING_GROUP_SIZE=${HEMELB_READING_GROUP_SIZE} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 32ab6b98b..84e4d7abf 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -13,7 +13,6 @@ option(HEMELB_BUILD_TESTS_ALL "Build all the tests" ON) option(HEMELB_BUILD_TESTS_UNIT "Build the unit-tests (HEMELB_BUILD_TESTS_ALL takes precedence)" ON) option(HEMELB_BUILD_TESTS_FUNCTIONAL "Build the functional tests (HEMELB_BUILD_TESTS_ALL takes precedence)" ON) option(HEMELB_USE_ALL_WARNINGS_GNU "Show all compiler warnings on development builds (gnu-style-compilers)" ON) -option(HEMELB_USE_STREAKLINES "Calculate streakline images" OFF) option(HEMELB_USE_BOOST "Use Boost" ON) option(HEMELB_DEPENDENCIES_SET_RPATH "Set runtime RPATH" ON) option(HEMELB_STATIC_ASSERT "Use simple compile-time assertions" ON) @@ -85,10 +84,6 @@ if(HEMELB_VALIDATE_GEOMETRY) add_definitions(-DHEMELB_VALIDATE_GEOMETRY) endif() -if (NOT HEMELB_USE_STREAKLINES) - add_definitions(-DNO_STREAKLINES) -endif() - if (HEMELB_WAIT_ON_CONNECT) add_definitions(-DHEMELB_WAIT_ON_CONNECT) endif() diff --git a/Code/reporting/BuildInfo.h.in b/Code/reporting/BuildInfo.h.in index f463bf61a..b7b6e9592 100644 --- a/Code/reporting/BuildInfo.h.in +++ b/Code/reporting/BuildInfo.h.in @@ -16,7 +16,6 @@ namespace hemelb { static const std::string mercurial_revision_number="@HEMELB_REVISION_NUMBER@"; static const std::string steering_lib="@HEMELB_STEERING_LIB@"; - static const std::string streaklines_on="@HEMELB_USE_STREAKLINES@"; static const std::string build_type="@CMAKE_BUILD_TYPE@"; static const std::string optimisation="@HEMELB_OPTIMISATION@"; static const std::string use_sse3="@HEMELB_USE_SSE3@"; @@ -39,7 +38,6 @@ namespace hemelb ctemplate::TemplateDictionary *build = dictionary.AddSectionDictionary("BUILD"); build->SetValue("REVISION", mercurial_revision_number); build->SetValue("STEERING", steering_lib); - build->SetValue("STREAKLINES", streaklines_on); build->SetValue("TYPE", build_type); build->SetValue("OPTIMISATION", optimisation); build->SetValue("USE_SSE3", use_sse3); diff --git a/Code/resources/report.txt.ctp b/Code/resources/report.txt.ctp index 7c73cfa0f..f6ab71b2b 100644 --- a/Code/resources/report.txt.ctp +++ b/Code/resources/report.txt.ctp @@ -28,7 +28,6 @@ Name Local Min Mean Max {{#BUILD}} Revision number:{{REVISION}} Steering mode: {{STEERING}} -Streaklines: {{STREAKLINES}} Build type: {{TYPE}} Optimisation level: {{OPTIMISATION}} Use SSE3: {{USE_SSE3}} diff --git a/Code/resources/report.xml.ctp b/Code/resources/report.xml.ctp index 51a4e6e64..78b3e8c9c 100644 --- a/Code/resources/report.xml.ctp +++ b/Code/resources/report.xml.ctp @@ -11,7 +11,6 @@ {{REVISION}} {{STEERING}} - {{STREAKLINES}} {{TYPE}} {{OPTIMISATION}} {{USE_SSE3}} diff --git a/deploy/compile_options.yml b/deploy/compile_options.yml index 8839e5e23..1c32b5130 100644 --- a/deploy/compile_options.yml +++ b/deploy/compile_options.yml @@ -22,8 +22,6 @@ gdb: HEMELB_USE_DEBUGGER: "ON" multiscale: HEMELB_BUILD_MULTISCALE: "ON" -no_streaklines: - HEMELB_USE_STREAKLINES: "OFF" wait_on_connect: HEMELB_WAIT_ON_CONNECT: "ON" d3q15: From dbc08ee1a91e6d0311bd0544524d479957ba73ea Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 4 Dec 2015 16:21:08 +0000 Subject: [PATCH 35/99] Remove redundant all-to-all from mixins --- Code/net/mixins/alltoall/SeparatedAllToAll.cc | 37 ---------- Code/net/mixins/alltoall/SeparatedAllToAll.h | 29 -------- .../mixins/alltoall/ViaPointPointAllToAll.cc | 70 ------------------- .../mixins/alltoall/ViaPointPointAllToAll.h | 29 -------- 4 files changed, 165 deletions(-) delete mode 100644 Code/net/mixins/alltoall/SeparatedAllToAll.cc delete mode 100644 Code/net/mixins/alltoall/SeparatedAllToAll.h delete mode 100644 Code/net/mixins/alltoall/ViaPointPointAllToAll.cc delete mode 100644 Code/net/mixins/alltoall/ViaPointPointAllToAll.h diff --git a/Code/net/mixins/alltoall/SeparatedAllToAll.cc b/Code/net/mixins/alltoall/SeparatedAllToAll.cc deleted file mode 100644 index 6ecca3937..000000000 --- a/Code/net/mixins/alltoall/SeparatedAllToAll.cc +++ /dev/null @@ -1,37 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "net/mixins/alltoall/SeparatedAllToAll.h" -#include -namespace hemelb -{ - namespace net - { - SeparatedAllToAll::SeparatedAllToAll(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms) - { - } - - void SeparatedAllToAll::WaitAllToAll() - { - assert(allToAllReceiveProcComms.size() == allToAllSendProcComms.size()); - for (unsigned int i = 0; i < allToAllReceiveProcComms.size(); i++) - { - SimpleRequest & sendreq = allToAllSendProcComms[i]; - SimpleRequest & receivereq = allToAllReceiveProcComms[i]; - HEMELB_MPI_CALL(MPI_Alltoall, - (sendreq.Pointer, sendreq.Count, sendreq.Type, receivereq.Pointer, receivereq.Count, receivereq.Type, communicator)); - } - - allToAllReceiveProcComms.clear(); - allToAllSendProcComms.clear(); - } - } -} - diff --git a/Code/net/mixins/alltoall/SeparatedAllToAll.h b/Code/net/mixins/alltoall/SeparatedAllToAll.h deleted file mode 100644 index 4a525c408..000000000 --- a/Code/net/mixins/alltoall/SeparatedAllToAll.h +++ /dev/null @@ -1,29 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_NET_MIXINS_ALLTOALL_SEPARATEDALLTOALL_H -#define HEMELB_NET_MIXINS_ALLTOALL_SEPARATEDALLTOALL_H - -#include "net/mixins/StoringNet.h" - -namespace hemelb{ - namespace net{ - - class SeparatedAllToAll : public virtual StoringNet - { - public: - SeparatedAllToAll(const MpiCommunicator& comms); - private: - void ReceiveAllToAll(){} - void SendAllToAll(){} - void WaitAllToAll(); - }; - } -} -#endif diff --git a/Code/net/mixins/alltoall/ViaPointPointAllToAll.cc b/Code/net/mixins/alltoall/ViaPointPointAllToAll.cc deleted file mode 100644 index d6812e5f4..000000000 --- a/Code/net/mixins/alltoall/ViaPointPointAllToAll.cc +++ /dev/null @@ -1,70 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "net/mixins/alltoall/ViaPointPointAllToAll.h" -namespace hemelb -{ - namespace net - { - ViaPointPointAllToAll::ViaPointPointAllToAll(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms) - { - } - void ViaPointPointAllToAll::ReceiveAllToAll() - { - - for (AllToAllProcComms::iterator receivereq = allToAllReceiveProcComms.begin(); - receivereq != allToAllReceiveProcComms.end(); receivereq++) - { - - int size; - MPI_Type_size(receivereq->Type, &size); - - for (int source_rank = 0; source_rank < communicator.Size(); source_rank++) - { - - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - RequestReceiveImpl(static_cast(receivereq->Pointer) + size * source_rank, - 1, - source_rank, - receivereq->Type); - } - - } - - allToAllReceiveProcComms.clear(); - } - - void ViaPointPointAllToAll::SendAllToAll() - { - - for (AllToAllProcComms::iterator sendreq = allToAllSendProcComms.begin(); - sendreq != allToAllSendProcComms.end(); sendreq++) - { - - int size; - MPI_Type_size(sendreq->Type, &size); - - for (int dest_rank = 0; dest_rank < communicator.Size(); dest_rank++) - { - - RequestSendImpl(static_cast(sendreq->Pointer) + size * dest_rank, - 1, - dest_rank, - sendreq->Type); - } - - } - - allToAllSendProcComms.clear(); - } - - } -} diff --git a/Code/net/mixins/alltoall/ViaPointPointAllToAll.h b/Code/net/mixins/alltoall/ViaPointPointAllToAll.h deleted file mode 100644 index 8181234fc..000000000 --- a/Code/net/mixins/alltoall/ViaPointPointAllToAll.h +++ /dev/null @@ -1,29 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_NET_MIXINS_ALLTOALL_VIAPOINTPOINTALLTOALL_H -#define HEMELB_NET_MIXINS_ALLTOALL_VIAPOINTPOINTALLTOALL_H - -#include "net/mixins/StoringNet.h" - -namespace hemelb{ - namespace net{ - - class ViaPointPointAllToAll : public virtual StoringNet - { - public: - ViaPointPointAllToAll(const MpiCommunicator& comms); - private: - void ReceiveAllToAll(); - void SendAllToAll(); - void WaitAllToAll(){} - }; - } -} -#endif From 3e0d62a2f9fef2bb3534cd7fa470c4be88bc86df Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 5 Apr 2016 10:27:00 +0100 Subject: [PATCH 36/99] Add CMake to check MPI standard version --- Code/CMakeLists.txt | 3 +++ Code/cmake/mpi.cmake | 49 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 5ba2b2658..ba62012f6 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -149,6 +149,9 @@ if(NOT HAVE_XDRUINTXX_T) endif() include(mpi) +if(MPI_STANDARD_VERSION VERSION_LESS 3) + message(FATAL_ERROR "Must have MPI 3.0 or greater") +endif() include(dependencies) #-------------Resources ----------------------- diff --git a/Code/cmake/mpi.cmake b/Code/cmake/mpi.cmake index e11b4f093..c153075df 100644 --- a/Code/cmake/mpi.cmake +++ b/Code/cmake/mpi.cmake @@ -27,3 +27,52 @@ int main(int argc, char* argv[]) { MPI_Finalize(); }" HAVE_CONSTCORRECTMPI) + +function(TEST_MPI_VERSION_EQUAL ver output_var) + message("Remove c++11 flag when cpp11-mutex branch merged") + set(CMAKE_REQUIRED_FLAGS "-std=c++11") + set(CMAKE_REQUIRED_QUIET 1) + CHECK_CXX_SOURCE_COMPILES("#include +int main(int argc, char* argv[]) { + static_assert(MPI_VERSION == ${ver}, \"\"); +}" HAVE_MPI_STANDARD_VERSION_${ver}) + SET(${output_var} ${HAVE_MPI_STANDARD_VERSION_${ver}} PARENT_SCOPE) +endfunction() + +function(TEST_MPI_SUBVERSION_EQUAL ver output_var) + message("Remove c++11 flag when cpp11-mutex branch merged") + set(CMAKE_REQUIRED_FLAGS "-std=c++11") + set(CMAKE_REQUIRED_QUIET 1) + CHECK_CXX_SOURCE_COMPILES("#include +int main(int argc, char* argv[]) { + static_assert(MPI_SUBVERSION == ${ver}, \"\"); +}" HAVE_MPI_STANDARD_SUBVERSION_${ver}) + SET(${output_var} ${HAVE_MPI_STANDARD_SUBVERSION_${ver}} PARENT_SCOPE) +endfunction() + +function(GET_MPI_VERSION output_var) + # Future proof as MPI 4.0 will appear eventually + foreach(version RANGE 1 4) + TEST_MPI_VERSION_EQUAL(${version} tmp) + if(${tmp}) + SET(${output_var} ${version} PARENT_SCOPE) + return() + endif() + endforeach() + message(FATAL_ERROR "Could not determine MPI_VERSION") +endfunction() + +function(GET_MPI_SUBVERSION output_var) + foreach(version RANGE 9) + TEST_MPI_SUBVERSION_EQUAL(${version} tmp) + if(${tmp}) + SET(${output_var} ${version} PARENT_SCOPE) + return() + endif() + endforeach() + message(FATAL_ERROR "Could not determine MPI_SUBVERSION") +endfunction() + +GET_MPI_VERSION(MPI_STANDARD_VERSION_MAJOR) +GET_MPI_SUBVERSION(MPI_STANDARD_VERSION_MINOR) +SET(MPI_STANDARD_VERSION "${MPI_STANDARD_VERSION_MAJOR}.${MPI_STANDARD_VERSION_MINOR}") From 4f4ad37f2dcac8b46d9dcaba410843d1fe0d864e Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 5 Apr 2016 14:36:05 +0100 Subject: [PATCH 37/99] fix capitalisation - stupid case insensitive OS --- Code/steering/SteeringComponent.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/steering/SteeringComponent.h b/Code/steering/SteeringComponent.h index faf7f4443..6d6a95c92 100644 --- a/Code/steering/SteeringComponent.h +++ b/Code/steering/SteeringComponent.h @@ -7,7 +7,7 @@ #ifndef HEMELB_STEERING_STEERINGCOMPONENT_H #define HEMELB_STEERING_STEERINGCOMPONENT_H -#include "net/Net.h" +#include "net/net.h" #include "net/CollectiveAction.h" #include "lb/SimulationState.h" #include "configuration/SimConfig.h" From 0549aec8170fb8d3a77e71b216fbba63f4b36df4 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 11 Apr 2016 15:21:18 +0100 Subject: [PATCH 38/99] minor MPI cmake edits --- Code/CMakeLists.txt | 1 + Code/cmake/mpi.cmake | 23 +---------------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 2d3e38283..120bbb8fe 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -154,6 +154,7 @@ include(mpi) if(MPI_STANDARD_VERSION VERSION_LESS 3) message(FATAL_ERROR "Must have MPI 3.0 or greater") endif() +set(HAVE_CONSTCORRECTMPI TRUE) include(dependencies) #-------------Resources ----------------------- diff --git a/Code/cmake/mpi.cmake b/Code/cmake/mpi.cmake index c153075df..477f10789 100644 --- a/Code/cmake/mpi.cmake +++ b/Code/cmake/mpi.cmake @@ -10,27 +10,7 @@ set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}") set(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} ${CMAKE_CXX_LINK_FLAGS}") include_directories(${MPI_INCLUDE_PATH}) -# Figure out if this MPI implementation has a const-correct API (supports MPI 3) -set(CMAKE_REQUIRED_FLAGS -Werror) -set(CMAKE_REQUIRED_DEFINITIONS ${MPI_COMPILE_FLAGS}) -set(CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH}) -set(CMAKE_REQUIRED_LIBRARIES ${MPI_LIBRARIES}) -CHECK_CXX_SOURCE_COMPILES("#include -int main(int argc, char* argv[]) { - const int send = 0; - int recv; - MPI_Request req; - MPI_Init(&argc, &argv); - MPI_Irecv(&recv, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &req); - MPI_Send(&send, 1, MPI_INT, 0, 0, MPI_COMM_WORLD); - MPI_Wait(&req, MPI_STATUS_IGNORE); - MPI_Finalize(); -}" HAVE_CONSTCORRECTMPI) - - function(TEST_MPI_VERSION_EQUAL ver output_var) - message("Remove c++11 flag when cpp11-mutex branch merged") - set(CMAKE_REQUIRED_FLAGS "-std=c++11") set(CMAKE_REQUIRED_QUIET 1) CHECK_CXX_SOURCE_COMPILES("#include int main(int argc, char* argv[]) { @@ -40,8 +20,6 @@ int main(int argc, char* argv[]) { endfunction() function(TEST_MPI_SUBVERSION_EQUAL ver output_var) - message("Remove c++11 flag when cpp11-mutex branch merged") - set(CMAKE_REQUIRED_FLAGS "-std=c++11") set(CMAKE_REQUIRED_QUIET 1) CHECK_CXX_SOURCE_COMPILES("#include int main(int argc, char* argv[]) { @@ -76,3 +54,4 @@ endfunction() GET_MPI_VERSION(MPI_STANDARD_VERSION_MAJOR) GET_MPI_SUBVERSION(MPI_STANDARD_VERSION_MINOR) SET(MPI_STANDARD_VERSION "${MPI_STANDARD_VERSION_MAJOR}.${MPI_STANDARD_VERSION_MINOR}") +message(STATUS "MPI library claims to implement standard version ${MPI_STANDARD_VERSION}") \ No newline at end of file From da47fbd77520dd574d0562c19124bff7fdd1a782 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 13 Apr 2016 14:06:22 +0100 Subject: [PATCH 39/99] tidy up ARCHER section of machines.yml #649 --- deploy/machines.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/machines.yml b/deploy/machines.yml index 92848855b..bd188324b 100644 --- a/deploy/machines.yml +++ b/deploy/machines.yml @@ -70,7 +70,7 @@ archer: #the ARCHER supercomputer at EPCC home_path_template: "/home/$project/$project/$username" runtime_path_template: "/work/$project/$project/$username" fabric_dir: "FabricHemeLb" - modules: ["load cmake", "swap PrgEnv-cray PrgEnv-gnu", "load anaconda"] + modules: ["load cmake/3.2.3", "swap PrgEnv-cray PrgEnv-gnu"] # Tell autoconf for dependencies where the compilers are build_prefix_commands: ["export LDFLAGS=-dynamic","export CXX=CC","export CC=cc", "export LD=CC", "export XTPE_LINK_TYPE=dynamic" ] temp_path_template: "$work_path/tmp" @@ -88,8 +88,8 @@ archer: #the ARCHER supercomputer at EPCC CMAKE_CXX_FLAGS: '"-dynamic -Wno-unused-local-typedefs"' CMAKE_C_FLAGS: "-dynamic" HEMELB_USE_ALL_WARNINGS_GNU: "OFF" - CMAKE_CXX_COMPILER: "/opt/cray/craype/2.1.1/bin/CC" - CMAKE_C_COMPILER: "/opt/cray/craype/2.1.1/bin/cc" + CMAKE_CXX_COMPILER: "CC" + CMAKE_C_COMPILER: "cc" CMAKE_CXX_FLAGS_RELEASE: "" CPPUNIT_PATCH_LDL: OFF CPPUNIT_PATCH_DYNAMIC: ON @@ -107,8 +107,8 @@ archer_dmapp: CMAKE_CXX_FLAGS: '"-dynamic -Wno-unused-local-typedefs"' CMAKE_C_FLAGS: "-dynamic" HEMELB_USE_ALL_WARNINGS_GNU: "OFF" - CMAKE_CXX_COMPILER: "/opt/cray/craype/2.1.1/bin/CC" - CMAKE_C_COMPILER: "/opt/cray/craype/2.1.1/bin/cc" + CMAKE_CXX_COMPILER: "CC" + CMAKE_C_COMPILER: "cc" CMAKE_CXX_FLAGS_RELEASE: "" CPPUNIT_PATCH_LDL: OFF CPPUNIT_PATCH_DYNAMIC: ON From 8d7a04d0c83168829dcc5ec424e17ca69771e58c Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 13 Apr 2016 14:11:17 +0100 Subject: [PATCH 40/99] sort out the cmake and c++11 --- Code/CMakeLists.txt | 6 +++--- Code/cmake/cxx11.cmake | 8 ++++++++ Code/cmake/intel_cpp11.cmake | 3 --- Code/cmake/mpi.cmake | 22 +++++++++++++++++++--- Code/cmake/platform_checks.cmake | 1 - 5 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 Code/cmake/cxx11.cmake delete mode 100644 Code/cmake/intel_cpp11.cmake diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 120bbb8fe..2896e554a 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -7,8 +7,9 @@ cmake_minimum_required (VERSION 3.2) project(HemeLB) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED ON) +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" "${HEMELB_DEPENDENCIES_PATH}/Modules/") + +include(cxx11) #---- OPTION switches accepted by the build ------- @@ -114,7 +115,6 @@ if (HEMELB_USE_VELOCITY_WEIGHTS_FILE) add_definitions(-DHEMELB_USE_VELOCITY_WEIGHTS_FILE) endif() -list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" "${HEMELB_DEPENDENCIES_PATH}/Modules/") list(APPEND CMAKE_INCLUDE_PATH ${HEMELB_DEPENDENCIES_INSTALL_PATH}/include) list(APPEND CMAKE_LIBRARY_PATH ${HEMELB_DEPENDENCIES_INSTALL_PATH}/lib) diff --git a/Code/cmake/cxx11.cmake b/Code/cmake/cxx11.cmake new file mode 100644 index 000000000..d063ae308 --- /dev/null +++ b/Code/cmake/cxx11.cmake @@ -0,0 +1,8 @@ +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(CXX11_FLAGS -std=c++11) + +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX11_FLAGS}") +endif() diff --git a/Code/cmake/intel_cpp11.cmake b/Code/cmake/intel_cpp11.cmake deleted file mode 100644 index d5c97939e..000000000 --- a/Code/cmake/intel_cpp11.cmake +++ /dev/null @@ -1,3 +0,0 @@ -if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") -endif() diff --git a/Code/cmake/mpi.cmake b/Code/cmake/mpi.cmake index 477f10789..102cd3cb2 100644 --- a/Code/cmake/mpi.cmake +++ b/Code/cmake/mpi.cmake @@ -11,7 +11,6 @@ set(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} ${CMAKE_CXX_LINK_FLAGS}") include_directories(${MPI_INCLUDE_PATH}) function(TEST_MPI_VERSION_EQUAL ver output_var) - set(CMAKE_REQUIRED_QUIET 1) CHECK_CXX_SOURCE_COMPILES("#include int main(int argc, char* argv[]) { static_assert(MPI_VERSION == ${ver}, \"\"); @@ -20,7 +19,6 @@ int main(int argc, char* argv[]) { endfunction() function(TEST_MPI_SUBVERSION_EQUAL ver output_var) - set(CMAKE_REQUIRED_QUIET 1) CHECK_CXX_SOURCE_COMPILES("#include int main(int argc, char* argv[]) { static_assert(MPI_SUBVERSION == ${ver}, \"\"); @@ -51,7 +49,25 @@ function(GET_MPI_SUBVERSION output_var) message(FATAL_ERROR "Could not determine MPI_SUBVERSION") endfunction() +# CMake doens't seem to properly pass through the C++11 flag +# This is probably wrong +set(CMAKE_REQUIRED_FLAGS ${CXX11_FLAGS}) +set(CMAKE_REQUIRED_QUIET 1) + +# Put this in here as this is the key feature needed by the test programs +CHECK_CXX_SOURCE_COMPILES("int main(int, char**) { +static_assert(true, \"must not fail\"); +}" + HAVE_STATIC_ASSERT) + +if(NOT HAVE_STATIC_ASSERT) + message(FATAL_ERROR "No static_assert!") +endif() + GET_MPI_VERSION(MPI_STANDARD_VERSION_MAJOR) GET_MPI_SUBVERSION(MPI_STANDARD_VERSION_MINOR) SET(MPI_STANDARD_VERSION "${MPI_STANDARD_VERSION_MAJOR}.${MPI_STANDARD_VERSION_MINOR}") -message(STATUS "MPI library claims to implement standard version ${MPI_STANDARD_VERSION}") \ No newline at end of file +message(STATUS "MPI library claims to implement standard version ${MPI_STANDARD_VERSION}") + +unset(CMAKE_REQUIRED_FLAGS) +unset(CMAKE_REQUIRED_QUIET) diff --git a/Code/cmake/platform_checks.cmake b/Code/cmake/platform_checks.cmake index d338bb3d3..00c0ca2d3 100644 --- a/Code/cmake/platform_checks.cmake +++ b/Code/cmake/platform_checks.cmake @@ -5,7 +5,6 @@ # license in the file LICENSE. include(gnu_bug) include(mountain_lion_scandir) -include(intel_cpp11) include(CheckCXXSourceCompiles) CHECK_CXX_SOURCE_COMPILES("#include \n int main(int c,char** v){ return isnan(1.0); }" HAVE_ISNAN) From 7b85ddb119c7fc8cf4f537f18864d6b520c39b4e Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 13 Apr 2016 14:11:58 +0100 Subject: [PATCH 41/99] have the cmake use the source dir to look for the git repo --- Code/cmake/build_environment.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/cmake/build_environment.cmake b/Code/cmake/build_environment.cmake index 58df48676..974bb0e7a 100644 --- a/Code/cmake/build_environment.cmake +++ b/Code/cmake/build_environment.cmake @@ -5,7 +5,7 @@ # license in the file LICENSE. #------Capture build environment ------------- find_package(Git REQUIRED) -execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse HEAD +execute_process(COMMAND ${GIT_EXECUTABLE} --git-dir ${CMAKE_SOURCE_DIR}/../.git rev-parse HEAD RESULT_VARIABLE rev_ok OUTPUT_VARIABLE HEMELB_REVISION_NUMBER OUTPUT_STRIP_TRAILING_WHITESPACE) if (NOT rev_ok EQUAL 0) message("Could not get revision number from Git, looking for revision_info.txt") From 638320385913267b5dd34e9f6b8bf764983f350d Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 13 Apr 2016 15:52:32 +0100 Subject: [PATCH 42/99] Dependencies cmake: tweak boost install and update ctemplate to 2.3 from github --- dependencies/CMakeLists.txt | 61 ++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/dependencies/CMakeLists.txt b/dependencies/CMakeLists.txt index af74bf892..0ca0fd090 100644 --- a/dependencies/CMakeLists.txt +++ b/dependencies/CMakeLists.txt @@ -179,7 +179,7 @@ else() INSTALL_DIR ${HEMELB_DEPENDENCIES_INSTALL_PATH} URL ${BOOST_TARBALL} BUILD_COMMAND "" - INSTALL_COMMAND cp -r /boost /include + INSTALL_COMMAND "cp -r /boost /include" CONFIGURE_COMMAND "" BUILD_IN_SOURCE 1 ) @@ -191,37 +191,36 @@ option(CTEMPLATE_PATCH_VACOPY "Define va_copy macro through patch" OFF) option(CTEMPLATE_PATCH_ALIGN "Define GTL align macros as gnu" OFF) find_package(CTemplate) if (CTEMPLATE_FOUND) - message("Ctemplate already installed, no need to download") + message("Ctemplate already installed, no need to download") else() - message("Ctemplate not installed, will build from source") - find_file(CTEMPLATE_TARBALL ctemplate-2.2.tar.gz - DOC "Path to download CTemplate (can be url http://)" - PATHS ${HEMELB_DEPENDENCIES_PATH}/distributions - ) - if(NOT CTEMPLATE_TARBALL) - message("No ctemplate source found, will download.") - set(CTEMPLATE_TARBALL http://ctemplate.googlecode.com/files/ctemplate-2.2.tar.gz - CACHE STRING "Path to download CTemplate (can be local file://)" FORCE) - endif() - if (CTEMPLATE_PATCH_VACOPY) - set(PATCH_COMMAND_VACOPY patch -p1 < ${HEMELB_DEPENDENCIES_PATH}/patches/ctemplate_vacopy.diff) - else() - set(PATCH_COMMAND_VACOPY echo novacopy) - endif() - if (CTEMPLATE_PATCH_ALIGN) - set(PATCH_COMMAND_ALIGN patch -p1 < ${HEMELB_DEPENDENCIES_PATH}/patches/ctemplate_align.diff) - else() - set(PATCH_COMMAND_ALIGN echo noalign) - endif() - ExternalProject_Add( - ctemplate - INSTALL_DIR ${HEMELB_DEPENDENCIES_INSTALL_PATH} - URL ${CTEMPLATE_TARBALL} - CONFIGURE_COMMAND /configure --prefix= ${CTEMPLATE_CONFIGURE_OPTIONS} - BUILD_COMMAND make -j${HEMELB_SUBPROJECT_MAKE_JOBS} - BUILD_IN_SOURCE 1 - PATCH_COMMAND ${PATCH_COMMAND_ALIGN} && ${PATCH_COMMAND_VACOPY} - ) + message("Ctemplate not installed, will build from source") + + if (CTEMPLATE_PATCH_VACOPY) + set(PATCH_COMMAND_VACOPY patch -p1 < ${HEMELB_DEPENDENCIES_PATH}/patches/ctemplate_vacopy.diff) + else() + set(PATCH_COMMAND_VACOPY echo novacopy) + endif() + if (CTEMPLATE_PATCH_ALIGN) + set(PATCH_COMMAND_ALIGN patch -p1 < ${HEMELB_DEPENDENCIES_PATH}/patches/ctemplate_align.diff) + else() + set(PATCH_COMMAND_ALIGN echo noalign) + endif() + + ExternalProject_Add(ctemplate + # Download + GIT_REPOSITORY https://github.com/OlafvdSpek/ctemplate.git + GIT_TAG ctemplate-2.3 + # Patch + PATCH_COMMAND ${PATCH_COMMAND_ALIGN} && ${PATCH_COMMAND_VACOPY} + # Configure + CONFIGURE_COMMAND /configure --prefix= ${CTEMPLATE_CONFIGURE_OPTIONS} + # Build + BUILD_COMMAND make -j${HEMELB_SUBPROJECT_MAKE_JOBS} + BUILD_IN_SOURCE 1 + # Install + INSTALL_DIR ${HEMELB_DEPENDENCIES_INSTALL_PATH} + INSTALL_COMMAND make install + ) endif() #-----------------ZLIB -------------------------------- From 58a79f772a44a7f663bf4e1759a8ac75abaced1b Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 13 Apr 2016 22:08:09 +0100 Subject: [PATCH 43/99] cmake: ensure boost install copies correctly --- dependencies/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/CMakeLists.txt b/dependencies/CMakeLists.txt index 0ca0fd090..a4e2ec563 100644 --- a/dependencies/CMakeLists.txt +++ b/dependencies/CMakeLists.txt @@ -179,7 +179,7 @@ else() INSTALL_DIR ${HEMELB_DEPENDENCIES_INSTALL_PATH} URL ${BOOST_TARBALL} BUILD_COMMAND "" - INSTALL_COMMAND "cp -r /boost /include" + INSTALL_COMMAND "mkdir -p /include && cp -r /boost /include" CONFIGURE_COMMAND "" BUILD_IN_SOURCE 1 ) From 67bb525ed6af9f08d999d20ce43f048bebff27b5 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 13 Apr 2016 22:12:06 +0100 Subject: [PATCH 44/99] cmake: remove quoting of boost INSTALL_COMMAND --- dependencies/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies/CMakeLists.txt b/dependencies/CMakeLists.txt index a4e2ec563..0d3a74cec 100644 --- a/dependencies/CMakeLists.txt +++ b/dependencies/CMakeLists.txt @@ -179,7 +179,7 @@ else() INSTALL_DIR ${HEMELB_DEPENDENCIES_INSTALL_PATH} URL ${BOOST_TARBALL} BUILD_COMMAND "" - INSTALL_COMMAND "mkdir -p /include && cp -r /boost /include" + INSTALL_COMMAND mkdir -p /include && cp -r /boost /include CONFIGURE_COMMAND "" BUILD_IN_SOURCE 1 ) From 8c9e590c0a948291af3a85f673f9b9b2c9230079 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 13 Apr 2016 22:32:22 +0100 Subject: [PATCH 45/99] cmake: revert move of CMAKE_MODULE_PATH addition that wasn't meant to be committed --- Code/CMakeLists.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 2896e554a..c84eb36cf 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -7,10 +7,6 @@ cmake_minimum_required (VERSION 3.2) project(HemeLB) -list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" "${HEMELB_DEPENDENCIES_PATH}/Modules/") - -include(cxx11) - #---- OPTION switches accepted by the build ------- option(HEMELB_BUILD_DEBUGGER "Build the built in debugger" ON) @@ -115,6 +111,8 @@ if (HEMELB_USE_VELOCITY_WEIGHTS_FILE) add_definitions(-DHEMELB_USE_VELOCITY_WEIGHTS_FILE) endif() + +list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" "${HEMELB_DEPENDENCIES_PATH}/Modules/") list(APPEND CMAKE_INCLUDE_PATH ${HEMELB_DEPENDENCIES_INSTALL_PATH}/include) list(APPEND CMAKE_LIBRARY_PATH ${HEMELB_DEPENDENCIES_INSTALL_PATH}/lib) @@ -124,6 +122,7 @@ if(HEMELB_DEPENDENCIES_SET_RPATH) SET(CMAKE_INSTALL_RPATH "${HEMELB_DEPENDENCIES_INSTALL_PATH}/lib") endif() +include(cxx11) include(build_environment) include(platform_checks) From 8470814ac5c9fb24fe5133f262e33a9c2e9e61d2 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 4 Oct 2016 10:51:47 +0100 Subject: [PATCH 46/99] implement MPI GatherV --- Code/net/MpiCommunicator.h | 6 +++++- Code/net/MpiCommunicator.hpp | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/Code/net/MpiCommunicator.h b/Code/net/MpiCommunicator.h index e9ae7e13f..e1b63339b 100644 --- a/Code/net/MpiCommunicator.h +++ b/Code/net/MpiCommunicator.h @@ -123,7 +123,11 @@ namespace hemelb template std::vector Gather(const T& val, const int root) const; - + + template + std::vector GatherV(const std::vector senddata, const std::vector recvcounts, + const int root) const; + template std::vector AllGather(const T& val) const; diff --git a/Code/net/MpiCommunicator.hpp b/Code/net/MpiCommunicator.hpp index 1e16e73d3..4c1a21cdb 100644 --- a/Code/net/MpiCommunicator.hpp +++ b/Code/net/MpiCommunicator.hpp @@ -149,6 +149,39 @@ namespace hemelb return ans; } + template + std::vector MpiCommunicator::GatherV(const std::vector senddata, + const std::vector recvcounts, + const int root) const + { + const int np = Size(); + const int sendcount = senddata.size(); + std::vector displs; + std::vector ans; + T* recvbuf = nullptr; + if (Rank() == root) + { + // Compute the displacements from the counts + displs.resize(np); + int total = 0; + for(size_t i = 0; i < np; ++i) { + displs[i] = total; + total += recvcounts[i]; + } + // set up recv buffer + ans.resize(np); + recvbuf = ans.data(); + } + + HEMELB_MPI_CALL( + MPI_Gatherv, + (MpiConstCast(senddata.data()), sendcount, MpiDataType(), + recvbuf, recvcounts.data(), displs.data(), MpiDataType(), + root, *this) + ); + return ans; + } + template std::vector MpiCommunicator::AllGather(const T& val) const { From 7e28e85902e6fae26c8a4a5b0d7ede04d2c2a564 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 14 Oct 2016 13:25:37 +0100 Subject: [PATCH 47/99] deal with NBC synchronisation issues --- Code/SimulationMaster.cc | 29 ++++++++++++++++++++++++++--- Code/lb/SimulationState.cc | 3 +++ Code/lb/StabilityTester.h | 2 +- Code/net/MpiRequest.cc | 18 +++++++----------- Code/net/MpiRequest.h | 7 ++----- 5 files changed, 39 insertions(+), 20 deletions(-) diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index 04e86dccb..7b0fd2fc9 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -341,12 +341,21 @@ void SimulationMaster::RunSimulation() hemelb::log::Logger::Log("Beginning to run simulation."); timings[hemelb::reporting::Timers::simulation].Start(); - while (simulationState->GetTimeStep() <= simulationState->GetTotalTimeSteps()) + int LastTS = simulationState->GetTotalTimeSteps(); + while (simulationState->GetTimeStep() <= LastTS) { + // We need to keep the master rank in sync with the workers + // Here, each rank notifies that it is beginning a time step + auto syncReq = ioComms.Ibarrier(); + DoTimeStep(); + + timeStepSyncReq.Wait(); + // Now all ranks have at least started this time step + if (simulationState->IsTerminating()) { - break; + LastTS = std::min(simulationState->GetTimeStep(), simulationState->GetTotalTimeSteps()); } } @@ -373,6 +382,20 @@ void SimulationMaster::Finalise() void SimulationMaster::DoTimeStep() { + // If the simulation is finishing, all running collective actions + // must wait this time step + if (simulationState->IsTerminating()) + { + if (stabilityTester) + stabilityTester->MustFinishThisTimeStep(); + if (entropyTester) + entropyTester->MustFinishThisTimeStep(); + if (incompressibilityChecker) + incompressibilityChecker->MustFinishThisTimeStep(); + if (steeringCpt) + steeringCpt->MustFinishThisTimeStep(); + } + if (simulationState->GetTimeStep() % 100 == 0) { hemelb::log::Logger::Log("time step %i", @@ -390,7 +413,7 @@ void SimulationMaster::DoTimeStep() } // If the user requested to terminate converged steady flow simulations, mark - // simulation to be finished at the end of the current timestep. + // simulation to be finished ASAP. if ( (simulationState->GetStability() == hemelb::lb::StableAndConverged) && monitoringConfig->convergenceTerminate) { diff --git a/Code/lb/SimulationState.cc b/Code/lb/SimulationState.cc index 333e80d91..984df0f1a 100644 --- a/Code/lb/SimulationState.cc +++ b/Code/lb/SimulationState.cc @@ -56,6 +56,9 @@ namespace hemelb bool SimulationState::IsTerminating() const { + if (timeStep >= totalTimeSteps) + return true; + return isTerminating; } diff --git a/Code/lb/StabilityTester.h b/Code/lb/StabilityTester.h index 0cf1f47a0..8a5670a52 100644 --- a/Code/lb/StabilityTester.h +++ b/Code/lb/StabilityTester.h @@ -40,7 +40,7 @@ namespace hemelb /** * Initiate the collective. */ - void Send(void); + virtual void Send(void); /** * Computes the relative difference between the densities at the beginning and end of a diff --git a/Code/net/MpiRequest.cc b/Code/net/MpiRequest.cc index 67451fa5c..59045c3d2 100644 --- a/Code/net/MpiRequest.cc +++ b/Code/net/MpiRequest.cc @@ -15,30 +15,26 @@ namespace hemelb namespace net { MpiRequest::MpiRequest() : - reqPtr() + req(MPI_REQUEST_NULL) { } - MpiRequest::MpiRequest(MPI_Request req) : - reqPtr() + MpiRequest::MpiRequest(MPI_Request req_) : + req(req_) { - reqPtr.reset(new MPI_Request(req)); } MpiRequest::operator bool() const { - if ((bool)reqPtr) - return *reqPtr != MPI_REQUEST_NULL; - - return false; + return req != MPI_REQUEST_NULL; } void MpiRequest::Wait() { - HEMELB_MPI_CALL(MPI_Wait, (reqPtr.get(), MPI_STATUS_IGNORE)); + HEMELB_MPI_CALL(MPI_Wait, (&req, MPI_STATUS_IGNORE)); } void MpiRequest::Wait(MpiStatus& stat) { - HEMELB_MPI_CALL(MPI_Wait, (reqPtr.get(), stat.statPtr.get())); + HEMELB_MPI_CALL(MPI_Wait, (&req, stat.statPtr.get())); } void MpiRequest::WaitAll(ReqVec& reqs) @@ -114,7 +110,7 @@ namespace hemelb bool MpiRequest::Test() { int flag; - HEMELB_MPI_CALL(MPI_Test, (reqPtr.get(), &flag, MPI_STATUS_IGNORE)); + HEMELB_MPI_CALL(MPI_Test, (&req, &flag, MPI_STATUS_IGNORE)); return flag; } diff --git a/Code/net/MpiRequest.h b/Code/net/MpiRequest.h index f7098cecd..d35c3dbc4 100644 --- a/Code/net/MpiRequest.h +++ b/Code/net/MpiRequest.h @@ -12,7 +12,6 @@ #include #include "net/MpiError.h" -#include namespace hemelb { @@ -38,9 +37,8 @@ namespace hemelb */ operator MPI_Request() const { - return *reqPtr; + return req; } - operator bool() const; void Wait(); @@ -53,8 +51,7 @@ namespace hemelb static bool TestAll(ReqVec& reqs); private: - - boost::shared_ptr reqPtr; + MPI_Request req; }; } From a8cdd6907ac5967b07cd57c8be371b0485473cd4 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 14 Oct 2016 13:26:18 +0100 Subject: [PATCH 48/99] revise debug resume script for new OS X --- Code/debug/OSX/resume.lldb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/debug/OSX/resume.lldb b/Code/debug/OSX/resume.lldb index 0bd336b81..2e557434f 100644 --- a/Code/debug/OSX/resume.lldb +++ b/Code/debug/OSX/resume.lldb @@ -4,7 +4,7 @@ expr amWaiting = 0 # Set up a breakpoint for our break function and a command to finish # it automatically -breakpoint set -F hemelb::debug::ActiveDebugger::BreakHere() +breakpoint set -M ActiveDebugger::BreakHere() breakpoint command add -o 'finish' 1 # Carry on From 9074b549786d780d51b8975f39330604370a8386 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 14 Oct 2016 15:48:17 +0100 Subject: [PATCH 49/99] fix mis-renamed variable --- Code/SimulationMaster.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index 7b0fd2fc9..655594ba1 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -350,7 +350,7 @@ void SimulationMaster::RunSimulation() DoTimeStep(); - timeStepSyncReq.Wait(); + syncReq.Wait(); // Now all ranks have at least started this time step if (simulationState->IsTerminating()) From 579ba09b21890a0eac611d0bbe939d7701f454ad Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 14 Oct 2016 16:07:24 +0100 Subject: [PATCH 50/99] Replace HEMELB_STATIC_ASSERT with c++11 static_assert --- CMakeLists.txt | 2 -- Code/CMakeLists.txt | 5 ---- Code/util/Vector3D.h | 7 +++-- Code/util/Vector3DArithmeticTraits.h | 34 ++++++++++----------- Code/util/static_assert.h | 44 ---------------------------- 5 files changed, 19 insertions(+), 73 deletions(-) delete mode 100644 Code/util/static_assert.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 94a925f93..7f6d11949 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -25,7 +25,6 @@ set(HEMELB_STEERING_LIB basic option(HEMELB_USE_MULTIMACHINE "Use multi-level parallelism support" OFF) option(HEMELB_BUILD_UNITTESTS "Build the unit-tests" ON) option(HEMELB_USE_ALL_WARNINGS_GNU "Show all compiler warnings on development builds (gnu-style-compilers)" ON) -option(HEMELB_STATIC_ASSERT "Use simple compile-time assertions" ON) set(HEMELB_OPTIMISATION "-O4" CACHE STRING "Optimisation level (can be blank or -O1 to -O4)") option(HEMELB_BUILD_MULTISCALE "Build HemeLB Multiscale functionality" OFF) set(HEMELB_LATTICE "D3Q15" @@ -77,7 +76,6 @@ ExternalProject_Add( -DCTEMPLATE_USE_STATIC=${CTEMPLATE_USE_STATIC} -DCPPUNIT_USE_STATIC=${CPPUNIT_USE_STATIC} -DHEMELB_DEPENDENCIES_SET_RPATH=${HEMELB_DEPENDENCIES_SET_RPATH} - -DHEMELB_STATIC_ASSERT=${HEMELB_STATIC_ASSERT} -DHEMELB_LATTICE=${HEMELB_LATTICE} -DHEMELB_KERNEL=${HEMELB_KERNEL} -DHEMELB_WALL_BOUNDARY=${HEMELB_WALL_BOUNDARY} diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index c84eb36cf..f8f1c2e8c 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -19,7 +19,6 @@ option(HEMELB_BUILD_TESTS_UNIT "Build the unit-tests (HEMELB_BUILD_TESTS_ALL tak option(HEMELB_BUILD_TESTS_FUNCTIONAL "Build the functional tests (HEMELB_BUILD_TESTS_ALL takes precedence)" ON) option(HEMELB_USE_ALL_WARNINGS_GNU "Show all compiler warnings on development builds (gnu-style-compilers)" ON) option(HEMELB_DEPENDENCIES_SET_RPATH "Set runtime RPATH" ON) -option(HEMELB_STATIC_ASSERT "Use simple compile-time assertions" ON) option(HEMELB_WAIT_ON_CONNECT "Wait for steering client" OFF) option(HEMELB_BUILD_MULTISCALE "Build HemeLB Multiscale functionality" OFF) option(HEMELB_IMAGES_TO_NULL "Write images to null" OFF) @@ -92,10 +91,6 @@ if (HEMELB_WAIT_ON_CONNECT) add_definitions(-DHEMELB_WAIT_ON_CONNECT) endif() -if (NOT HEMELB_STATIC_ASSERT) - add_definitions(-DHEMELB_NO_STATIC_ASSERT) -endif() - if (HEMELB_IMAGES_TO_NULL) add_definitions(-DHEMELB_IMAGES_TO_NULL) endif() diff --git a/Code/util/Vector3D.h b/Code/util/Vector3D.h index 93f455d30..a21232fd5 100644 --- a/Code/util/Vector3D.h +++ b/Code/util/Vector3D.h @@ -12,7 +12,6 @@ #include #include #include -#include "util/static_assert.h" #include "util/utilityFunctions.h" #include "util/Vector3DArithmeticTraits.h" @@ -343,8 +342,10 @@ namespace hemelb */ T GetMagnitude() const { - HEMELB_STATIC_ASSERT(std::numeric_limits::is_specialized); - HEMELB_STATIC_ASSERT(!std::numeric_limits::is_integer); + static_assert(std::numeric_limits::is_specialized, + "type must have std::numeric_limits specialized"); + static_assert(!std::numeric_limits::is_integer, + "type must not be an integer"); return std::sqrt(GetMagnitudeSquared()); } diff --git a/Code/util/Vector3DArithmeticTraits.h b/Code/util/Vector3DArithmeticTraits.h index 8e3aa497b..a0443b065 100644 --- a/Code/util/Vector3DArithmeticTraits.h +++ b/Code/util/Vector3DArithmeticTraits.h @@ -19,25 +19,21 @@ namespace hemelb template struct Vector3DArithmeticTraits { - /* - * If this assertion trips, it means that Vector3D::operator* or Vector3D::operator/ - * have not been tested with this combination of types (OperatorArgument1Type, OperatorArgument2Type) - * - * One would like to write HEMELB_STATIC_ASSERT(false), however if the - * static assertion is not dependent upon one or more template parameters, then - * the compiler is permitted to evaluate the static assertion at the point it is - * first seen, irrespective of whether the template is ever instantiated. - * - * See http://www.boost.org/doc/libs/1_49_0/doc/html/boost_staticassert.html - */ - HEMELB_STATIC_ASSERT(sizeof(OperatorArgument1Type) == 0); - - /* - * Boost alternative including an error message. If the C++0x static_assert feature is - * not available, BOOST_STATIC_ASSERT_MSG(x, msg) will be treated as BOOST_STATIC_ASSERT(x) - * and unfortunately you won't see the message. - */ - //BOOST_STATIC_ASSERT_MSG(sizeof(T1) == 0, "Vector3D has not been tested with this combination of types"); + /* + * If this assertion trips, it means that Vector3D::operator* or + * Vector3D::operator/ have not been tested with this + * combination of types (OperatorArgument1Type, + * OperatorArgument2Type) + * + * One would like to write static_assert(false), however if the + * static assertion is not dependent upon one or more template + * parameters, then the compiler is permitted to evaluate the + * static assertion at the point it is first seen, irrespective + * of whether the template is ever instantiated. + * + */ + static_assert(sizeof(OperatorArgument1Type) == 0, + "Vector3D has not been tested with this combination of types"); }; // Trivial case: both arguments share type. diff --git a/Code/util/static_assert.h b/Code/util/static_assert.h deleted file mode 100644 index 395b8ebec..000000000 --- a/Code/util/static_assert.h +++ /dev/null @@ -1,44 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UTIL_STATIC_ASSERT_H -#define HEMELB_UTIL_STATIC_ASSERT_H - -#ifdef HEMELB_NO_STATIC_ASSERT - -#define HEMELB_STATIC_ASSERT(B) - -#else // HEMELB_NO_STATIC_ASSERT - -#define HEMELB_UTIL_STATIC_ASSERT_BOOL_CAST( x ) ((x) == 0 ? false : true) -#define HEMELB_JOIN(X,Y) HEMELB_DO_JOIN(X,Y) -#define HEMELB_DO_JOIN(X,Y) X##Y -namespace hemelb -{ - namespace util - { - - template struct STATIC_ASSERTION_FAILURE; - - template<> struct STATIC_ASSERTION_FAILURE - { - enum - { - value = 1 - }; - }; - - template struct static_assert_test - { - }; - -#define HEMELB_STATIC_ASSERT( B ) typedef ::hemelb::util::static_assert_test< sizeof(::hemelb::util::STATIC_ASSERTION_FAILURE< HEMELB_UTIL_STATIC_ASSERT_BOOL_CAST( B ) >) > HEMELB_JOIN(static_assert_typedef,__LINE__) - } -} - -#endif // HEMELB_NO_STATIC_ASSERT - -#endif // HEMELB_UTIL_STATIC_ASSERT_H From 2347e75639e52984ef03d40e829fc90a9d24852f Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 14 Oct 2016 18:18:20 +0100 Subject: [PATCH 51/99] replace boost::shared_ptr with std::shared_ptr --- Code/net/MpiCommunicator.h | 6 ++---- Code/net/MpiFile.h | 4 ++-- Code/net/MpiGroup.h | 4 ++-- Code/net/MpiStatus.h | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Code/net/MpiCommunicator.h b/Code/net/MpiCommunicator.h index e1b63339b..08b8e7a4f 100644 --- a/Code/net/MpiCommunicator.h +++ b/Code/net/MpiCommunicator.h @@ -7,11 +7,9 @@ #ifndef HEMELB_NET_MPICOMMUNICATOR_H #define HEMELB_NET_MPICOMMUNICATOR_H -//#include "units.h" -//#include "net/mpi.h" #include +#include #include "net/MpiError.h" -#include namespace hemelb { @@ -176,7 +174,7 @@ namespace hemelb */ MpiCommunicator(MPI_Comm communicator, bool willOwn); - boost::shared_ptr commPtr; + std::shared_ptr commPtr; }; bool operator==(const MpiCommunicator& comm1, const MpiCommunicator& comm2); diff --git a/Code/net/MpiFile.h b/Code/net/MpiFile.h index 71da5dd53..5af910cfb 100644 --- a/Code/net/MpiFile.h +++ b/Code/net/MpiFile.h @@ -7,7 +7,7 @@ #ifndef HEMELB_NET_MPIFILE_H #define HEMELB_NET_MPIFILE_H -#include +#include #include "net/MpiError.h" #include "net/MpiCommunicator.h" @@ -60,7 +60,7 @@ namespace hemelb MpiFile(const MpiCommunicator& parentComm, MPI_File fh); const MpiCommunicator* comm; - boost::shared_ptr filePtr; + std::shared_ptr filePtr; }; } diff --git a/Code/net/MpiGroup.h b/Code/net/MpiGroup.h index 07bdf374e..723967c29 100644 --- a/Code/net/MpiGroup.h +++ b/Code/net/MpiGroup.h @@ -10,7 +10,7 @@ #include "units.h" #include "net/mpi.h" #include "net/MpiCommunicator.h" -#include +#include namespace hemelb { @@ -71,7 +71,7 @@ namespace hemelb */ MpiGroup(MPI_Group grp, bool own); - boost::shared_ptr groupPtr; + std::shared_ptr groupPtr; }; } } diff --git a/Code/net/MpiStatus.h b/Code/net/MpiStatus.h index 2d08c07e6..1c060963c 100644 --- a/Code/net/MpiStatus.h +++ b/Code/net/MpiStatus.h @@ -11,7 +11,7 @@ #define HEMELB_NET_MPISTATUS_H #include "net/MpiError.h" -#include +#include namespace hemelb { @@ -35,7 +35,7 @@ namespace hemelb } private: - boost::shared_ptr statPtr; + std::shared_ptr statPtr; // Request needs to be able to access statPtr. friend class MpiRequest; }; From 7e7a6ae58182562e6663b9a21da8f041b5b2845a Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 17 Oct 2016 12:21:15 +0100 Subject: [PATCH 52/99] replace net-based gathers with communicator gathers --- Code/geometry/GeometryReader.cc | 4 ++-- Code/geometry/needs/Needs.cc | 30 +++++++++++------------------- Code/geometry/needs/Needs.h | 10 ++++++---- Code/net/MpiCommunicator.hpp | 2 +- 4 files changed, 20 insertions(+), 26 deletions(-) diff --git a/Code/geometry/GeometryReader.cc b/Code/geometry/GeometryReader.cc index de8624c46..a156f69ba 100644 --- a/Code/geometry/GeometryReader.cc +++ b/Code/geometry/GeometryReader.cc @@ -321,11 +321,11 @@ namespace hemelb // Next we spread round the lists of which blocks each core needs access to. log::Logger::Log("Informing reading cores of block needs"); - net::Net net = net::Net(computeComms); + Needs needs(geometry.GetBlockCount(), readBlock, util::NumericalFunctions::min(READING_GROUP_SIZE, computeComms.Size()), - net, + computeComms, ShouldValidate()); timings[hemelb::reporting::Timers::readBlocksPrelim].Stop(); diff --git a/Code/geometry/needs/Needs.cc b/Code/geometry/needs/Needs.cc index 8c9ed0bab..bf1bcb561 100644 --- a/Code/geometry/needs/Needs.cc +++ b/Code/geometry/needs/Needs.cc @@ -15,9 +15,9 @@ namespace hemelb Needs::Needs(const site_t blockCount, const std::vector& readBlock, const proc_t readingGroupSize, - net::InterfaceDelegationNet & net, + net::MpiCommunicator& comm, bool shouldValidate_) : - procsWantingBlocksBuffer(blockCount), communicator(net.GetCommunicator()), readingGroupSize(readingGroupSize), shouldValidate(shouldValidate_) + procsWantingBlocksBuffer(blockCount), communicator(comm), readingGroupSize(readingGroupSize), shouldValidate(shouldValidate_) { // Compile the blocks needed here into an array of indices, instead of an array of bools std::vector > blocksNeededHere(readingGroupSize); @@ -31,33 +31,25 @@ namespace hemelb // Share the counts of needed blocks int blocksNeededSize[readingGroupSize]; - std::vector blocksNeededSizes(communicator.Size()); + std::vector blocksNeededSizes; for (proc_t readingCore = 0; readingCore < readingGroupSize; readingCore++) { blocksNeededSize[readingCore] = blocksNeededHere[readingCore].size(); - net.RequestGatherSend(blocksNeededSize[readingCore], readingCore); - - } - if (communicator.Rank() < readingGroupSize) - { - net.RequestGatherReceive(blocksNeededSizes); + auto tmp = communicator.Gather(blocksNeededSize[readingCore], readingCore); + if (readingCore == communicator.Rank()) + blocksNeededSizes = std::move(tmp); } - net.Dispatch(); + // Communicate the arrays of needed blocks - + std::vector blocksNeededOn; for (proc_t readingCore = 0; readingCore < readingGroupSize; readingCore++) { - net.RequestGatherVSend(blocksNeededHere[readingCore], readingCore); + auto tmp = communicator.GatherV(blocksNeededHere[readingCore], blocksNeededSizes, readingCore); + if (readingCore == communicator.Rank()) + blocksNeededOn = std::move(tmp); } - std::vector blocksNeededOn; - - if (communicator.Rank() < readingGroupSize) - { - net.RequestGatherVReceive(blocksNeededOn, blocksNeededSizes); - } - net.Dispatch(); if (communicator.Rank() < readingGroupSize) { int needsPassed = 0; diff --git a/Code/geometry/needs/Needs.h b/Code/geometry/needs/Needs.h index f6c8ab31e..c49d0da1f 100644 --- a/Code/geometry/needs/Needs.h +++ b/Code/geometry/needs/Needs.h @@ -7,8 +7,9 @@ #ifndef HEMELB_GEOMETRY_NEEDS_NEEDS_H #define HEMELB_GEOMETRY_NEEDS_NEEDS_H #include -#include "net/net.h" -#include "net/IOCommunicator.h" +#include "units.h" +#include "net/MpiCommunicator.h" + namespace hemelb { namespace geometry @@ -20,6 +21,7 @@ namespace hemelb * Class defining HemeLB needs communication Used by geometry reader to know where to send which blocks. */ + class Needs { public: @@ -28,12 +30,12 @@ namespace hemelb * @param BlockCount Count of blocks * @param readBlock Which cores need which blocks, as an array of booleans. * @param readingGroupSize Number sof cores to use for reading blocks - * @param net Instance of Net communication class to use. + * @param comm MPI communicator. */ Needs(const site_t blockCount, const std::vector& readBlock, const proc_t readingGroupSize, - net::InterfaceDelegationNet &net, + net::MpiCommunicator& comm, bool shouldValidate); // Temporarily during the refactor, constructed just to abstract the block sharing bit /*** diff --git a/Code/net/MpiCommunicator.hpp b/Code/net/MpiCommunicator.hpp index 4c1a21cdb..64807ac10 100644 --- a/Code/net/MpiCommunicator.hpp +++ b/Code/net/MpiCommunicator.hpp @@ -169,7 +169,7 @@ namespace hemelb total += recvcounts[i]; } // set up recv buffer - ans.resize(np); + ans.resize(total); recvbuf = ans.data(); } From 8acfa4ae6a839adc4a7de34258b794f9ca86749a Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 17 Oct 2016 12:33:51 +0100 Subject: [PATCH 53/99] remove gathering from the net object --- Code/CMakeLists.txt | 2 - Code/net/BaseNet.cc | 6 - Code/net/BaseNet.h | 19 +-- Code/net/BuildInfo.h.in | 1 - Code/net/CMakeLists.txt | 2 - Code/net/mixins/InterfaceDelegationNet.h | 92 +------------- Code/net/mixins/StoringNet.cc | 34 ------ Code/net/mixins/StoringNet.h | 10 -- Code/net/mixins/gathers/SeparatedGathers.cc | 114 ------------------ Code/net/mixins/gathers/SeparatedGathers.h | 30 ----- .../mixins/gathers/ViaPointPointGathers.cc | 98 --------------- .../net/mixins/gathers/ViaPointPointGathers.h | 33 ----- Code/net/mixins/mixins.h | 2 - Code/net/net.h | 6 +- Code/reporting/BuildInfo.h.in | 2 - Code/resources/report.txt.ctp | 1 - Code/resources/report.xml.ctp | 1 - 17 files changed, 4 insertions(+), 449 deletions(-) delete mode 100644 Code/net/mixins/gathers/SeparatedGathers.cc delete mode 100644 Code/net/mixins/gathers/SeparatedGathers.h delete mode 100644 Code/net/mixins/gathers/ViaPointPointGathers.cc delete mode 100644 Code/net/mixins/gathers/ViaPointPointGathers.h diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index f8f1c2e8c..08d9a55f1 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -56,8 +56,6 @@ set(HEMELB_WALL_OUTLET_BOUNDARY "NASHZEROTHORDERPRESSURESBB" CACHE STRING "Select the boundary conditions to be used at corners between walls and outlets (NASHZEROTHORDERPRESSURESBB,NASHZEROTHORDERPRESSUREBFL,LADDIOLETSBB,LADDIOLETBFL)") set(HEMELB_POINTPOINT_IMPLEMENTATION Coalesce CACHE STRING "Point to point comms implementation, choose 'Coalesce', 'Separated', or 'Immediate'" ) -set(HEMELB_GATHERS_IMPLEMENTATION Separated - CACHE STRING "Gather comms implementation, choose 'Separated', or 'ViaPointPoint'" ) option(HEMELB_SEPARATE_CONCERNS "Communicate for each concern separately" OFF) # Add warnings flags to development build types diff --git a/Code/net/BaseNet.cc b/Code/net/BaseNet.cc index 633132a97..46109924a 100644 --- a/Code/net/BaseNet.cc +++ b/Code/net/BaseNet.cc @@ -37,8 +37,6 @@ namespace hemelb void BaseNet::Receive() { - ReceiveGathers(); - ReceiveGatherVs(); // Ensure collectives are called before point-to-point, as some implementing mixins implement collectives via point-to-point ReceivePointToPoint(); } @@ -46,8 +44,6 @@ namespace hemelb void BaseNet::Send() { - SendGathers(); - SendGatherVs(); // Ensure collectives are called before point-to-point, as some implementing mixins implement collectives via point-to-point SendPointToPoint(); } @@ -56,8 +52,6 @@ namespace hemelb { SyncPointsCounted++; //DTMP: counter for monitoring purposes. - WaitGathers(); - WaitGatherVs(); WaitPointToPoint(); displacementsBuffer.clear(); diff --git a/Code/net/BaseNet.h b/Code/net/BaseNet.h index c886fa827..b636cb12f 100644 --- a/Code/net/BaseNet.h +++ b/Code/net/BaseNet.h @@ -59,32 +59,15 @@ namespace hemelb } protected: virtual void SendPointToPoint()=0; - virtual void SendGathers()=0; - virtual void SendGatherVs()=0; - virtual void ReceiveGathers()=0; - virtual void ReceiveGatherVs()=0; virtual void ReceivePointToPoint()=0; virtual void WaitPointToPoint()=0; - virtual void WaitGathers()=0; - virtual void WaitGatherVs()=0; // Interfaces exposing MPI_Datatype, not intended for client class use virtual void RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type)=0; virtual void RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type)=0; - - - /* - * Blocking gathers are implemented in MPI as a single call for both send/receive - * But, here we separate send and receive parts, since this interface may one day be used for - * nonblocking collectives. - */ - virtual void RequestGatherVSendImpl(void* buffer, int count, proc_t toRank, MPI_Datatype type)=0; - virtual void RequestGatherReceiveImpl(void* buffer, MPI_Datatype type)=0; - virtual void RequestGatherSendImpl(void* buffer, proc_t toRank, MPI_Datatype type)=0; - virtual void RequestGatherVReceiveImpl(void* buffer, int * displacements, int *counts, MPI_Datatype type)=0; - + std::vector & GetDisplacementsBuffer(); std::vector & GetCountsBuffer(); diff --git a/Code/net/BuildInfo.h.in b/Code/net/BuildInfo.h.in index 334124fad..28778d73b 100644 --- a/Code/net/BuildInfo.h.in +++ b/Code/net/BuildInfo.h.in @@ -11,7 +11,6 @@ namespace hemelb namespace net { typedef @HEMELB_POINTPOINT_IMPLEMENTATION@PointPoint PointPointImpl ; - typedef @HEMELB_GATHERS_IMPLEMENTATION@Gathers GathersImpl ; #cmakedefine HEMELB_SEPARATE_CONCERNS #ifdef HEMELB_SEPARATE_CONCERNS static const bool separate_communications = true; diff --git a/Code/net/CMakeLists.txt b/Code/net/CMakeLists.txt index a7d9da579..63f3b0503 100644 --- a/Code/net/CMakeLists.txt +++ b/Code/net/CMakeLists.txt @@ -13,8 +13,6 @@ IOCommunicator.cc mixins/pointpoint/CoalescePointPoint.cc mixins/pointpoint/SeparatedPointPoint.cc mixins/pointpoint/ImmediatePointPoint.cc -mixins/gathers/SeparatedGathers.cc -mixins/gathers/ViaPointPointGathers.cc mixins/StoringNet.cc ProcComms.cc phased/StepManager.cc) configure_file ( diff --git a/Code/net/mixins/InterfaceDelegationNet.h b/Code/net/mixins/InterfaceDelegationNet.h index 5b33bf260..b9d0f01c2 100644 --- a/Code/net/mixins/InterfaceDelegationNet.h +++ b/Code/net/mixins/InterfaceDelegationNet.h @@ -47,65 +47,6 @@ namespace hemelb RequestReceive(&payload[0], payload.size(), toRank); } - template - void RequestGatherVReceive(std::vector& bigBuffer, const std::vector& countsIn) - { - // It may seem inefficient to go through the list twice (it is), but it avoids any nastiness - // in 32 / 64 bit problems, because we can take our displacement as simply the difference - // between two addresses (by allocating the bigBuffer before the address calculation). - - int totalCount = 0; - for (std::vector::const_iterator count_iterator = countsIn.begin(); - count_iterator != countsIn.end(); count_iterator++) - { - totalCount += *count_iterator; - } - - // Allocate the large buffer we'll receive all data into. - bigBuffer.resize(totalCount); - bigBuffer.reserve(1); - - std::vector & displacements = this->GetDisplacementsBuffer(); - std::vector &counts = this->GetCountsBuffer(); - - // Now store the displacement and count for each sending proc. - int countSoFar = 0; - - for (std::vector::const_iterator count_iterator = countsIn.begin(); - count_iterator != countsIn.end(); count_iterator++) - { - int nextCount = *count_iterator; - - counts.push_back(nextCount); - displacements.push_back(&bigBuffer[countSoFar] - &bigBuffer[0]); - - countSoFar += nextCount; - } - - // And store the pointer and culminating arrays. - RequestGatherVReceive(&bigBuffer.front(), &displacements.front(), &counts.front()); - } - - template - void RequestGatherReceive(std::vector &buffer) - { - // Ensure vector has some underlying array, even if it's unused. - buffer.reserve(1); - RequestGatherReceive(&buffer.front()); - } - - template - void RequestGatherSend(T& value, proc_t toRank) - { - RequestGatherSend(&value, toRank); - } - - template - void RequestGatherVSend(std::vector &payload, proc_t toRank) - { - RequestGatherVSend(&payload.front(), payload.size(), toRank); - } - template void RequestSend(const T* pointer, int count, proc_t rank) { @@ -118,38 +59,7 @@ namespace hemelb RequestReceiveImpl(pointer, count, rank, MpiDataType()); } - /* - * Blocking gathers are implemented in MPI as a single call for both send/receive - * But, here we separate send and receive parts, since this interface may one day be used for - * nonblocking collectives. - */ - - template - void RequestGatherVSend(T* buffer, int count, proc_t toRank) - { - RequestGatherVSendImpl(buffer, count, toRank, MpiDataType()); - } - - template - void RequestGatherReceive(T* buffer) - { - RequestGatherReceiveImpl(buffer, MpiDataType()); - } - - template - void RequestGatherSend(T* buffer, proc_t toRank) - { - RequestGatherSendImpl(buffer, toRank, MpiDataType()); - } - - template - void RequestGatherVReceive(T* buffer, int * displacements, int *counts) - { - RequestGatherVReceiveImpl(buffer, displacements, counts, MpiDataType()); - } - - } - ; + }; } } #endif diff --git a/Code/net/mixins/StoringNet.cc b/Code/net/mixins/StoringNet.cc index 595b7aa02..9e8529e16 100644 --- a/Code/net/mixins/StoringNet.cc +++ b/Code/net/mixins/StoringNet.cc @@ -29,39 +29,5 @@ namespace hemelb } } - /* - * Blocking gathers are implemented in MPI as a single call for both send/receive - * But, here we separate send and receive parts, since this interface may one day be used for - * nonblocking collectives. - */ - - void StoringNet::RequestGatherVSendImpl(void* buffer, int count, proc_t toRank, - MPI_Datatype type) - { - gatherVSendProcessorComms[toRank].push_back(SimpleRequest(buffer, count, type, toRank)); - } - - void StoringNet::RequestGatherReceiveImpl(void* buffer, MPI_Datatype type) - { - /* - * Dummy rank to ScalarRequest of zero -- gathers always receive to the core where the receive request is made. - * Avoids defining another type of request. - */ - gatherReceiveProcessorComms.push_back(ScalarRequest(buffer, type, 0)); - } - - void StoringNet::RequestGatherSendImpl(void* buffer, proc_t toRank, MPI_Datatype type) - { - gatherSendProcessorComms[toRank].push_back(ScalarRequest(buffer, type, toRank)); - } - - void StoringNet::RequestGatherVReceiveImpl(void* buffer, int * displacements, int *counts, - MPI_Datatype type) - { - gatherVReceiveProcessorComms.push_back(GatherVReceiveRequest(buffer, - displacements, - counts, - type)); - } } } diff --git a/Code/net/mixins/StoringNet.h b/Code/net/mixins/StoringNet.h index 2c873827c..1536fc628 100644 --- a/Code/net/mixins/StoringNet.h +++ b/Code/net/mixins/StoringNet.h @@ -21,11 +21,6 @@ namespace hemelb virtual void RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); virtual void RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); - void RequestGatherVSendImpl(void* buffer, int count, proc_t toRank, MPI_Datatype type); - void RequestGatherReceiveImpl(void* buffer, MPI_Datatype type); - - void RequestGatherSendImpl(void* buffer, proc_t toRank, MPI_Datatype type); - void RequestGatherVReceiveImpl(void* buffer, int * displacements, int *counts, MPI_Datatype type); protected: /** @@ -35,11 +30,6 @@ namespace hemelb std::map sendProcessorComms; std::map receiveProcessorComms; - std::map gatherVSendProcessorComms; - GatherVReceiveProcComms gatherVReceiveProcessorComms; - - std::map gatherSendProcessorComms; - GatherProcComms gatherReceiveProcessorComms; }; } } diff --git a/Code/net/mixins/gathers/SeparatedGathers.cc b/Code/net/mixins/gathers/SeparatedGathers.cc deleted file mode 100644 index 2b0ed7ac6..000000000 --- a/Code/net/mixins/gathers/SeparatedGathers.cc +++ /dev/null @@ -1,114 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/gathers/SeparatedGathers.h" -namespace hemelb -{ - namespace net - { - SeparatedGathers::SeparatedGathers(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms) - { - } - - void SeparatedGathers::WaitGathers() - { - /* - * For each of the stored gather send requests... - * Each of which, is all the requests to gather to a certain root... - */ - for (std::map::iterator send_it = gatherSendProcessorComms.begin(); - send_it != gatherSendProcessorComms.end(); ++send_it) - { - /* - * send_it->first is the rank to which the gather must be sent - * send_it->second is a VECTOR of ScalarRequests, each a stored request defining Type and Pointer. - * There is one entry, for each request to gather to that root. - */ - /* - * If I am sending to MYSELF, then I must assemble a gather request for sending and receiving myself - */ - if (send_it->first == communicator.Rank()) - { - /* - * I may be the ROOT of several gathers. - * If I am, then we hope, the sends and receives were defined in the same order. - * So, we assume the first of the receive requests, matches the first of the send-to-self requests - */ - int gather_index = 0; - for (GatherProcComms::iterator receive_it = gatherReceiveProcessorComms.begin(); - receive_it != gatherReceiveProcessorComms.end(); ++receive_it) - { - ScalarRequest toself = send_it->second[gather_index]; - - /* - * So now, I can send/receive to myself, for this request - */HEMELB_MPI_CALL(MPI_Gather, - (toself.Pointer, 1, toself.Type, receive_it->Pointer, 1, receive_it->Type, communicator.Rank(), communicator)); - ++gather_index; - } - } - else - /* - * I am not sending to myself, so I just need to send each of the gathers to the given root - * No attempt is made to coalesce several gathers to the same root. - */ - { - for (GatherProcComms::iterator req = send_it->second.begin(); - req != send_it->second.end(); req++) - { - HEMELB_MPI_CALL(MPI_Gather, - (req->Pointer, 1, req->Type, NULL, 1, req->Type, send_it->first, communicator)); - } - } - } - - gatherSendProcessorComms.clear(); - gatherReceiveProcessorComms.clear(); - } - - void SeparatedGathers::WaitGatherVs() - { - - for (std::map::iterator send_it = gatherVSendProcessorComms.begin(); - send_it != gatherVSendProcessorComms.end(); ++send_it) - { - - if (send_it->first == communicator.Rank()) - { - int gather_index = 0; - - for (GatherVReceiveProcComms::iterator receive_it = gatherVReceiveProcessorComms.begin(); - receive_it != gatherVReceiveProcessorComms.end(); ++receive_it) - { - /*** - * Again, as for WaitGather, we assume that send/receive requests were placed in parallel ordering... - */ - SimpleRequest toself = send_it->second[gather_index]; - - HEMELB_MPI_CALL(MPI_Gatherv, - ( toself.Pointer, toself.Count, toself.Type, receive_it->Pointer, receive_it->Counts, receive_it->Displacements, receive_it->Type, communicator.Rank(), communicator)); - ++gather_index; - } - } - else - { - - for (ProcComms::iterator req = send_it->second.begin(); req != send_it->second.end(); - req++) - { - HEMELB_MPI_CALL(MPI_Gatherv, - ( req->Pointer, req->Count, req->Type, NULL, NULL, NULL, req->Type, send_it->first, communicator)); - } - - } - } - gatherVSendProcessorComms.clear(); - gatherVReceiveProcessorComms.clear(); - } - } -} - diff --git a/Code/net/mixins/gathers/SeparatedGathers.h b/Code/net/mixins/gathers/SeparatedGathers.h deleted file mode 100644 index 91423e73b..000000000 --- a/Code/net/mixins/gathers/SeparatedGathers.h +++ /dev/null @@ -1,30 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_GATHERS_SEPARATEDGATHERS_H -#define HEMELB_NET_MIXINS_GATHERS_SEPARATEDGATHERS_H - -#include "net/mixins/StoringNet.h" - -namespace hemelb{ - namespace net{ - - class SeparatedGathers : public virtual StoringNet - { - public: - SeparatedGathers(const MpiCommunicator& comms); - private: - void ReceiveGathers(){} - void SendGathers(){} - void ReceiveGatherVs(){} - void SendGatherVs(){} - void WaitGathers(); - void WaitGatherVs(); - }; - - } -} -#endif diff --git a/Code/net/mixins/gathers/ViaPointPointGathers.cc b/Code/net/mixins/gathers/ViaPointPointGathers.cc deleted file mode 100644 index 933917978..000000000 --- a/Code/net/mixins/gathers/ViaPointPointGathers.cc +++ /dev/null @@ -1,98 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/gathers/ViaPointPointGathers.h" -namespace hemelb -{ - namespace net - { - ViaPointPointGathers::ViaPointPointGathers(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms) - { - } - - void ViaPointPointGathers::ReceiveGathers() - { - for (GatherProcComms::iterator receive_it = gatherReceiveProcessorComms.begin(); - receive_it != gatherReceiveProcessorComms.end(); ++receive_it) - { - - int size; - MPI_Type_size(receive_it->Type, &size); - - for (int source_rank = 0; source_rank < communicator.Size(); source_rank++) - { - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - RequestReceiveImpl(static_cast(receive_it->Pointer) + size * source_rank, - 1, - source_rank, - receive_it->Type); - } - } - - gatherReceiveProcessorComms.clear(); - } - void ViaPointPointGathers::SendGathers() - { - for (std::map::iterator send_it = gatherSendProcessorComms.begin(); - send_it != gatherSendProcessorComms.end(); ++send_it) - { - - for (GatherProcComms::iterator req = send_it->second.begin(); req != send_it->second.end(); - req++) - { - RequestSendImpl(req->Pointer, 1, send_it->first, req->Type); - } - - } - - gatherSendProcessorComms.clear(); - } - void ViaPointPointGathers::ReceiveGatherVs() - { - for (GatherVReceiveProcComms::iterator receive_it = gatherVReceiveProcessorComms.begin(); - receive_it != gatherVReceiveProcessorComms.end(); ++receive_it) - { - - int size; - MPI_Type_size(receive_it->Type, &size); - - for (int source_rank = 0; source_rank < communicator.Size(); source_rank++) - { - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - // Note that MPI Displacements are given in the arithmetic appropriate to the MPI_Datatype, not void*, i.e. in units of the size - // It will also potentially fail, if the MPI_Datatype used, is a sparse (strided) type. - // This class is intended for timing and testing use, not production use. - RequestReceiveImpl(static_cast(receive_it->Pointer) - + receive_it->Displacements[source_rank] * size, - receive_it->Counts[source_rank], - source_rank, - receive_it->Type); - } - } - - gatherVReceiveProcessorComms.clear(); - } - void ViaPointPointGathers::SendGatherVs() - { - for (std::map::iterator send_it = gatherVSendProcessorComms.begin(); - send_it != gatherVSendProcessorComms.end(); ++send_it) - { - - for (ProcComms::iterator req = send_it->second.begin(); req != send_it->second.end(); req++) - { - RequestSendImpl(req->Pointer, req->Count, send_it->first, req->Type); - } - - } - - gatherVSendProcessorComms.clear(); - } - } -} - diff --git a/Code/net/mixins/gathers/ViaPointPointGathers.h b/Code/net/mixins/gathers/ViaPointPointGathers.h deleted file mode 100644 index eb4445c38..000000000 --- a/Code/net/mixins/gathers/ViaPointPointGathers.h +++ /dev/null @@ -1,33 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_GATHERS_VIAPOINTPOINTGATHERS_H -#define HEMELB_NET_MIXINS_GATHERS_VIAPOINTPOINTGATHERS_H - -#include "net/mixins/StoringNet.h" - -namespace hemelb{ - namespace net{ - /*** - * Reimplement gathers via point-point calls - * This code is not robust for working with gathers of complex defined datatypes. - * It is intended for testing and performance measurement use, and should not be used in production. - */ - class ViaPointPointGathers : public virtual StoringNet - { - public: - ViaPointPointGathers(const MpiCommunicator& comms); - private: - void ReceiveGathers(); - void SendGathers(); - void ReceiveGatherVs(); - void SendGatherVs(); - void WaitGathers(){} - void WaitGatherVs(){} - }; - } -} -#endif diff --git a/Code/net/mixins/mixins.h b/Code/net/mixins/mixins.h index d887058ea..68c752031 100644 --- a/Code/net/mixins/mixins.h +++ b/Code/net/mixins/mixins.h @@ -11,7 +11,5 @@ #include "net/mixins/pointpoint/ImmediatePointPoint.h" #include "net/mixins/pointpoint/SeparatedPointPoint.h" #include "net/mixins/StoringNet.h" -#include "net/mixins/gathers/SeparatedGathers.h" #include "net/mixins/InterfaceDelegationNet.h" -#include "net/mixins/gathers/ViaPointPointGathers.h" #endif diff --git a/Code/net/net.h b/Code/net/net.h index 36f3c73fa..ccd6f491d 100644 --- a/Code/net/net.h +++ b/Code/net/net.h @@ -15,14 +15,12 @@ namespace hemelb namespace net { class Net : public PointPointImpl, - public InterfaceDelegationNet, - public GathersImpl + public InterfaceDelegationNet { public: Net(const MpiCommunicator &communicator) : BaseNet(communicator), StoringNet(communicator), PointPointImpl(communicator), - InterfaceDelegationNet(communicator), - GathersImpl(communicator) + InterfaceDelegationNet(communicator) { } }; diff --git a/Code/reporting/BuildInfo.h.in b/Code/reporting/BuildInfo.h.in index 07b01c5e3..d8b5fe34f 100644 --- a/Code/reporting/BuildInfo.h.in +++ b/Code/reporting/BuildInfo.h.in @@ -26,7 +26,6 @@ namespace hemelb static const std::string wall_inlet_boundary_condition="@HEMELB_WALL_INLET_BOUNDARY@"; static const std::string wall_outlet_boundary_condition="@HEMELB_WALL_OUTLET_BOUNDARY@"; static const std::string point_point_impl="@HEMELB_POINTPOINT_IMPLEMENTATION@"; - static const std::string gathers_impl="@HEMELB_GATHERS_IMPLEMENTATION@"; static const std::string separate_concerns="@HEMELB_SEPARATE_CONCERNS@"; @@ -48,7 +47,6 @@ namespace hemelb build->SetValue("WALL_INLET_BOUNDARY_CONDITION", wall_inlet_boundary_condition); build->SetValue("WALL_OUTLET_BOUNDARY_CONDITION", wall_outlet_boundary_condition); build->SetValue("SEPARATE_CONCERNS",separate_concerns); - build->SetValue("GATHERS_IMPLEMENTATION",gathers_impl); build->SetValue("POINTPOINT_IMPLEMENTATION",point_point_impl); } }; diff --git a/Code/resources/report.txt.ctp b/Code/resources/report.txt.ctp index f6ab71b2b..8b74dc4e5 100644 --- a/Code/resources/report.txt.ctp +++ b/Code/resources/report.txt.ctp @@ -41,6 +41,5 @@ Wall/iolet boundary condition: {{WALL_IOLET_BOUNDARY_CONDITION}} Communications options: Point to point implementation: {{POINTPOINT_IMPLEMENTATION}} -Gathers implementation: {{GATHERS_IMPLEMENTATION}} Separated concerns: {{SEPARATE_CONCERNS}} {{/BUILD}} \ No newline at end of file diff --git a/Code/resources/report.xml.ctp b/Code/resources/report.xml.ctp index 78b3e8c9c..6e8ee0e92 100644 --- a/Code/resources/report.xml.ctp +++ b/Code/resources/report.xml.ctp @@ -26,7 +26,6 @@ {{POINTPOINT_IMPLEMENTATION}} - {{GATHERS_IMPLEMENTATION}} {{SEPARATE_CONCERNS}} From b6a7b3f047e43c13f09ed4c341227e3770a86737 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 18 Oct 2016 14:48:46 +0100 Subject: [PATCH 54/99] move debug over to new style comms --- Code/{net => comm}/MpiDataType.cc | 0 Code/{net => comm}/MpiDataType.h | 0 Code/{net => comm}/MpiEnvironment.cc | 0 Code/{net => comm}/MpiEnvironment.h | 0 Code/{net => comm}/MpiError.cc | 0 Code/{net => comm}/MpiError.h | 0 Code/{net => comm}/MpiFile.cc | 0 Code/{net => comm}/MpiFile.h | 0 Code/{net => comm}/MpiFile.hpp | 0 Code/{net => comm}/MpiGroup.cc | 0 Code/{net => comm}/MpiGroup.h | 0 Code/{net => comm}/MpiRequest.h | 0 Code/{net => comm}/MpiStatus.cc | 0 Code/{net => comm}/MpiStatus.h | 0 Code/debug/Debugger.cc | 4 +- Code/debug/Debugger.h | 8 +- Code/debug/OSX/OsxDebugger.cc | 4 +- Code/debug/OSX/OsxDebugger.h | 2 +- Code/debug/common/ActiveDebugger.cc | 10 +- Code/debug/common/ActiveDebugger.h | 2 +- Code/debug/none/NullDebugger.cc | 2 +- Code/debug/none/NullDebugger.h | 2 +- Code/net/IOCommunicator.cc | 29 --- Code/net/IOCommunicator.h | 33 --- Code/net/MpiCommunicator.cc | 139 ------------ Code/net/MpiCommunicator.h | 188 ---------------- Code/net/MpiCommunicator.hpp | 322 --------------------------- Code/net/MpiConstness.h.in | 30 --- 28 files changed, 16 insertions(+), 759 deletions(-) rename Code/{net => comm}/MpiDataType.cc (100%) rename Code/{net => comm}/MpiDataType.h (100%) rename Code/{net => comm}/MpiEnvironment.cc (100%) rename Code/{net => comm}/MpiEnvironment.h (100%) rename Code/{net => comm}/MpiError.cc (100%) rename Code/{net => comm}/MpiError.h (100%) rename Code/{net => comm}/MpiFile.cc (100%) rename Code/{net => comm}/MpiFile.h (100%) rename Code/{net => comm}/MpiFile.hpp (100%) rename Code/{net => comm}/MpiGroup.cc (100%) rename Code/{net => comm}/MpiGroup.h (100%) rename Code/{net => comm}/MpiRequest.h (100%) rename Code/{net => comm}/MpiStatus.cc (100%) rename Code/{net => comm}/MpiStatus.h (100%) delete mode 100644 Code/net/IOCommunicator.cc delete mode 100644 Code/net/IOCommunicator.h delete mode 100644 Code/net/MpiCommunicator.cc delete mode 100644 Code/net/MpiCommunicator.h delete mode 100644 Code/net/MpiCommunicator.hpp delete mode 100644 Code/net/MpiConstness.h.in diff --git a/Code/net/MpiDataType.cc b/Code/comm/MpiDataType.cc similarity index 100% rename from Code/net/MpiDataType.cc rename to Code/comm/MpiDataType.cc diff --git a/Code/net/MpiDataType.h b/Code/comm/MpiDataType.h similarity index 100% rename from Code/net/MpiDataType.h rename to Code/comm/MpiDataType.h diff --git a/Code/net/MpiEnvironment.cc b/Code/comm/MpiEnvironment.cc similarity index 100% rename from Code/net/MpiEnvironment.cc rename to Code/comm/MpiEnvironment.cc diff --git a/Code/net/MpiEnvironment.h b/Code/comm/MpiEnvironment.h similarity index 100% rename from Code/net/MpiEnvironment.h rename to Code/comm/MpiEnvironment.h diff --git a/Code/net/MpiError.cc b/Code/comm/MpiError.cc similarity index 100% rename from Code/net/MpiError.cc rename to Code/comm/MpiError.cc diff --git a/Code/net/MpiError.h b/Code/comm/MpiError.h similarity index 100% rename from Code/net/MpiError.h rename to Code/comm/MpiError.h diff --git a/Code/net/MpiFile.cc b/Code/comm/MpiFile.cc similarity index 100% rename from Code/net/MpiFile.cc rename to Code/comm/MpiFile.cc diff --git a/Code/net/MpiFile.h b/Code/comm/MpiFile.h similarity index 100% rename from Code/net/MpiFile.h rename to Code/comm/MpiFile.h diff --git a/Code/net/MpiFile.hpp b/Code/comm/MpiFile.hpp similarity index 100% rename from Code/net/MpiFile.hpp rename to Code/comm/MpiFile.hpp diff --git a/Code/net/MpiGroup.cc b/Code/comm/MpiGroup.cc similarity index 100% rename from Code/net/MpiGroup.cc rename to Code/comm/MpiGroup.cc diff --git a/Code/net/MpiGroup.h b/Code/comm/MpiGroup.h similarity index 100% rename from Code/net/MpiGroup.h rename to Code/comm/MpiGroup.h diff --git a/Code/net/MpiRequest.h b/Code/comm/MpiRequest.h similarity index 100% rename from Code/net/MpiRequest.h rename to Code/comm/MpiRequest.h diff --git a/Code/net/MpiStatus.cc b/Code/comm/MpiStatus.cc similarity index 100% rename from Code/net/MpiStatus.cc rename to Code/comm/MpiStatus.cc diff --git a/Code/net/MpiStatus.h b/Code/comm/MpiStatus.h similarity index 100% rename from Code/net/MpiStatus.h rename to Code/comm/MpiStatus.h diff --git a/Code/debug/Debugger.cc b/Code/debug/Debugger.cc index beb6f5b0e..6611c5183 100644 --- a/Code/debug/Debugger.cc +++ b/Code/debug/Debugger.cc @@ -13,7 +13,7 @@ namespace hemelb namespace debug { - Debugger* Debugger::Init(bool active, const char * const executable, const net::MpiCommunicator& comm) + Debugger* Debugger::Init(bool active, const char * const executable, const comm::Communicator* comm) { /* Static member function that implements the singleton pattern. * Use the namespace function PlatformDebuggerFactory to @@ -40,7 +40,7 @@ namespace hemelb // Init static members Debugger* Debugger::singleton = NULL; - Debugger::Debugger(const char* const executable, const net::MpiCommunicator& comm) : + Debugger::Debugger(const char* const executable, const comm::Communicator* comm) : mExecutable(executable), mCommunicator(comm) { } diff --git a/Code/debug/Debugger.h b/Code/debug/Debugger.h index acf3889c8..705fa99d9 100644 --- a/Code/debug/Debugger.h +++ b/Code/debug/Debugger.h @@ -8,7 +8,7 @@ #define HEMELB_DEBUG_DEBUGGER_H #include -#include "net/MpiCommunicator.h" +#include "comm/Communicator.h" namespace hemelb { @@ -22,20 +22,20 @@ namespace hemelb */ public: // the singleton pattern - static Debugger* Init(bool active, const char *const, const net::MpiCommunicator& comm); + static Debugger* Init(bool active, const char *const, const comm::Communicator* comm); static Debugger* Get(void); virtual void BreakHere(void) = 0; virtual void Print(const char* iFormat, ...) = 0; protected: - Debugger(const char* const executable, const net::MpiCommunicator& comm); + Debugger(const char* const executable, const comm::Communicator* comm); virtual ~Debugger(); virtual void Attach() = 0; std::string mExecutable; - const net::MpiCommunicator mCommunicator; + const comm::Communicator* mCommunicator; // Singleton pattern static Debugger* singleton; diff --git a/Code/debug/OSX/OsxDebugger.cc b/Code/debug/OSX/OsxDebugger.cc index 2d642cac8..245a6d5e0 100644 --- a/Code/debug/OSX/OsxDebugger.cc +++ b/Code/debug/OSX/OsxDebugger.cc @@ -5,14 +5,14 @@ // license in the file LICENSE. #include - +#include "Exception.h" #include "debug/OSX/OsxDebugger.h" #include namespace hemelb { namespace debug { - OsxDebugger::OsxDebugger(const char* const executable, const net::MpiCommunicator& comm) : + OsxDebugger::OsxDebugger(const char* const executable, const comm::Communicator* comm) : ActiveDebugger(executable, comm) { } diff --git a/Code/debug/OSX/OsxDebugger.h b/Code/debug/OSX/OsxDebugger.h index 1fe56f5a7..e570bd4b8 100644 --- a/Code/debug/OSX/OsxDebugger.h +++ b/Code/debug/OSX/OsxDebugger.h @@ -22,7 +22,7 @@ namespace hemelb const std::string GetPlatformScript(void) const; // C'tor... - OsxDebugger(const char* const executable, const net::MpiCommunicator& comm); + OsxDebugger(const char* const executable, const comm::Communicator* comm); // ... which the factory function needs to be able to get at. friend class Debugger; diff --git a/Code/debug/common/ActiveDebugger.cc b/Code/debug/common/ActiveDebugger.cc index 165f6e5fb..da9d3e365 100644 --- a/Code/debug/common/ActiveDebugger.cc +++ b/Code/debug/common/ActiveDebugger.cc @@ -19,8 +19,6 @@ #include #include -#include "net/mpi.h" - #include "debug/common/ActiveDebugger.h" namespace hemelb @@ -28,7 +26,7 @@ namespace hemelb namespace debug { - ActiveDebugger::ActiveDebugger(const char* const executable, const net::MpiCommunicator& comm) : + ActiveDebugger::ActiveDebugger(const char* const executable, const comm::Communicator* comm) : Debugger(executable, comm), mAmAttached(false), mPIds() { } @@ -69,7 +67,7 @@ namespace hemelb // To rank 0 GatherProcessIds(); - if (mCommunicator.Rank() == 0) + if (mCommunicator->Rank() == 0) { childPid = fork(); // Fork gives the PID of the child to the parent and zero to the child @@ -84,7 +82,7 @@ namespace hemelb while (amWaiting) sleep(5); - if (mCommunicator.Rank() == 0) + if (mCommunicator->Rank() == 0) { // Reap the spawner int deadPid = waitpid(childPid, NULL, 0); @@ -99,7 +97,7 @@ namespace hemelb { int pId = getpid(); - mPIds = mCommunicator.Gather(pId, 0); + mPIds = mCommunicator->Gather(pId, 0); } void ActiveDebugger::SpawnDebuggers(void) diff --git a/Code/debug/common/ActiveDebugger.h b/Code/debug/common/ActiveDebugger.h index cfb110e4c..5aac28404 100644 --- a/Code/debug/common/ActiveDebugger.h +++ b/Code/debug/common/ActiveDebugger.h @@ -33,7 +33,7 @@ namespace hemelb typedef std::vector VoS;// Vector of Strings // C'tor - ActiveDebugger(const char* const executable, const net::MpiCommunicator& comm); + ActiveDebugger(const char* const executable, const comm::Communicator* comm); bool mAmAttached;// Indicate attachment state VoI mPIds;// vector of process IDs diff --git a/Code/debug/none/NullDebugger.cc b/Code/debug/none/NullDebugger.cc index 4c9a487e3..876aa128b 100644 --- a/Code/debug/none/NullDebugger.cc +++ b/Code/debug/none/NullDebugger.cc @@ -22,7 +22,7 @@ namespace hemelb { } - NullDebugger::NullDebugger(const char* const executable, const net::MpiCommunicator& comm) : + NullDebugger::NullDebugger(const char* const executable, const comm::Communicator* comm) : Debugger(executable, comm) { } diff --git a/Code/debug/none/NullDebugger.h b/Code/debug/none/NullDebugger.h index b36d07107..21e7ab2b6 100644 --- a/Code/debug/none/NullDebugger.h +++ b/Code/debug/none/NullDebugger.h @@ -21,7 +21,7 @@ namespace hemelb protected: void Attach(void); - NullDebugger(const char* const executable, const net::MpiCommunicator& comm); + NullDebugger(const char* const executable, const comm::Communicator* comm); friend class Debugger; }; diff --git a/Code/net/IOCommunicator.cc b/Code/net/IOCommunicator.cc deleted file mode 100644 index a2f88c97f..000000000 --- a/Code/net/IOCommunicator.cc +++ /dev/null @@ -1,29 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/IOCommunicator.h" -#include "net/mpi.h" - -namespace hemelb -{ - namespace net - { - IOCommunicator::IOCommunicator(const MpiCommunicator& comm) : MpiCommunicator(comm) - { - } - - bool IOCommunicator::OnIORank() const - { - return Rank() == GetIORank(); - } - - int IOCommunicator::GetIORank() const - { - return 0; - } - - } -} diff --git a/Code/net/IOCommunicator.h b/Code/net/IOCommunicator.h deleted file mode 100644 index 2d0e57590..000000000 --- a/Code/net/IOCommunicator.h +++ /dev/null @@ -1,33 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_IOCOMMUNICATOR_H -#define HEMELB_NET_IOCOMMUNICATOR_H - -//#include -//#include -// -//#include "constants.h" -#include "net/MpiCommunicator.h" - -namespace hemelb -{ - namespace net - { - /** - * An MPI communicator with a special I/O rank. - */ - class IOCommunicator : public MpiCommunicator - { - public: - IOCommunicator(const MpiCommunicator& comm); - bool OnIORank() const; - int GetIORank() const; - }; - } -} - -#endif /* HEMELB_NET_IOCOMMUNICATOR_H */ diff --git a/Code/net/MpiCommunicator.cc b/Code/net/MpiCommunicator.cc deleted file mode 100644 index 8ffff034e..000000000 --- a/Code/net/MpiCommunicator.cc +++ /dev/null @@ -1,139 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/MpiCommunicator.h" -#include "net/MpiGroup.h" -#include "net/MpiRequest.h" - -namespace hemelb -{ - namespace net - { - namespace - { - void Deleter(MPI_Comm* comm) - { - int finalized; - HEMELB_MPI_CALL(MPI_Finalized, (&finalized)); - if (!finalized) - HEMELB_MPI_CALL(MPI_Comm_free, (comm)); - delete comm; - } - } - - MpiCommunicator MpiCommunicator::World() - { - return MpiCommunicator(MPI_COMM_WORLD, false); - } - - MpiCommunicator::MpiCommunicator() : commPtr() - { - } - - MpiCommunicator::MpiCommunicator(MPI_Comm communicator, bool owner) : commPtr() - { - if (communicator == MPI_COMM_NULL) - return; - - if (owner) - { - commPtr.reset(new MPI_Comm(communicator), Deleter); - } - else - { - commPtr.reset(new MPI_Comm(communicator)); - } - } - - MpiCommunicator::~MpiCommunicator() - { - } - - bool operator==(const MpiCommunicator& comm1, const MpiCommunicator& comm2) - { - if (comm1) - { - if (comm2) - { - int result; - HEMELB_MPI_CALL(MPI_Comm_compare, - (comm1, comm2, &result)); - return result == MPI_IDENT; - } - return false; - } - return (!comm2); - } - - bool operator!=(const MpiCommunicator& comm1, const MpiCommunicator& comm2) - { - return ! (comm1 == comm2); - } - - int MpiCommunicator::Rank() const - { - int rank; - HEMELB_MPI_CALL(MPI_Comm_rank, (*commPtr, &rank)); - return rank; - } - - int MpiCommunicator::Size() const - { - int size; - HEMELB_MPI_CALL(MPI_Comm_size, (*commPtr, &size)); - return size; - } - - MpiGroup MpiCommunicator::Group() const - { - MPI_Group grp; - HEMELB_MPI_CALL(MPI_Comm_group, (*commPtr, &grp)); - return MpiGroup(grp, true); - } - - MpiCommunicator MpiCommunicator::Create(const MpiGroup& grp) const - { - MPI_Comm newComm; - HEMELB_MPI_CALL(MPI_Comm_create, (*commPtr, grp, &newComm)); - return MpiCommunicator(newComm, true); - } - - void MpiCommunicator::Abort(int errCode) const - { - HEMELB_MPI_CALL(MPI_Abort, (*commPtr, errCode)); - } - - MpiCommunicator MpiCommunicator::Duplicate() const - { - MPI_Comm newComm; - HEMELB_MPI_CALL(MPI_Comm_dup, (*commPtr, &newComm)); - return MpiCommunicator(newComm, true); - } - - void MpiCommunicator::Barrier() const - { - HEMELB_MPI_CALL(MPI_Barrier, (*commPtr)); - } - - MpiRequest MpiCommunicator::Ibarrier() const - { - MPI_Request req; - HEMELB_MPI_CALL(MPI_Ibarrier, (*commPtr, &req)); - return MpiRequest(req); - } - - bool MpiCommunicator::Iprobe(int source, int tag, MPI_Status* stat) const - { - int flag; - HEMELB_MPI_CALL( - MPI_Iprobe, - (source, tag, *commPtr, &flag, stat) - ); - return flag; - } - - } -} diff --git a/Code/net/MpiCommunicator.h b/Code/net/MpiCommunicator.h deleted file mode 100644 index 08b8e7a4f..000000000 --- a/Code/net/MpiCommunicator.h +++ /dev/null @@ -1,188 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MPICOMMUNICATOR_H -#define HEMELB_NET_MPICOMMUNICATOR_H - -#include -#include -#include "net/MpiError.h" - -namespace hemelb -{ - namespace net - { - class MpiGroup; - class MpiRequest; - class MpiStatus; - - class MpiCommunicator - { - public: - static MpiCommunicator World(); - - /** - * Constructor for an uninitialised communicator, equivalent to - * MPI_COMM_NULL - * @param communicator - */ - MpiCommunicator(); - - /** - * Class has virtual methods so should have virtual d'tor. - */ - virtual ~MpiCommunicator(); - - /** - * Returns the local rank on the communicator - * @return - */ - virtual int Rank() const; - - /** - * Returns the size of the communicator (i.e. total number of procs involved). - * @return - */ - virtual int Size() const; - - /** - * Creates a new communicator - see MPI_COMM_GROUP - * @param Group which is a subset of the group of this communicator. - * @return New communicator. - */ - MpiCommunicator Create(const MpiGroup& grp) const; - - /** - * Allow implicit casts to MPI_Comm - * @return The underlying MPI communicator. - */ - operator MPI_Comm() const - { - return *commPtr; - } - /** - * Is this communicator valid? I.e. not equal to MPI_COMM_NULL. - */ - operator bool() const - { - return (bool)commPtr; - } - /** - * Returns the MPI group being used. - * @return - */ - MpiGroup Group() const; - - /** - * Abort - should try to bring down all tasks, but no guarantees - * @param errCode - */ - void Abort(int errCode) const; - - /** - * Duplicate the communicator - see MPI_COMM_DUP - * @return - */ - MpiCommunicator Duplicate() const; - - void Barrier() const; - MpiRequest Ibarrier() const; - - bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const; - - template - void Broadcast(T& val, const int root) const; - template - void Broadcast(std::vector& vals, const int root) const; - - template - MpiRequest Ibcast(T& val, const int root) const; - template - MpiRequest Ibcast(std::vector& vals, const int root) const; - - template - T AllReduce(const T& val, const MPI_Op& op) const; - template - std::vector AllReduce(const std::vector& vals, const MPI_Op& op) const; - - template - MpiRequest Iallreduce(const T& val, const MPI_Op& op, T& out) const; - - template - MpiRequest Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const; - - template - T Reduce(const T& val, const MPI_Op& op, const int root) const; - template - std::vector Reduce(const std::vector& vals, const MPI_Op& op, const int root) const; - - template - std::vector Gather(const T& val, const int root) const; - - template - std::vector GatherV(const std::vector senddata, const std::vector recvcounts, - const int root) const; - - template - std::vector AllGather(const T& val) const; - - template - std::vector AllToAll(const std::vector& vals) const; - - template - void Send(const std::vector& val, int dest, int tag=0) const; - template - void Send(const T& val, int dest, int tag=0) const; - template - void Send(const T* valPtr, int count, int dest, int tag) const; - - template - void Recv(std::vector& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; - template - void Recv(T& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; - template - void Recv(T* val, int count, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; - - template - MpiRequest Isend(const T& val, int dest, int tag=0) const; - template - MpiRequest Isend(const std::vector& vals, int dest, int tag=0) const; - template - MpiRequest Isend(const T* valPtr, int count, int dest, int tag=0) const; - - template - MpiRequest Issend(const T& val, int dest, int tag=0) const; - template - MpiRequest Issend(const std::vector& vals, int dest, int tag=0) const; - template - MpiRequest Issend(const T* valPtr, int count, int dest, int tag=0) const; - - template - MpiRequest Irecv(T& val, int source, int tag=0) const; - template - MpiRequest Irecv(std::vector& vals, int source, int tag=0) const; - template - MpiRequest Irecv(T* valPtr, int count, int source, int tag=0) const; - - protected: - /** - * Constructor to get data needed from an MPI communicator - * @param communicator - */ - MpiCommunicator(MPI_Comm communicator, bool willOwn); - - std::shared_ptr commPtr; - }; - - bool operator==(const MpiCommunicator& comm1, const MpiCommunicator& comm2); - bool operator!=(const MpiCommunicator& comm1, const MpiCommunicator& comm2); - - } -} - -#include "net/MpiCommunicator.hpp" - -#endif /* HEMELB_NET_MPICOMMUNICATOR_H */ diff --git a/Code/net/MpiCommunicator.hpp b/Code/net/MpiCommunicator.hpp deleted file mode 100644 index 64807ac10..000000000 --- a/Code/net/MpiCommunicator.hpp +++ /dev/null @@ -1,322 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. -#ifndef HEMELB_NET_MPICOMMUNICATOR_HPP -#define HEMELB_NET_MPICOMMUNICATOR_HPP - -#include "net/MpiDataType.h" -#include "net/MpiConstness.h" -#include "net/MpiRequest.h" - -namespace hemelb -{ - namespace net - { - template - void MpiCommunicator::Broadcast(T& val, const int root) const - { - HEMELB_MPI_CALL( - MPI_Bcast, - (&val, 1, MpiDataType(), root, *this) - ); - } - template - void MpiCommunicator::Broadcast(std::vector& vals, const int root) const - { - HEMELB_MPI_CALL( - MPI_Bcast, - (&vals[0], vals.size(), MpiDataType(), root, *this) - ); - } - - template - MpiRequest MpiCommunicator::Ibcast(T& val, const int root) const - { - MPI_Request req; - HEMELB_MPI_CALL( - MPI_Ibcast, - (&val, 1, MpiDataType(), root, *commPtr, &req) - ); - return MpiRequest(req); - } - template - MpiRequest MpiCommunicator::Ibcast(std::vector& vals, const int root) const - { - MPI_Request req; - HEMELB_MPI_CALL( - MPI_Ibcast, - (&vals[0], vals.size(), MpiDataType(), root, *commPtr, &req) - ); - return MpiRequest(req); - } - - template - T MpiCommunicator::AllReduce(const T& val, const MPI_Op& op) const - { - T ans; - HEMELB_MPI_CALL( - MPI_Allreduce, - (MpiConstCast(&val), &ans, 1, MpiDataType(), op, *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::AllReduce(const std::vector& vals, const MPI_Op& op) const - { - std::vector ans(vals.size()); - HEMELB_MPI_CALL( - MPI_Allreduce, - (MpiConstCast(&vals[0]), &ans[0], vals.size(), MpiDataType(), op, *this) - ); - return ans; - } - - template - MpiRequest MpiCommunicator::Iallreduce(const T& val, const MPI_Op& op, T& out) const - { - MPI_Request req; - HEMELB_MPI_CALL( - MPI_Iallreduce, - (&val, &out, 1, MpiDataType(), op, *commPtr, &req) - ); - return MpiRequest(req); - } - - template - MpiRequest MpiCommunicator::Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const - { - MPI_Request req; - HEMELB_MPI_CALL( - MPI_Ireduce, - (&val, &out, 1, MpiDataType(), op, root, *commPtr, &req) - ); - return MpiRequest(req); - } - - template - T MpiCommunicator::Reduce(const T& val, const MPI_Op& op, const int root) const - { - T ans; - HEMELB_MPI_CALL( - MPI_Reduce, - (MpiConstCast(&val), &ans, 1, MpiDataType(), op, root, *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::Reduce(const std::vector& vals, const MPI_Op& op, - const int root) const - { - std::vector ans; - T* recvbuf = NULL; - - if (Rank() == root) - { - // Standard says the address of receive buffer only matters at the root. - ans.resize(vals.size()); - recvbuf = &ans[0]; - } - - HEMELB_MPI_CALL( - MPI_Reduce, - (MpiConstCast(&vals[0]), recvbuf, vals.size(), MpiDataType(), op, root, *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::Gather(const T& val, const int root) const - { - std::vector ans; - T* recvbuf = NULL; - - if (Rank() == root) - { - // Standard says the address of receive buffer only matters at the root. - ans.resize(Size()); - recvbuf = &ans[0]; - } - HEMELB_MPI_CALL( - MPI_Gather, - (MpiConstCast(&val), 1, MpiDataType(), - recvbuf, 1, MpiDataType(), - root, *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::GatherV(const std::vector senddata, - const std::vector recvcounts, - const int root) const - { - const int np = Size(); - const int sendcount = senddata.size(); - std::vector displs; - std::vector ans; - T* recvbuf = nullptr; - if (Rank() == root) - { - // Compute the displacements from the counts - displs.resize(np); - int total = 0; - for(size_t i = 0; i < np; ++i) { - displs[i] = total; - total += recvcounts[i]; - } - // set up recv buffer - ans.resize(total); - recvbuf = ans.data(); - } - - HEMELB_MPI_CALL( - MPI_Gatherv, - (MpiConstCast(senddata.data()), sendcount, MpiDataType(), - recvbuf, recvcounts.data(), displs.data(), MpiDataType(), - root, *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::AllGather(const T& val) const - { - std::vector ans(Size()); - T* recvbuf = &ans[0]; - - HEMELB_MPI_CALL( - MPI_Allgather, - (MpiConstCast(&val), 1, MpiDataType(), - recvbuf, 1, MpiDataType(), - *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::AllToAll(const std::vector& vals) const - { - std::vector ans(vals.size()); - HEMELB_MPI_CALL( - MPI_Alltoall, - (MpiConstCast(&vals[0]), 1, MpiDataType(), - &ans[0], 1, MpiDataType(), - *this) - ); - return ans; - } - - // Send implementations - template - void MpiCommunicator::Send(const T* valPtr, int count, int dest, int tag) const - { - HEMELB_MPI_CALL( - MPI_Send, - (valPtr, count, MpiDataType(), dest, tag, *this) - ); - } - template - void MpiCommunicator::Send(const T& val, int dest, int tag) const - { - Send(&val, 1, dest, tag); - } - template - void MpiCommunicator::Send(const std::vector& vals, int dest, int tag) const - { - Send(&vals[0], vals.size(), dest, tag); - } - - // Recv implementations - template - void MpiCommunicator::Recv(T* valPtr, int count, int src, int tag, MPI_Status* stat) const - { - HEMELB_MPI_CALL( - MPI_Recv, - (valPtr, count, MpiDataType(), src, tag, *this, stat) - ); - } - template - void MpiCommunicator::Recv(T& val, int src, int tag, MPI_Status* stat) const - { - Recv(&val, 1, src, tag); - } - template - void MpiCommunicator::Recv(std::vector& vals, int src, int tag, MPI_Status* stat) const - { - Recv(&vals[0], vals.size(), src, tag); - } - - // Isend implementations - template - MpiRequest MpiCommunicator::Isend(const T* valPtr, int count, int dest, int tag) const - { - MPI_Request req; - HEMELB_MPI_CALL( - MPI_Isend, - (const_cast(valPtr), count, MpiDataType(), dest, tag, *this, &req) - ); - return MpiRequest(req); - } - template - MpiRequest MpiCommunicator::Isend(const T& val, int dest, int tag) const - { - return Isend(&val, 1, dest, tag); - } - template - MpiRequest MpiCommunicator::Isend(const std::vector& vals, int dest, int tag) const - { - return Isend(&vals[0], vals.size(), dest, tag); - } - - // Issend implementations - template - MpiRequest MpiCommunicator::Issend(const T* valPtr, int count, int dest, int tag) const - { - MPI_Request req; - HEMELB_MPI_CALL( - MPI_Issend, - (const_cast(valPtr), count, MpiDataType(), dest, tag, *this, &req) - ); - return MpiRequest(req); - } - template - MpiRequest MpiCommunicator::Issend(const T& val, int dest, int tag) const - { - return Issend(&val, 1, dest, tag); - } - template - MpiRequest MpiCommunicator::Issend(const std::vector& vals, int dest, int tag) const - { - return Issend(&vals[0], vals.size(), dest, tag); - } - - // Irecv implementations - template - MpiRequest MpiCommunicator::Irecv(T* valPtr, int count, int source, int tag) const - { - MPI_Request req; - HEMELB_MPI_CALL( - MPI_Irecv, - (valPtr, count, MpiDataType(), source, tag, *this, &req) - ); - return MpiRequest(req); - } - template - MpiRequest MpiCommunicator::Irecv(T& val, int source, int tag) const - { - return Irecv(&val, 1, source, tag); - } - template - MpiRequest MpiCommunicator::Irecv(std::vector& vals, int source, int tag) const - { - return Irecv(&vals[0], vals.size(), source, tag); - } - - } -} - -#endif // HEMELB_NET_MPICOMMUNICATOR_HPP diff --git a/Code/net/MpiConstness.h.in b/Code/net/MpiConstness.h.in deleted file mode 100644 index ff1a237a1..000000000 --- a/Code/net/MpiConstness.h.in +++ /dev/null @@ -1,30 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MPICONSTNESS_H_IN -#define HEMELB_NET_MPICONSTNESS_H_IN -namespace hemelb -{ - namespace net - { -#cmakedefine HAVE_CONSTCORRECTMPI - -#ifdef HAVE_CONSTCORRECTMPI - template - const T* MpiConstCast(const T* ptr) - { - return ptr; - } -#else - template - T* MpiConstCast(const T* ptr) - { - return const_cast(ptr); - } -#endif - } -} -#endif // HEMELB_NET_MPICONSTNESS_H_IN \ No newline at end of file From 28ae19df617bbf97cb57c0a2494f39aef83a62c2 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 18 Oct 2016 17:14:50 +0100 Subject: [PATCH 55/99] move extraction to new style comms --- Code/extraction/LocalPropertyOutput.cc | 38 +++++++++++++------------- Code/extraction/LocalPropertyOutput.h | 16 +++++------ Code/extraction/PropertyActor.cc | 2 +- Code/extraction/PropertyActor.h | 2 +- Code/extraction/PropertyWriter.cc | 2 +- Code/extraction/PropertyWriter.h | 4 +-- 6 files changed, 31 insertions(+), 33 deletions(-) diff --git a/Code/extraction/LocalPropertyOutput.cc b/Code/extraction/LocalPropertyOutput.cc index c5170c509..5396c1be6 100644 --- a/Code/extraction/LocalPropertyOutput.cc +++ b/Code/extraction/LocalPropertyOutput.cc @@ -9,7 +9,7 @@ #include "io/formats/formats.h" #include "io/formats/extraction.h" #include "io/writers/xdr/XdrMemWriter.h" -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" #include "constants.h" namespace hemelb @@ -18,13 +18,13 @@ namespace hemelb { LocalPropertyOutput::LocalPropertyOutput(IterableDataSource& dataSource, const PropertyOutputFile* outputSpec, - const net::IOCommunicator& ioComms) : + const comm::Communicator* ioComms) : comms(ioComms), dataSource(dataSource), outputSpec(outputSpec) { // Open the file as write-only, create it if it doesn't exist, don't create if the file // already exists. - outputFile = net::MpiFile::Open(comms, outputSpec->filename, - MPI_MODE_WRONLY | MPI_MODE_CREATE | MPI_MODE_EXCL); + outputFile = comms->OpenFile(outputSpec->filename, + MPI_MODE_WRONLY | MPI_MODE_CREATE | MPI_MODE_EXCL); // Count sites on this task uint64_t siteCount = 0; dataSource.Reset(); @@ -53,7 +53,7 @@ namespace hemelb writeLength *= siteCount; // The IO proc also writes the iteration number - if (comms.OnIORank()) + if (comms->OnIORank()) { writeLength += 8; } @@ -61,17 +61,17 @@ namespace hemelb //! @TODO: These two MPI calls can be replaced with one // Everyone needs to know the total length written during one iteration. - allCoresWriteLength = comms.AllReduce(writeLength, MPI_SUM); + allCoresWriteLength = comms->AllReduce(writeLength, MPI_SUM); // Only the root process needs to know the total number of sites written // Note this has a garbage value on other procs. - uint64_t allSiteCount = comms.Reduce(siteCount, MPI_SUM, - comms.GetIORank()); + uint64_t allSiteCount = comms->Reduce(siteCount, MPI_SUM, + comms->GetIORank()); unsigned totalHeaderLength = 0; // Write the header information on the IO proc. - if (comms.OnIORank()) + if (comms->OnIORank()) { // Compute the length of the field header unsigned fieldHeaderLength = 0; @@ -125,29 +125,29 @@ namespace hemelb } // Write from the buffer - outputFile.WriteAt(0, headerBuffer); + outputFile->WriteAt(0, headerBuffer); } // Calculate where each core should start writing - if (comms.OnIORank()) + if (comms->OnIORank()) { // For core 0 this is easy: it passes the value for core 1 to the core. localDataOffsetIntoFile = totalHeaderLength; - if (comms.Size() > 1) + if (comms->Size() > 1) { - comms.Send(localDataOffsetIntoFile+writeLength, 1, 1); + comms->Send(localDataOffsetIntoFile+writeLength, 1, 1); } } else { // Receive the writing start position from the previous core. - comms.Recv(localDataOffsetIntoFile, comms.Rank()-1, 1); + comms->Recv(localDataOffsetIntoFile, comms->Rank()-1, 1); // Send the next core its start position. - if (comms.Rank() != (comms.Size() - 1)) + if (comms->Rank() != (comms->Size() - 1)) { - comms.Send(localDataOffsetIntoFile+writeLength, comms.Rank() + 1, 1); + comms->Send(localDataOffsetIntoFile+writeLength, comms->Rank() + 1, 1); } } @@ -188,7 +188,7 @@ namespace hemelb io::writers::xdr::XdrMemWriter xdrWriter(&buffer[0], buffer.size()); // Firstly, the IO proc must write the iteration number. - if (comms.OnIORank()) + if (comms->OnIORank()) { xdrWriter << (uint64_t) timestepNumber; } @@ -252,7 +252,7 @@ namespace hemelb break; case OutputField::MpiRank: xdrWriter - << static_cast (comms.Rank()); + << static_cast (comms->Rank()); break; default: // This should never trip. It only occurs when a new OutputField field is added and no @@ -264,7 +264,7 @@ namespace hemelb } // Actually do the MPI writing. - outputFile.WriteAt(localDataOffsetIntoFile, buffer); + outputFile->WriteAt(localDataOffsetIntoFile, buffer); // Set the offset to the right place for writing on the next iteration. localDataOffsetIntoFile += allCoresWriteLength; diff --git a/Code/extraction/LocalPropertyOutput.h b/Code/extraction/LocalPropertyOutput.h index 60494204c..f2330aedb 100644 --- a/Code/extraction/LocalPropertyOutput.h +++ b/Code/extraction/LocalPropertyOutput.h @@ -9,15 +9,12 @@ #include "extraction/IterableDataSource.h" #include "extraction/PropertyOutputFile.h" -#include "net/mpi.h" -#include "net/MpiFile.h" + +#include "comm/Communicator.h" +#include "comm/MpiFile.h" namespace hemelb { - namespace net - { - class IOCommunicator; - } namespace extraction { /** @@ -32,7 +29,8 @@ namespace hemelb * @param offset * @return */ - LocalPropertyOutput(IterableDataSource& dataSource, const PropertyOutputFile* outputSpec, const net::IOCommunicator& ioComms); + LocalPropertyOutput(IterableDataSource& dataSource, const PropertyOutputFile* outputSpec, + const comm::Communicator* ioComms); /** * Tidies up the LocalPropertyOutput (close files etc). @@ -72,11 +70,11 @@ namespace hemelb */ double GetOffset(OutputField::FieldType field) const; - const net::IOCommunicator& comms; + const comm::Communicator* comms; /** * The MPI file to write into. */ - net::MpiFile outputFile; + comm::MpiFile* outputFile; /** * The data source to use for file output. diff --git a/Code/extraction/PropertyActor.cc b/Code/extraction/PropertyActor.cc index b4a9caa0b..43d047018 100644 --- a/Code/extraction/PropertyActor.cc +++ b/Code/extraction/PropertyActor.cc @@ -14,7 +14,7 @@ namespace hemelb const std::vector& propertyOutputs, IterableDataSource& dataSource, reporting::Timers& timers, - const net::IOCommunicator& ioComms) : + const comm::Communicator* ioComms) : simulationState(simulationState), timers(timers) { propertyWriter = new PropertyWriter(dataSource, propertyOutputs, ioComms); diff --git a/Code/extraction/PropertyActor.h b/Code/extraction/PropertyActor.h index 15bccb913..bb82a69d7 100644 --- a/Code/extraction/PropertyActor.h +++ b/Code/extraction/PropertyActor.h @@ -31,7 +31,7 @@ namespace hemelb const std::vector& propertyOutputs, IterableDataSource& dataSource, reporting::Timers& timers, - const net::IOCommunicator& ioComms); + const comm::Communicator* ioComms); ~PropertyActor(); diff --git a/Code/extraction/PropertyWriter.cc b/Code/extraction/PropertyWriter.cc index db9e9a539..7c016f779 100644 --- a/Code/extraction/PropertyWriter.cc +++ b/Code/extraction/PropertyWriter.cc @@ -12,7 +12,7 @@ namespace hemelb { PropertyWriter::PropertyWriter(IterableDataSource& dataSource, const std::vector& propertyOutputs, - const net::IOCommunicator& ioComms) + const comm::Communicator* ioComms) { for (unsigned outputNumber = 0; outputNumber < propertyOutputs.size(); ++outputNumber) { diff --git a/Code/extraction/PropertyWriter.h b/Code/extraction/PropertyWriter.h index ffb43c617..0a4a1f9ed 100644 --- a/Code/extraction/PropertyWriter.h +++ b/Code/extraction/PropertyWriter.h @@ -9,7 +9,7 @@ #include "extraction/LocalPropertyOutput.h" #include "extraction/PropertyOutputFile.h" -#include "net/mpi.h" +#include "comm/Communicator.h" namespace hemelb { @@ -23,7 +23,7 @@ namespace hemelb * @param propertyOutputs * @return */ - PropertyWriter(IterableDataSource& dataSource, const std::vector& propertyOutputs, const net::IOCommunicator& ioComms); + PropertyWriter(IterableDataSource& dataSource, const std::vector& propertyOutputs, const comm::Communicator* ioComms); /** * Destructor; deallocates memory used to store property info. From 9250bda23200b8acdd9e0f3990061188fec30406 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 18 Oct 2016 17:17:38 +0100 Subject: [PATCH 56/99] move log to new style comms --- Code/log/Logger.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Code/log/Logger.cc b/Code/log/Logger.cc index 21744132f..da105c0d5 100644 --- a/Code/log/Logger.cc +++ b/Code/log/Logger.cc @@ -13,7 +13,7 @@ #include #include "util/utilityFunctions.h" -#include "net/mpi.h" +#include "comm/MpiEnvironment.h" #include "log/Logger.h" namespace hemelb @@ -31,9 +31,9 @@ namespace hemelb if (thisRank < 0) { // Check that MPI is ready - if (net::MpiEnvironment::Initialized()) + if (comm::MpiEnvironment::Initialized()) { - thisRank = net::MpiCommunicator::World().Rank(); + thisRank = comm::MpiEnvironment::World()->Rank(); } startTime = util::myClock(); } From 12f8ded7a8f880ee12a2ee20c3255e36539d68d9 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 18 Oct 2016 17:20:40 +0100 Subject: [PATCH 57/99] move reporting to new style comms --- Code/reporting/Policies.h | 86 +++++++++++++++++++-------------------- Code/reporting/Timers.cc | 2 +- Code/reporting/Timers.h | 17 ++++---- Code/reporting/Timers.hpp | 40 ++++++------------ 4 files changed, 65 insertions(+), 80 deletions(-) diff --git a/Code/reporting/Policies.h b/Code/reporting/Policies.h index b9fbdc37d..0858303db 100644 --- a/Code/reporting/Policies.h +++ b/Code/reporting/Policies.h @@ -15,8 +15,8 @@ */ #include -#include "net/mpi.h" -#include "net/IOCommunicator.h" +//#include "net/mpi.h" +//#include "net/IOCommunicator.h" #include "util/utilityFunctions.h" namespace hemelb @@ -24,49 +24,49 @@ namespace hemelb namespace reporting { - /** - * Abstraction of interaction with MPI system. - * Mocked by hemelb::unittests::reporting::MPICommsMock - */ - class MPICommsPolicy - { - public: - /** - * Stores a pointer to the MPI Network topology singleton. - */ - MPICommsPolicy(const net::IOCommunicator& inst_) : - instance(inst_) - { - } - protected: - /** - * Wrap the MPI reduce call. - * @param sendbuf Pointer to start of send buffer - * @param recvbuf Pointer to start of receive buffer - * @param count Number of data to send/receive - * @param datatype MPI Datatype - * @param op Operator to reduce under - * @param root Target processor where the reduction is reduced to. - * @param comm MPI Communicator to reduce over - * @return - */ - int Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root) - { - return MPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, instance); - } + // /** + // * Abstraction of interaction with MPI system. + // * Mocked by hemelb::unittests::reporting::MPICommsMock + // */ + // class MPICommsPolicy + // { + // public: + // /** + // * Stores a pointer to the MPI Network topology singleton. + // */ + // MPICommsPolicy(const comm::Communicator* inst_) : + // instance(inst_) + // { + // } + // protected: + // /** + // * Wrap the MPI reduce call. + // * @param sendbuf Pointer to start of send buffer + // * @param recvbuf Pointer to start of receive buffer + // * @param count Number of data to send/receive + // * @param datatype MPI Datatype + // * @param op Operator to reduce under + // * @param root Target processor where the reduction is reduced to. + // * @param comm MPI Communicator to reduce over + // * @return + // */ + // int Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root) + // { + // return MPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, instance); + // } - /** - * Total number of MPI nodes in the communicator. - * @return Total number of processors, some of which may be shared on a single machine. - */ - proc_t GetProcessorCount() const - { - return instance.Size(); - } + // /** + // * Total number of MPI nodes in the communicator. + // * @return Total number of processors, some of which may be shared on a single machine. + // */ + // proc_t GetProcessorCount() const + // { + // return instance->Size(); + // } - private: - const net::IOCommunicator& instance; //! Reference to the singleton instance of the MPI topology - }; + // private: + // const comm::Communicator* instance; //! Reference to the singleton instance of the MPI topology + // }; /** * A way to get the time. diff --git a/Code/reporting/Timers.cc b/Code/reporting/Timers.cc index 267ccf07b..916679040 100644 --- a/Code/reporting/Timers.cc +++ b/Code/reporting/Timers.cc @@ -9,6 +9,6 @@ namespace hemelb { namespace reporting { - template class TimersBase; // explicit instantiate + template class TimersBase; // explicit instantiate } } diff --git a/Code/reporting/Timers.h b/Code/reporting/Timers.h index 2def88523..a39936437 100644 --- a/Code/reporting/Timers.h +++ b/Code/reporting/Timers.h @@ -11,6 +11,8 @@ #include "reporting/Reportable.h" #include "util/utilityFunctions.h" #include "reporting/Policies.h" +#include "comm/Communicator.h" + namespace hemelb { namespace reporting @@ -69,10 +71,9 @@ namespace hemelb /** * Class which manages a set of timers timing aspects of a HemeLB run * @tparam ClockPolicy How to get the current time - * @tparam CommsPolicy How to share information between processes */ - template - class TimersBase : public CommsPolicy, public Reportable + template + class TimersBase : public Reportable { public: typedef TimerBase Timer; @@ -126,8 +127,7 @@ namespace hemelb */ static const std::string timerNames[TimersBase::numberOfTimers]; - TimersBase(const net::IOCommunicator& comms) : - CommsPolicy(comms), + TimersBase(const comm::Communicator* commPtr) : comms(commPtr), timers(numberOfTimers), maxes(numberOfTimers), mins(numberOfTimers), means(numberOfTimers) { } @@ -202,16 +202,17 @@ namespace hemelb void Report(ctemplate::TemplateDictionary& dictionary); private: + const comm::Communicator* comms; std::vector timers; //! The set of timers std::vector maxes; //! Max across processes std::vector mins; //! Min across processes std::vector means; //! Average across processes }; typedef TimerBase Timer; - typedef TimersBase Timers; + typedef TimersBase Timers; - template - const std::string TimersBase::timerNames[TimersBase::numberOfTimers] = + template + const std::string TimersBase::timerNames[TimersBase::numberOfTimers] = { "Total", "Seed Decomposition", "Domain Decomposition", "File Read", "Re Read", "Unzip", "Moves", "Parmetis", "Lattice Data initialisation", "Lattice Boltzmann", "LB calc only", "Monitoring", "MPI Send", diff --git a/Code/reporting/Timers.hpp b/Code/reporting/Timers.hpp index 18f574867..e22cb04d5 100644 --- a/Code/reporting/Timers.hpp +++ b/Code/reporting/Timers.hpp @@ -13,43 +13,27 @@ namespace hemelb { namespace reporting { - template - void TimersBase::Reduce() + template + void TimersBase::Reduce() { - double timings[numberOfTimers]; + std::vector timings(numberOfTimers); for (unsigned int ii = 0; ii < numberOfTimers; ii++) { timings[ii] = timers[ii].Get(); } - CommsPolicy::Reduce(timings, - &maxes[0], - numberOfTimers, - net::MpiDataType(), - MPI_MAX, - 0); - CommsPolicy::Reduce(timings, - &means[0], - numberOfTimers, - net::MpiDataType(), - MPI_SUM, - 0); - CommsPolicy::Reduce(timings, - &mins[0], - numberOfTimers, - net::MpiDataType(), - MPI_MIN, - 0); - for (unsigned int ii = 0; ii < numberOfTimers; ii++) - { - means[ii] /= double(CommsPolicy::GetProcessorCount()); - } + maxes = comms->Reduce(timings, MPI_MAX, 0); + means = comms->Reduce(timings, MPI_SUM, 0); + mins = comms->Reduce(timings, MPI_MIN, 0); + const int np = comms->Size(); + std::transform(means.begin(), means.end(), means.begin(), + [&](double x) { return x / np; }); } - template - void TimersBase::Report(ctemplate::TemplateDictionary& dictionary) + template + void TimersBase::Report(ctemplate::TemplateDictionary& dictionary) { - dictionary.SetIntValue("THREADS", CommsPolicy::GetProcessorCount()); + dictionary.SetIntValue("THREADS", comms->Size()); for (unsigned int ii = 0; ii < numberOfTimers; ii++) { From 1bf551563803cca7a9df24ae2c87db24ade3e00d Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 18 Oct 2016 17:21:42 +0100 Subject: [PATCH 58/99] move util to new style comms --- Code/util/Vector3D.h | 4 ++-- Code/util/Vector3DHemeLb.cc | 8 +++++--- Code/util/utilityFunctions.cc | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Code/util/Vector3D.h b/Code/util/Vector3D.h index a21232fd5..ed2cbcbdb 100644 --- a/Code/util/Vector3D.h +++ b/Code/util/Vector3D.h @@ -721,11 +721,11 @@ namespace hemelb } #ifdef HEMELB_CODE -#include "net/MpiDataType.h" +#include "comm/MpiDataType.h" namespace hemelb { - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits >::RegisterMpiDataType(); diff --git a/Code/util/Vector3DHemeLb.cc b/Code/util/Vector3DHemeLb.cc index 9f2e36df4..eb2a3ce70 100644 --- a/Code/util/Vector3DHemeLb.cc +++ b/Code/util/Vector3DHemeLb.cc @@ -4,13 +4,15 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. +#include "units.h" #include "util/Vector3D.h" -#include "net/mpi.h" +#include "comm/MpiError.h" +#include "comm/MpiDataType.h" #include "Exception.h" namespace hemelb { - namespace net + namespace comm { template MPI_Datatype GenerateTypeForVector() @@ -18,7 +20,7 @@ namespace hemelb const int typeCount = 1; int blocklengths[typeCount] = { 3 }; - MPI_Datatype types[typeCount] = { net::MpiDataType() }; + MPI_Datatype types[typeCount] = { comm::MpiDataType() }; MPI_Aint displacements[typeCount] = { 0 }; diff --git a/Code/util/utilityFunctions.cc b/Code/util/utilityFunctions.cc index 16ecb8b76..3b5dfb6b6 100644 --- a/Code/util/utilityFunctions.cc +++ b/Code/util/utilityFunctions.cc @@ -4,7 +4,7 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/mpi.h" +#include namespace hemelb { namespace util From 8338e77f0685386f7679e7e825ee8556b20581cb Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 19 Oct 2016 10:32:02 +0100 Subject: [PATCH 59/99] move colloids to new style comms --- Code/colloids/ColloidController.cc | 12 ++++++------ Code/colloids/ColloidController.h | 4 ++-- Code/colloids/Particle.h | 4 ++-- Code/colloids/ParticleMpiDatatypes.cc | 6 +++--- Code/colloids/ParticleSet.cc | 24 ++++++++++++------------ Code/colloids/ParticleSet.h | 9 ++++----- Code/colloids/PersistedParticle.h | 4 ++-- 7 files changed, 31 insertions(+), 32 deletions(-) diff --git a/Code/colloids/ColloidController.cc b/Code/colloids/ColloidController.cc index a2825de7e..ac116a441 100644 --- a/Code/colloids/ColloidController.cc +++ b/Code/colloids/ColloidController.cc @@ -34,7 +34,7 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, const std::string& outputPath, - const net::IOCommunicator& ioComms_, + const comm::Communicator* ioComms_, reporting::Timers& timers) : ioComms(ioComms_), simulationState(simulationState), timers(timers) { @@ -50,10 +50,10 @@ namespace hemelb // determine information about neighbour sites and processors for all local fluid sites InitialiseNeighbourList(latDatLBM, gmyResult, neighbourhood); - bool allGood = ioComms.OnIORank() || (neighbourProcessors.size() > 0); + bool allGood = ioComms->OnIORank() || (neighbourProcessors.size() > 0); log::Logger::Log( "[Rank %i]: ColloidController - neighbourhood %i, neighbours %i, allGood %i\n", - ioComms.Rank(), neighbourhood.size(), neighbourProcessors.size(), allGood); + ioComms->Rank(), neighbourhood.size(), neighbourProcessors.size(), allGood); io::xml::Element particlesElem = xml.GetRoot().GetChildOrThrow("colloids").GetChildOrThrow("particles"); particleSet = new ParticleSet(latDatLBM, particlesElem, propertyCache, @@ -108,7 +108,7 @@ namespace hemelb // if site is local site_t siteId = siteTraverser.GetCurrentIndex(); - if (gmyResult.Blocks[blockId].Sites[siteId].targetProcessor != this->ioComms.Rank()) + if (gmyResult.Blocks[blockId].Sites[siteId].targetProcessor != this->ioComms->Rank()) { log::Logger::Log( "ColloidController: site with id %i and coords (%i,%i,%i) has proc %i (non-local).\n", @@ -143,7 +143,7 @@ namespace hemelb &neighbourBlockId, &neighbourSiteId, &neighbourRank); // if neighbour is remote - if (!isValid || neighbourRank == this->ioComms.Rank()) + if (!isValid || neighbourRank == this->ioComms->Rank()) continue; // if new neighbourRank @@ -159,7 +159,7 @@ namespace hemelb // debug message so this neighbour list can be compared to the LatticeData one log::Logger::Log( "ColloidController: added %i as neighbour for %i because site %i in block %i is neighbour to site %i in block %i in direction (%i,%i,%i)\n", - (int)neighbourRank, (int)(this->ioComms.Rank()), + (int)neighbourRank, (int)(this->ioComms->Rank()), (int)neighbourSiteId, (int)neighbourBlockId, (int)siteId, (int)blockId, (*itDirectionVector).x, (*itDirectionVector).y, (*itDirectionVector).z); diff --git a/Code/colloids/ColloidController.h b/Code/colloids/ColloidController.h index 1cd3b9d2a..64a235db5 100644 --- a/Code/colloids/ColloidController.h +++ b/Code/colloids/ColloidController.h @@ -34,7 +34,7 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, const std::string& outputPath, - const net::IOCommunicator& ioComms_, + const comm::Communicator* ioComms_, reporting::Timers& timers); /** destructor - releases resources allocated by this class */ @@ -50,7 +50,7 @@ namespace hemelb private: /** Main code communicator */ - const net::IOCommunicator& ioComms; + const comm::Communicator* ioComms; const lb::SimulationState& simulationState; diff --git a/Code/colloids/Particle.h b/Code/colloids/Particle.h index db460d781..b58eb72da 100644 --- a/Code/colloids/Particle.h +++ b/Code/colloids/Particle.h @@ -7,7 +7,7 @@ #ifndef HEMELB_COLLOIDS_PARTICLE_H #define HEMELB_COLLOIDS_PARTICLE_H -#include "net/mpi.h" +#include "comm/MpiDataType.h" #include "colloids/PersistedParticle.h" #include "geometry/LatticeData.h" #include "io/xml/XmlAbstractionLayer.h" @@ -195,7 +195,7 @@ namespace hemelb friend struct ParticleSorter; }; } - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); diff --git a/Code/colloids/ParticleMpiDatatypes.cc b/Code/colloids/ParticleMpiDatatypes.cc index a3610a00a..02f08af2d 100644 --- a/Code/colloids/ParticleMpiDatatypes.cc +++ b/Code/colloids/ParticleMpiDatatypes.cc @@ -9,7 +9,7 @@ namespace hemelb { - namespace net + namespace comm { // boiler-plate template specialisation for colloids Particle object template<> @@ -138,7 +138,7 @@ namespace hemelb MPI_DOUBLE, MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG, - net::MpiDataType > () }; + comm::MpiDataType > () }; // create a first draft of the MPI datatype for a Particle // the lower bound and displacements of fields are correct @@ -205,7 +205,7 @@ namespace hemelb displacementOfEachFieldBlock[1] -= baseAddress; // the built-in MPI datatype of each field must match the C++ type - MPI_Datatype datatypeOfEachFieldBlock[] = { MPI_UNSIGNED_LONG, net::MpiDataType > () }; // create a first draft of the MPI datatype for a Particle diff --git a/Code/colloids/ParticleSet.cc b/Code/colloids/ParticleSet.cc index 35cf8b2bf..f0709f403 100644 --- a/Code/colloids/ParticleSet.cc +++ b/Code/colloids/ParticleSet.cc @@ -43,16 +43,16 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, std::vector& neighbourProcessors, - const net::IOCommunicator& ioComms_, + const comm::Communicator* ioComms_, const std::string& outputPath) : - ioComms(ioComms_), localRank(ioComms.Rank()), latDatLBM(latDatLBM), propertyCache(propertyCache), path(outputPath), net(ioComms) + ioComms(ioComms_), localRank(ioComms->Rank()), latDatLBM(latDatLBM), propertyCache(propertyCache), path(outputPath), net(ioComms) { /** * Open the file, unless it already exists, for writing only, creating it if it doesn't exist. */ - file = net::MpiFile::Open(ioComms, path, MPI_MODE_EXCL | MPI_MODE_WRONLY | MPI_MODE_CREATE); + file = ioComms->OpenFile(path, MPI_MODE_EXCL | MPI_MODE_WRONLY | MPI_MODE_CREATE); - if (ioComms.OnIORank()) + if (ioComms->OnIORank()) { // Write out the header information from core 0 buffer.resize(io::formats::colloids::MagicLength); @@ -60,10 +60,10 @@ namespace hemelb writer << (uint32_t) io::formats::HemeLbMagicNumber; writer << (uint32_t) io::formats::colloids::MagicNumber; writer << (uint32_t) io::formats::colloids::VersionNumber; - file.Write(buffer); + file->Write(buffer); } - HEMELB_MPI_CALL(MPI_File_seek_shared, (file, 0, MPI_SEEK_END)); + HEMELB_MPI_CALL(MPI_File_seek_shared, (*file, 0, MPI_SEEK_END)); // add an element into scanMap for each neighbour rank with zero for both counts // sorting the list of neighbours allows the position in the map to be predicted @@ -132,35 +132,35 @@ namespace hemelb // Find how far we currently are into the file. MPI_Offset positionBeforeWriting; - HEMELB_MPI_CALL(MPI_File_get_position_shared, (file, &positionBeforeWriting)); + HEMELB_MPI_CALL(MPI_File_get_position_shared, (*file, &positionBeforeWriting)); log::Logger::Log("from offsetEOF: %i\n", positionBeforeWriting); // Go past the header (which we'll write at the end) unsigned int sizeOfHeader = io::formats::colloids::HeaderLength; - HEMELB_MPI_CALL(MPI_File_seek_shared, (file, sizeOfHeader, MPI_SEEK_END)); + HEMELB_MPI_CALL(MPI_File_seek_shared, (*file, sizeOfHeader, MPI_SEEK_END)); // Collective write: the effect is as though all writes are done // in serialised order, i.e. as if rank 0 writes first, followed // by rank 1, and so on, until all ranks have written their data - HEMELB_MPI_CALL(MPI_File_write_ordered, (file, &buffer.front(), count, MPI_CHAR, MPI_STATUS_IGNORE)); + HEMELB_MPI_CALL(MPI_File_write_ordered, (*file, &buffer.front(), count, MPI_CHAR, MPI_STATUS_IGNORE)); // the collective ordered write modifies the shared file pointer // it should point to the byte following the highest rank's data // (should be true for all ranks but) we only need it for rank 0 MPI_Offset positionAferWriting; - HEMELB_MPI_CALL(MPI_File_get_position_shared, (file, &positionAferWriting)); + HEMELB_MPI_CALL(MPI_File_get_position_shared, (*file, &positionAferWriting)); log::Logger::Log("new offsetEOF: %i\n", positionBeforeWriting); // Now write the header section, only on rank 0. - if (ioComms.OnIORank()) + if (ioComms->OnIORank()) { writer << (uint32_t) io::formats::colloids::HeaderLength; writer << (uint32_t) io::formats::colloids::RecordLength; writer << (uint64_t) (positionAferWriting - positionBeforeWriting - io::formats::colloids::HeaderLength); writer << (uint64_t) timestep; - HEMELB_MPI_CALL(MPI_File_write_at, (file, positionBeforeWriting, &buffer[count], sizeOfHeader, MPI_CHAR, MPI_STATUS_IGNORE)); + HEMELB_MPI_CALL(MPI_File_write_at, (*file, positionBeforeWriting, &buffer[count], sizeOfHeader, MPI_CHAR, MPI_STATUS_IGNORE)); } for (scanMapConstIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) diff --git a/Code/colloids/ParticleSet.h b/Code/colloids/ParticleSet.h index 4eb1fb512..09b382bc3 100644 --- a/Code/colloids/ParticleSet.h +++ b/Code/colloids/ParticleSet.h @@ -11,9 +11,8 @@ #include "geometry/LatticeData.h" #include "io/xml/XmlAbstractionLayer.h" #include "lb/MacroscopicPropertyCache.h" -#include "net/mpi.h" #include "colloids/Particle.h" -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" #include "units.h" namespace hemelb @@ -30,7 +29,7 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, std::vector& neighbourProcessors, - const net::IOCommunicator& ioComms_, + const comm::Communicator* ioComms_, const std::string& outputPath); /** destructor - de-allocates all Particle objects created by this Set */ @@ -61,7 +60,7 @@ namespace hemelb const void OutputInformation(const LatticeTimeStep timestep); private: - const net::IOCommunicator& ioComms; + const comm::Communicator* ioComms; /** cached copy of local rank (obtained from topology) */ const proc_t localRank; @@ -107,7 +106,7 @@ namespace hemelb /** * MPI File handle to write with */ - net::MpiFile file; + comm::MpiFile* file; }; } } diff --git a/Code/colloids/PersistedParticle.h b/Code/colloids/PersistedParticle.h index 2e7303e8b..cb12c3f9d 100644 --- a/Code/colloids/PersistedParticle.h +++ b/Code/colloids/PersistedParticle.h @@ -9,7 +9,7 @@ #include "io/xml/XmlAbstractionLayer.h" #include "units.h" -#include "net/MpiDataType.h" +#include "comm/MpiDataType.h" namespace hemelb { @@ -61,7 +61,7 @@ namespace hemelb LatticePosition globalPosition; }; } - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); From 03b1bef96d27edbafffdb7193a7879d8ecc0b129 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 19 Oct 2016 11:28:56 +0100 Subject: [PATCH 60/99] most of the actual new comms stuff --- Code/comm/CMakeLists.txt | 10 ++ Code/comm/Communicator.h | 191 +++++++++++++++++++++ Code/comm/Communicator.hpp | 248 +++++++++++++++++++++++++++ Code/comm/Group.h | 54 ++++++ Code/comm/MpiCommunicator.cc | 277 +++++++++++++++++++++++++++++++ Code/comm/MpiCommunicator.h | 110 ++++++++++++ Code/comm/MpiDataType.cc | 4 +- Code/comm/MpiDataType.h | 14 +- Code/comm/MpiEnvironment.cc | 15 +- Code/comm/MpiEnvironment.h | 12 +- Code/comm/MpiError.cc | 4 +- Code/comm/MpiError.h | 10 +- Code/comm/MpiFile.cc | 24 +-- Code/comm/MpiFile.h | 31 ++-- Code/comm/MpiFile.hpp | 16 +- Code/comm/MpiGroup.cc | 18 +- Code/comm/MpiGroup.h | 23 ++- Code/{net => comm}/MpiRequest.cc | 54 +----- Code/comm/MpiRequest.h | 18 +- Code/comm/Request.h | 55 ++++++ 20 files changed, 1035 insertions(+), 153 deletions(-) create mode 100644 Code/comm/CMakeLists.txt create mode 100644 Code/comm/Communicator.h create mode 100644 Code/comm/Communicator.hpp create mode 100644 Code/comm/Group.h create mode 100644 Code/comm/MpiCommunicator.cc create mode 100644 Code/comm/MpiCommunicator.h rename Code/{net => comm}/MpiRequest.cc (61%) create mode 100644 Code/comm/Request.h diff --git a/Code/comm/CMakeLists.txt b/Code/comm/CMakeLists.txt new file mode 100644 index 000000000..42f997e88 --- /dev/null +++ b/Code/comm/CMakeLists.txt @@ -0,0 +1,10 @@ + +# This file is part of HemeLB and is Copyright (C) +# the HemeLB team and/or their institutions, as detailed in the +# file AUTHORS. This software is provided under the terms of the +# license in the file LICENSE. +add_library(hemelb_comm + MpiDataType.cc MpiEnvironment.cc MpiError.cc + MpiCommunicator.cc MpiGroup.cc MpiFile.cc + MpiRequest.cc +) diff --git a/Code/comm/Communicator.h b/Code/comm/Communicator.h new file mode 100644 index 000000000..efc89566e --- /dev/null +++ b/Code/comm/Communicator.h @@ -0,0 +1,191 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_COMM_COMMUNICATOR_H +#define HEMELB_COMM_COMMUNICATOR_H + +#include +#include +#include + +namespace hemelb +{ + namespace comm + { + class Group; + class Request; + class MpiFile; + + // Base class for communicators (MPI, null, and mock) + class Communicator + { + public: + /** + * Class has virtual methods so should have virtual d'tor. + */ + virtual ~Communicator(); + + /** + * Returns the local rank on the communicator + * @return + */ + virtual int Rank() const = 0; + + inline bool OnIORank() const + { + return Rank() == GetIORank(); + } + + inline int GetIORank() const + { + return 0; + } + + /** + * Returns the size of the communicator (i.e. total number of procs involved). + * @return + */ + virtual int Size() const = 0; + + /** + * Abort - should try to bring down all tasks, but no guarantees + * @param errCode + */ + virtual void Abort(int errCode) const = 0; + + /** + * Duplicate the communicator - see MPI_COMM_DUP + * @return + */ + virtual Communicator* Duplicate() const = 0; + + /** + * Opens a file with MPI_File_open. A collective operation + * @param filename + * @param mode + * @param info + * @return + */ + virtual MpiFile* OpenFile(const std::string& filename, int mode, + const MPI_Info info = MPI_INFO_NULL) const = 0; + + virtual void Barrier() const = 0; + virtual Request* Ibarrier() const = 0; + + virtual bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const = 0; + + template + void Broadcast(T& val, const int root) const; + template + void Broadcast(std::vector& vals, const int root) const; + + template + Request* Ibcast(T& val, const int root) const; + template + Request* Ibcast(std::vector& vals, const int root) const; + + template + T AllReduce(const T& val, const MPI_Op& op) const; + template + std::vector AllReduce(const std::vector& vals, const MPI_Op& op) const; + + template + Request* Iallreduce(const T& val, const MPI_Op& op, T& out) const; + + template + Request* Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const; + + template + T Reduce(const T& val, const MPI_Op& op, const int root) const; + template + std::vector Reduce(const std::vector& vals, const MPI_Op& op, const int root) const; + + template + std::vector Gather(const T& val, const int root) const; + + template + std::vector GatherV(const std::vector senddata, const std::vector recvcounts, + const int root) const; + + template + std::vector AllGather(const T& val) const; + + template + std::vector AllToAll(const std::vector& vals) const; + + template + void Send(const std::vector& val, int dest, int tag=0) const; + template + void Send(const T& val, int dest, int tag=0) const; + template + void Send(const T* valPtr, int count, int dest, int tag) const; + + template + void Recv(std::vector& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + template + void Recv(T& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + template + void Recv(T* val, int count, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + + template + Request* Isend(const T& val, int dest, int tag=0) const; + template + Request* Isend(const std::vector& vals, int dest, int tag=0) const; + template + Request* Isend(const T* valPtr, int count, int dest, int tag=0) const; + + template + Request* Issend(const T& val, int dest, int tag=0) const; + template + Request* Issend(const std::vector& vals, int dest, int tag=0) const; + template + Request* Issend(const T* valPtr, int count, int dest, int tag=0) const; + + template + Request* Irecv(T& val, int source, int tag=0) const; + template + Request* Irecv(std::vector& vals, int source, int tag=0) const; + template + Request* Irecv(T* valPtr, int count, int source, int tag=0) const; + + + protected: + virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; + virtual Request* IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; + virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const = 0; + virtual Request* IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const = 0; + virtual Request* IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const = 0; + virtual void ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const = 0; + virtual void GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype, + int root) const = 0; + virtual void GathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype, + int root) const = 0; + virtual void AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const = 0; + virtual void AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const = 0; + virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const = 0; + virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int src, int tag, MPI_Status* stat) const = 0; + virtual Request* IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const = 0; + virtual Request* IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const = 0; + virtual Request* IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int tag) const = 0; + }; + + // bool operator==(const Communicator& comm1, const Communicator& comm2); + // bool operator!=(const Communicator& comm1, const Communicator& comm2); + + } +} + +#include "comm/Communicator.hpp" + +#endif /* HEMELB_COMM_MPICOMMUNICATOR_H */ diff --git a/Code/comm/Communicator.hpp b/Code/comm/Communicator.hpp new file mode 100644 index 000000000..088f23201 --- /dev/null +++ b/Code/comm/Communicator.hpp @@ -0,0 +1,248 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. +#ifndef HEMELB_COMM_COMMUNICATOR_HPP +#define HEMELB_COMM_COMMUNICATOR_HPP + +#include "comm/MpiDataType.h" + +namespace hemelb +{ + namespace comm + { + template + void Communicator::Broadcast(T& val, const int root) const + { + BcastImpl(&val, 1, MpiDataType(), root); + } + template + void Communicator::Broadcast(std::vector& vals, const int root) const + { + BcastImpl(vals.data(), vals.size(), MpiDataType(), root); + } + + template + Request* Communicator::Ibcast(T& val, const int root) const + { + return IbcastImpl(&val, 1, MpiDataType(), root); + } + template + Request* Communicator::Ibcast(std::vector& vals, const int root) const + { + return IbcastImpl(vals.data(), vals.size(), MpiDataType(), root); + } + + template + T Communicator::AllReduce(const T& val, const MPI_Op& op) const + { + T ans; + AllreduceImpl(&val, &ans, 1, MpiDataType(), op); + return ans; + } + + template + std::vector Communicator::AllReduce(const std::vector& vals, const MPI_Op& op) const + { + std::vector ans(vals.size()); + AllreduceImpl(vals.data(), ans.data(), vals.size(), MpiDataType(), op); + return ans; + } + + template + Request* Communicator::Iallreduce(const T& val, const MPI_Op& op, T& out) const + { + return IallreduceImpl(&val, &out, 1, MpiDataType(), op); + } + + template + Request* Communicator::Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const + { + return IreduceImpl(&val, &out, 1, MpiDataType(), op, root); + } + + template + T Communicator::Reduce(const T& val, const MPI_Op& op, const int root) const + { + T ans; + ReduceImpl(&val, &ans, 1, MpiDataType(), op, root); + return ans; + } + + template + std::vector Communicator::Reduce(const std::vector& vals, const MPI_Op& op, + const int root) const + { + std::vector ans; + T* recvbuf = nullptr; + + if (Rank() == root) + { + // Standard says the address of receive buffer only matters at the root. + ans.resize(vals.size()); + recvbuf = ans.data(); + } + + ReduceImpl(vals.data(), recvbuf, vals.size(), MpiDataType(), op, root); + return ans; + } + + template + std::vector Communicator::Gather(const T& val, const int root) const + { + std::vector ans; + T* recvbuf = nullptr; + + if (Rank() == root) + { + // Standard says the address of receive buffer only matters at the root. + ans.resize(Size()); + recvbuf = ans.data(); + } + + GatherImpl(&val, 1, MpiDataType(), + recvbuf, 1, MpiDataType(), + root); + return ans; + } + + template + std::vector Communicator::GatherV(const std::vector senddata, + const std::vector recvcounts, + const int root) const + { + const int np = Size(); + const int sendcount = senddata.size(); + std::vector displs; + std::vector ans; + T* recvbuf = nullptr; + if (Rank() == root) + { + // Compute the displacements from the counts + displs.resize(np); + int total = 0; + for(size_t i = 0; i < np; ++i) { + displs[i] = total; + total += recvcounts[i]; + } + // set up recv buffer + ans.resize(total); + recvbuf = ans.data(); + } + + GathervImpl(senddata.data(), sendcount, MpiDataType(), + recvbuf, recvcounts.data(), displs.data(), MpiDataType(), + root); + return ans; + } + + template + std::vector Communicator::AllGather(const T& val) const + { + std::vector ans(Size()); + T* recvbuf = ans.data(); + + AllgatherImpl(&val, 1, MpiDataType(), + recvbuf, 1, MpiDataType()); + return ans; + } + + template + std::vector Communicator::AllToAll(const std::vector& vals) const + { + std::vector ans(vals.size()); + AlltoallImpl(vals.data(), 1, MpiDataType(), + ans.data(), 1, MpiDataType()); + return ans; + } + + // Send implementations + template + void Communicator::Send(const T* valPtr, int count, int dest, int tag) const + { + SendImpl(valPtr, count, MpiDataType(), dest, tag); + } + template + void Communicator::Send(const T& val, int dest, int tag) const + { + Send(&val, 1, dest, tag); + } + template + void Communicator::Send(const std::vector& vals, int dest, int tag) const + { + Send(vals.data(), vals.size(), dest, tag); + } + + // Recv implementations + template + void Communicator::Recv(T* valPtr, int count, int src, int tag, MPI_Status* stat) const + { + RecvImpl(valPtr, count, MpiDataType(), src, tag, stat); + } + template + void Communicator::Recv(T& val, int src, int tag, MPI_Status* stat) const + { + Recv(&val, 1, src, tag, stat); + } + template + void Communicator::Recv(std::vector& vals, int src, int tag, MPI_Status* stat) const + { + Recv(vals.data(), vals.size(), src, tag, stat); + } + + // Isend implementations + template + Request* Communicator::Isend(const T* valPtr, int count, int dest, int tag) const + { + return IsendImpl(valPtr, count, MpiDataType(), dest, tag); + } + template + Request* Communicator::Isend(const T& val, int dest, int tag) const + { + return Isend(&val, 1, dest, tag); + } + template + Request* Communicator::Isend(const std::vector& vals, int dest, int tag) const + { + return Isend(vals.data(), vals.size(), dest, tag); + } + + // Issend implementations + template + Request* Communicator::Issend(const T* valPtr, int count, int dest, int tag) const + { + return IssendImpl(valPtr, count, MpiDataType(), dest, tag); + } + template + Request* Communicator::Issend(const T& val, int dest, int tag) const + { + return Issend(&val, 1, dest, tag); + } + template + Request* Communicator::Issend(const std::vector& vals, int dest, int tag) const + { + return Issend(vals.data(), vals.size(), dest, tag); + } + + // Irecv implementations + template + Request* Communicator::Irecv(T* valPtr, int count, int source, int tag) const + { + return IrecvImpl(valPtr, count, MpiDataType(), source, tag); + } + template + Request* Communicator::Irecv(T& val, int source, int tag) const + { + return Irecv(&val, 1, source, tag); + } + template + Request* Communicator::Irecv(std::vector& vals, int source, int tag) const + { + return Irecv(&vals[0], vals.size(), source, tag); + } + + } +} + +#endif // HEMELB_COMM_COMMUNICATOR_HPP diff --git a/Code/comm/Group.h b/Code/comm/Group.h new file mode 100644 index 000000000..45204a1ac --- /dev/null +++ b/Code/comm/Group.h @@ -0,0 +1,54 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_COMM_GROUP_H +#define HEMELB_COMM_GROUP_H + +#include "units.h" +#include + +namespace hemelb +{ + namespace comm + { + class Group + { + public: + /** + * Default c'tor - initialises equivalent to an empty group + * (i.e. MPI_GROUP_NULL) + */ + Group(); + + /** + * Returns the local rank within the group + * @return + */ + virtual int Rank() const = 0; + + /** + * Returns the size of the group + * @return + */ + virtual int Size() const = 0; + + /** + * Exclude the provided ranks + * @param ranksToExclude + * @return + */ + virtual Group* Exclude(const std::vector& ranksToExclude) = 0; + /** + * Include the provided ranks + * @param ranksToExclude + * @return + */ + virtual Group* Include(const std::vector& ranksToInclude) = 0; + }; + } +} + +#endif /* HEMELB_COMM_GROUP_H */ diff --git a/Code/comm/MpiCommunicator.cc b/Code/comm/MpiCommunicator.cc new file mode 100644 index 000000000..830c06404 --- /dev/null +++ b/Code/comm/MpiCommunicator.cc @@ -0,0 +1,277 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#include "comm/MpiCommunicator.h" +#include "comm/MpiError.h" +#include "comm/MpiFile.h" +#include "comm/MpiRequest.h" + +namespace hemelb +{ + namespace comm + { + + namespace + { + void Deleter(MPI_Comm* comm) + { + int finalized; + HEMELB_MPI_CALL(MPI_Finalized, (&finalized)); + if (!finalized) + HEMELB_MPI_CALL(MPI_Comm_free, (comm)); + delete comm; + } + } + + MpiCommunicator::MpiCommunicator() : commPtr() + { + } + + MpiCommunicator::MpiCommunicator(MPI_Comm communicator, bool owner) : commPtr() + { + if (communicator == MPI_COMM_NULL) + return; + + if (owner) + { + commPtr.reset(new MPI_Comm(communicator), Deleter); + } + else + { + commPtr.reset(new MPI_Comm(communicator)); + } + } + + MpiCommunicator::operator MPI_Comm() const + { + return *commPtr; + } + + MpiCommunicator::~MpiCommunicator() + { + } + int MpiCommunicator::Rank() const + { + int rank; + HEMELB_MPI_CALL(MPI_Comm_rank, (*commPtr, &rank)); + return rank; + } + + int MpiCommunicator::Size() const + { + int size; + HEMELB_MPI_CALL(MPI_Comm_size, (*commPtr, &size)); + return size; + } + + void MpiCommunicator::Abort(int errCode) const + { + HEMELB_MPI_CALL(MPI_Abort, (*commPtr, errCode)); + } + + Communicator* MpiCommunicator::Duplicate() const + { + MPI_Comm newComm; + HEMELB_MPI_CALL(MPI_Comm_dup, (*commPtr, &newComm)); + return new MpiCommunicator(newComm, true); + } + + MpiFile* MpiCommunicator::OpenFile(const std::string& filename, int mode, + const MPI_Info info) const + { + MPI_File ans; + HEMELB_MPI_CALL( + MPI_File_open, + (*this, filename.c_str(), mode, info, &ans) + ); + return new MpiFile(this, ans); + } + + void MpiCommunicator::Barrier() const + { + HEMELB_MPI_CALL(MPI_Barrier, (*commPtr)); + } + + Request* MpiCommunicator::Ibarrier() const + { + MPI_Request req; + HEMELB_MPI_CALL(MPI_Ibarrier, (*commPtr, &req)); + return new MpiRequest(req); + } + + bool MpiCommunicator::Iprobe(int source, int tag, MPI_Status* stat) const + { + int flag; + HEMELB_MPI_CALL( + MPI_Iprobe, + (source, tag, *commPtr, &flag, stat) + ); + return flag; + } + + + void MpiCommunicator::BcastImpl(void* buf, int count, MPI_Datatype dt, + int root) const + { + HEMELB_MPI_CALL( + MPI_Bcast, + (buf, count, dt, root, *commPtr) + ); + } + + Request* MpiCommunicator::IbcastImpl(void* buf, int count, MPI_Datatype dt, + int root) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Ibcast, + (buf, count, dt, root, *commPtr, &req) + ); + return new MpiRequest(req); + } + + void MpiCommunicator::AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, + MPI_Op op) const + { + HEMELB_MPI_CALL( + MPI_Allreduce, + (send, ans, count, dt, op, *commPtr) + ); + } + + Request* MpiCommunicator::IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, + MPI_Op op) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Iallreduce, + (send, ans, count, dt, op, *commPtr, &req) + ); + return new MpiRequest(req); + } + + Request* MpiCommunicator::IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, + MPI_Op op, int root) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Ireduce, + (send, ans, count, dt, op, root, *commPtr, &req) + ); + return new MpiRequest(req); + } + + void MpiCommunicator::ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, + MPI_Op op, int root) const + { + HEMELB_MPI_CALL( + MPI_Reduce, + (send, ans, count, dt, op, root, *commPtr) + ); + } + + void MpiCommunicator::GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype, + int root) const + { + HEMELB_MPI_CALL( + MPI_Gather, + (send, sendcount, sendtype, + recv, recvcount, recvtype, + root, *commPtr) + ); + } + + void MpiCommunicator::GathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype, + int root) const + { + HEMELB_MPI_CALL( + MPI_Gatherv, + (sendbuf, sendcount, sendtype, + recvbuf, recvcounts, displs, recvtype, + root, *commPtr) + ); + } + void MpiCommunicator::AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const + { + HEMELB_MPI_CALL( + MPI_Allgather, + (send, sendcount, sendtype, + recv, recvcount, recvtype, + *commPtr) + ); + } + + void MpiCommunicator::AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const + { + HEMELB_MPI_CALL( + MPI_Alltoall, + (send, sendcount, sendtype, + recv, recvcount, recvtype, + *commPtr) + ); + } + + void MpiCommunicator::SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const + { + HEMELB_MPI_CALL( + MPI_Send, + (sendbuf, sendcount, sendtype, + dest, tag, *commPtr) + ); + } + void MpiCommunicator::RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int src, int tag, MPI_Status* stat) const + { + HEMELB_MPI_CALL( + MPI_Recv, + (recvbuf, recvcount, recvtype, + src, tag, *commPtr, stat) + ); + } + + Request* MpiCommunicator::IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const + { + MPI_Request ans; + HEMELB_MPI_CALL( + MPI_Isend, + (sendbuf, sendcount, sendtype, + dest, tag, *commPtr, &ans) + ); + return new MpiRequest(ans); + } + + Request* MpiCommunicator::IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const + { + MPI_Request ans; + HEMELB_MPI_CALL( + MPI_Issend, + (sendbuf, sendcount, sendtype, + dest, tag, *commPtr, &ans) + ); + return new MpiRequest(ans); + } + + Request* MpiCommunicator::IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int tag) const + { + MPI_Request ans; + HEMELB_MPI_CALL( + MPI_Irecv, + (recvbuf, recvcount, recvtype, + source, tag, *commPtr, &ans) + ); + return new MpiRequest(ans); + } + + } +} + diff --git a/Code/comm/MpiCommunicator.h b/Code/comm/MpiCommunicator.h new file mode 100644 index 000000000..29d56084d --- /dev/null +++ b/Code/comm/MpiCommunicator.h @@ -0,0 +1,110 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_COMM_MPICOMMUNICATOR_H +#define HEMELB_COMM_MPICOMMUNICATOR_H + +#include "comm/Communicator.h" + +namespace hemelb +{ + namespace comm + { + + // MPI communicator + class MpiCommunicator : public Communicator + { + public: + /** + * Constructor for an uninitialised communicator, equivalent to + * MPI_COMM_NULL + * @param communicator + */ + MpiCommunicator(); + + operator MPI_Comm() const; + /** + * Class has virtual methods so should have virtual d'tor. + */ + virtual ~MpiCommunicator(); + + /** + * Returns the local rank on the communicator + * @return + */ + virtual int Rank() const; + + /** + * Returns the size of the communicator (i.e. total number of procs involved). + * @return + */ + virtual int Size() const; + + /** + * Abort - should try to bring down all tasks, but no guarantees + * @param errCode + */ + virtual void Abort(int errCode) const; + + /** + * Duplicate the communicator - see MPI_COMM_DUP + * @return + */ + virtual Communicator* Duplicate() const; + + virtual MpiFile* OpenFile(const std::string& filename, int mode, + const MPI_Info info = MPI_INFO_NULL) const; + + virtual void Barrier() const; + virtual Request* Ibarrier() const; + + virtual bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const; + + private: + friend class MpiEnvironment; + /** + * Constructor to get data needed from an MPI communicator + * @param communicator + */ + MpiCommunicator(MPI_Comm communicator, bool willOwn); + + virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const; + virtual Request* IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const; + virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const; + virtual Request* IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const; + virtual Request* IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const; + virtual void ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const; + virtual void GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype, + int root) const; + virtual void GathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype, + int root) const; + virtual void AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const; + virtual void AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const; + virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const; + virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int src, int tag, MPI_Status* stat) const; + virtual Request* IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const; + virtual Request* IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const; + virtual Request* IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int tag) const; + + std::shared_ptr commPtr; + }; + + // bool operator==(const Communicator& comm1, const Communicator& comm2); + // bool operator!=(const Communicator& comm1, const Communicator& comm2); + + } +} + + +#endif /* HEMELB_COMM_MPICOMMUNICATOR_H */ diff --git a/Code/comm/MpiDataType.cc b/Code/comm/MpiDataType.cc index 883d1c163..d25cc95db 100644 --- a/Code/comm/MpiDataType.cc +++ b/Code/comm/MpiDataType.cc @@ -4,11 +4,11 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/MpiDataType.h" +#include "comm/MpiDataType.h" namespace hemelb { - namespace net + namespace comm { // Specializations of the above getters for built in types. // These mappings are taken directly from the MPI standard version 2.2, diff --git a/Code/comm/MpiDataType.h b/Code/comm/MpiDataType.h index b82b3acd3..f28eeaa41 100644 --- a/Code/comm/MpiDataType.h +++ b/Code/comm/MpiDataType.h @@ -4,8 +4,8 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIDATATYPE_H -#define HEMELB_NET_MPIDATATYPE_H +#ifndef HEMELB_COMM_MPIDATATYPE_H +#define HEMELB_COMM_MPIDATATYPE_H #include #if HEMELB_HAVE_CSTDINT @@ -34,7 +34,7 @@ (&(typeInstances->name), elementDisplacements + elementCounter) \ ); \ elementDisplacements[elementCounter] -= instanceAddr; \ - elementTypes[elementCounter] = ::hemelb::net::MpiDataType(typeInstances->name); \ + elementTypes[elementCounter] = ::hemelb::comm::MpiDataType(typeInstances->name); \ ++elementCounter #define HEMELB_MPI_TYPE_ADD_MEMBER(name) HEMELB_MPI_TYPE_ADD_MEMBER_N(name, 1) @@ -59,7 +59,7 @@ namespace hemelb { - namespace net + namespace comm { /* * There are templated functions for getting the MPI_Datatype @@ -73,7 +73,7 @@ namespace hemelb * You must specialize the template, e.g. * * template<> - * MPI_Datatype hemelb::net::MpiDataTypeTraits::RegisterMpiDataType() + * MPI_Datatype hemelb::comm::MpiDataTypeTraits::RegisterMpiDataType() * { * // Create the type * MPI_Type_commit(&type); @@ -85,7 +85,7 @@ namespace hemelb * Important note: to ensure C++ standard compliance, you MUST declare your * specialisations before use and you MUST ensure that the definition * is compiled exactly once (standard ODR). These MUST both be in the - * namespace hemelb::net. + * namespace hemelb::comm. * * Declaration is best done in the relevant header file. These templates are * only used by MpiCommunicator's templated communication methods so the @@ -162,4 +162,4 @@ namespace hemelb } } -#endif // HEMELB_NET_MPIDATATYPE_H +#endif // HEMELB_COMM_MPIDATATYPE_H diff --git a/Code/comm/MpiEnvironment.cc b/Code/comm/MpiEnvironment.cc index d5116704a..f7a5ea652 100644 --- a/Code/comm/MpiEnvironment.cc +++ b/Code/comm/MpiEnvironment.cc @@ -5,13 +5,13 @@ // license in the file LICENSE. #include -#include "net/MpiEnvironment.h" -#include "net/MpiError.h" -#include "net/MpiCommunicator.h" +#include "comm/MpiEnvironment.h" +#include "comm/MpiError.h" +#include "comm/MpiCommunicator.h" namespace hemelb { - namespace net + namespace comm { MpiEnvironment::MpiEnvironment(int& argc, char**& argv) : @@ -33,6 +33,11 @@ namespace hemelb } } + Communicator* MpiEnvironment::World() + { + return new MpiCommunicator(MPI_COMM_WORLD, false); + } + bool MpiEnvironment::Initialized() { int flag; @@ -53,7 +58,7 @@ namespace hemelb void MpiEnvironment::Abort(int errorCode) { - MpiCommunicator::World().Abort(errorCode); + World()->Abort(errorCode); } } diff --git a/Code/comm/MpiEnvironment.h b/Code/comm/MpiEnvironment.h index d04dbc3b1..f430db9e4 100644 --- a/Code/comm/MpiEnvironment.h +++ b/Code/comm/MpiEnvironment.h @@ -4,13 +4,16 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIENVIRONMENT_H -#define HEMELB_NET_MPIENVIRONMENT_H +#ifndef HEMELB_COMM_MPIENVIRONMENT_H +#define HEMELB_COMM_MPIENVIRONMENT_H + +#include "comm/Communicator.h" namespace hemelb { - namespace net + namespace comm { + /** * Manage the MPI environment and provide query/abort functions. * @@ -35,6 +38,7 @@ namespace hemelb */ ~MpiEnvironment(); + static Communicator* World(); /** * Query if MPI is initialised * @return @@ -63,4 +67,4 @@ namespace hemelb }; } } -#endif //HEMELB_NET_MPIENVIRONMENT_H +#endif //HEMELB_COMM_MPIENVIRONMENT_H diff --git a/Code/comm/MpiError.cc b/Code/comm/MpiError.cc index 3f0a416ce..1e8dd72d4 100644 --- a/Code/comm/MpiError.cc +++ b/Code/comm/MpiError.cc @@ -4,12 +4,12 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/MpiError.h" +#include "comm/MpiError.h" #include namespace hemelb { - namespace net + namespace comm { MpiError::MpiError(const char* mpiFunc_, const int errorCode_, const char* fileName_, const int lineNo_) : diff --git a/Code/comm/MpiError.h b/Code/comm/MpiError.h index df783a738..47e57908d 100644 --- a/Code/comm/MpiError.h +++ b/Code/comm/MpiError.h @@ -4,15 +4,15 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIERROR_H -#define HEMELB_NET_MPIERROR_H +#ifndef HEMELB_COMM_MPIERROR_H +#define HEMELB_COMM_MPIERROR_H #include #include "Exception.h" namespace hemelb { - namespace net + namespace comm { /** * Indicate an error to do with MPI. @@ -39,7 +39,7 @@ namespace hemelb { \ int _check_result = mpiFunc args; \ if (_check_result != MPI_SUCCESS) \ - throw ::hemelb::net::MpiError(#mpiFunc, _check_result, __FILE__, __LINE__); \ + throw ::hemelb::comm::MpiError(#mpiFunc, _check_result, __FILE__, __LINE__); \ } -#endif // HEMELB_NET_MPIERROR_H +#endif // HEMELB_COMM_MPIERROR_H diff --git a/Code/comm/MpiFile.cc b/Code/comm/MpiFile.cc index 3ada89c12..623688363 100644 --- a/Code/comm/MpiFile.cc +++ b/Code/comm/MpiFile.cc @@ -4,13 +4,12 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/MpiFile.h" -#include "net/MpiCommunicator.h" -#include "net/MpiConstness.h" +#include "comm/MpiFile.h" +#include "comm/MpiCommunicator.h" namespace hemelb { - namespace net + namespace comm { namespace { @@ -25,23 +24,12 @@ namespace hemelb } - MpiFile::MpiFile(const MpiCommunicator& parentComm, MPI_File fh) : - comm(&parentComm) + MpiFile::MpiFile(const MpiCommunicator* parentComm, MPI_File fh) : + comm(parentComm) { filePtr.reset(new MPI_File(fh), Deleter); } - MpiFile MpiFile::Open(const MpiCommunicator& comm, const std::string& filename, int mode, - const MPI_Info info) - { - MPI_File ans; - HEMELB_MPI_CALL( - MPI_File_open, - (comm, MpiConstCast(filename.c_str()), mode, info, &ans) - ); - return MpiFile(comm, ans); - } - void MpiFile::Close() { @@ -65,7 +53,7 @@ namespace hemelb { HEMELB_MPI_CALL( MPI_File_set_view, - (*filePtr, disp, etype, filetype, MpiConstCast(datarep.c_str()), info) + (*filePtr, disp, etype, filetype, datarep.c_str(), info) ); } } diff --git a/Code/comm/MpiFile.h b/Code/comm/MpiFile.h index 5af910cfb..232470bf7 100644 --- a/Code/comm/MpiFile.h +++ b/Code/comm/MpiFile.h @@ -4,33 +4,22 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIFILE_H -#define HEMELB_NET_MPIFILE_H +#ifndef HEMELB_COMM_MPIFILE_H +#define HEMELB_COMM_MPIFILE_H #include -#include "net/MpiError.h" -#include "net/MpiCommunicator.h" +#include "comm/MpiError.h" +#include "comm/MpiCommunicator.h" namespace hemelb { - namespace net + namespace comm { class MpiFile { public: MpiFile(); - /** - * Opens a file with MPI_File_open. A collective operation on comm. - * @param comm - * @param filename - * @param mode - * @param info - * @return - */ - static MpiFile Open(const MpiCommunicator& comm, const std::string& filename, int mode, - const MPI_Info info = MPI_INFO_NULL); - /** * Closes the file with MPI_File_close. * Note this is a collective operation. @@ -56,8 +45,10 @@ namespace hemelb void Write(const std::vector& buffer, MPI_Status* stat = MPI_STATUS_IGNORE); template void WriteAt(MPI_Offset offset, const std::vector& buffer, MPI_Status* stat = MPI_STATUS_IGNORE); - protected: - MpiFile(const MpiCommunicator& parentComm, MPI_File fh); + private: + friend class MpiCommunicator; + + MpiFile(const MpiCommunicator* parentComm, MPI_File fh); const MpiCommunicator* comm; std::shared_ptr filePtr; @@ -66,6 +57,6 @@ namespace hemelb } } -#include "net/MpiFile.hpp" +#include "comm/MpiFile.hpp" -#endif /* HEMELB_NET_MPIFILE_H */ +#endif /* HEMELB_COMM_MPIFILE_H */ diff --git a/Code/comm/MpiFile.hpp b/Code/comm/MpiFile.hpp index c75ce6d09..eb0f089de 100644 --- a/Code/comm/MpiFile.hpp +++ b/Code/comm/MpiFile.hpp @@ -4,14 +4,14 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIFILE_HPP -#define HEMELB_NET_MPIFILE_HPP +#ifndef HEMELB_COMM_MPIFILE_HPP +#define HEMELB_COMM_MPIFILE_HPP -#include "net/MpiFile.h" +#include "comm/MpiFile.h" namespace hemelb { - namespace net + namespace comm { template @@ -19,7 +19,7 @@ namespace hemelb { HEMELB_MPI_CALL( MPI_File_read, - (*filePtr, &buffer[0], buffer.size(), MpiDataType(), stat) + (*filePtr, buffer.data(), buffer.size(), MpiDataType(), stat) ); } template @@ -27,7 +27,7 @@ namespace hemelb { HEMELB_MPI_CALL( MPI_File_read_at, - (*filePtr, offset, &buffer[0], buffer.size(), MpiDataType(), stat) + (*filePtr, offset, buffer.data(), buffer.size(), MpiDataType(), stat) ); } @@ -36,7 +36,7 @@ namespace hemelb { HEMELB_MPI_CALL( MPI_File_write, - (*filePtr, MpiConstCast(&buffer[0]), buffer.size(), MpiDataType(), stat) + (*filePtr, buffer.data(), buffer.size(), MpiDataType(), stat) ); } template @@ -44,7 +44,7 @@ namespace hemelb { HEMELB_MPI_CALL( MPI_File_write_at, - (*filePtr, offset, MpiConstCast(&buffer[0]), buffer.size(), MpiDataType(), stat) + (*filePtr, offset, buffer.data(), buffer.size(), MpiDataType(), stat) ); } diff --git a/Code/comm/MpiGroup.cc b/Code/comm/MpiGroup.cc index 196d14b42..274643a45 100644 --- a/Code/comm/MpiGroup.cc +++ b/Code/comm/MpiGroup.cc @@ -4,12 +4,12 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/MpiGroup.h" -#include "net/MpiConstness.h" +#include "comm/MpiGroup.h" +#include "comm/MpiError.h" namespace hemelb { - namespace net + namespace comm { namespace { @@ -41,20 +41,20 @@ namespace hemelb return size; } - MpiGroup MpiGroup::Exclude(const std::vector& ranksToExclude) + Group* MpiGroup::Exclude(const std::vector& ranksToExclude) { MPI_Group ans; HEMELB_MPI_CALL(MPI_Group_excl, - (*groupPtr, ranksToExclude.size(), MpiConstCast(&ranksToExclude.front()), &ans)) - return MpiGroup(ans, true); + (*groupPtr, ranksToExclude.size(), ranksToExclude.data(), &ans)); + return new MpiGroup(ans, true); } - MpiGroup MpiGroup::Include(const std::vector& ranksToInclude) + Group* MpiGroup::Include(const std::vector& ranksToInclude) { MPI_Group ans; HEMELB_MPI_CALL(MPI_Group_incl, - (*groupPtr, ranksToInclude.size(), MpiConstCast(&ranksToInclude.front()), &ans)) - return MpiGroup(ans, true); + (*groupPtr, ranksToInclude.size(), ranksToInclude.data(), &ans)); + return new MpiGroup(ans, true); } MpiGroup::MpiGroup(MPI_Group grp, bool own) diff --git a/Code/comm/MpiGroup.h b/Code/comm/MpiGroup.h index 723967c29..1a7f3963a 100644 --- a/Code/comm/MpiGroup.h +++ b/Code/comm/MpiGroup.h @@ -4,19 +4,18 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIGROUP_H -#define HEMELB_NET_MPIGROUP_H +#ifndef HEMELB_COMM_MPIGROUP_H +#define HEMELB_COMM_MPIGROUP_H -#include "units.h" -#include "net/mpi.h" -#include "net/MpiCommunicator.h" +#include "comm/Group.h" +#include #include namespace hemelb { - namespace net + namespace comm { - class MpiGroup + class MpiGroup : public Group { public: /** @@ -28,26 +27,26 @@ namespace hemelb * Returns the local rank within the group * @return */ - int Rank() const; + virtual int Rank() const; /** * Returns the size of the group * @return */ - int Size() const; + virtual int Size() const; /** * Exclude the provided ranks * @param ranksToExclude * @return */ - MpiGroup Exclude(const std::vector& ranksToExclude); + virtual Group* Exclude(const std::vector& ranksToExclude); /** * Include the provided ranks * @param ranksToExclude * @return */ - MpiGroup Include(const std::vector& ranksToInclude); + virtual Group* Include(const std::vector& ranksToInclude); /** * Implicit cast to the underlying MPI_group @@ -76,4 +75,4 @@ namespace hemelb } } -#endif /* HEMELB_NET_MPIGROUP_H */ +#endif /* HEMELB_COMM_MPIGROUP_H */ diff --git a/Code/net/MpiRequest.cc b/Code/comm/MpiRequest.cc similarity index 61% rename from Code/net/MpiRequest.cc rename to Code/comm/MpiRequest.cc index 59045c3d2..f4bc264fd 100644 --- a/Code/net/MpiRequest.cc +++ b/Code/comm/MpiRequest.cc @@ -7,12 +7,11 @@ // specifically made by you with University College London. // -#include "net/MpiRequest.h" -#include "net/MpiStatus.h" +#include "comm/MpiRequest.h" namespace hemelb { - namespace net + namespace comm { MpiRequest::MpiRequest() : req(MPI_REQUEST_NULL) @@ -32,10 +31,6 @@ namespace hemelb { HEMELB_MPI_CALL(MPI_Wait, (&req, MPI_STATUS_IGNORE)); } - void MpiRequest::Wait(MpiStatus& stat) - { - HEMELB_MPI_CALL(MPI_Wait, (&req, stat.statPtr.get())); - } void MpiRequest::WaitAll(ReqVec& reqs) { @@ -62,51 +57,6 @@ namespace hemelb delete[] rawreqs; } - void MpiRequest::WaitAll(ReqVec& reqs, StatVec& stats) - { - size_t n = reqs.size(); - MPI_Request* rawreqs = new MPI_Request[n]; - MPI_Status* rawstats = new MPI_Status[n]; - try - { - for (size_t i = 0; i < n; ++i) - { - rawreqs[i] = reqs[i]; - } - - HEMELB_MPI_CALL( - MPI_Waitall, - (n, rawreqs, rawstats) - ); - - for (size_t i = 0; i < n; ++i) - { - stats[i] = MpiStatus(rawstats[i]); - } - } - catch (const std::exception& e) - { - delete[] rawreqs; - delete[] rawstats; - throw; - } - - delete[] rawreqs; - delete[] rawstats; - } - /* - void MpiRequest::WaitAll(ReqVec& reqs, StatVec& stats) - { - stats.resize(reqs.size()); - ReqVec::iterator reqIt = reqs.begin(); - StatVec::iterator statIt = stats.begin(); - for (; reqIt != reqs.end(); ++reqIt, ++statIt) - { - *statIt = reqIt ->Wait(); - } - } - */ - bool MpiRequest::Test() { int flag; diff --git a/Code/comm/MpiRequest.h b/Code/comm/MpiRequest.h index d35c3dbc4..8d2af08fb 100644 --- a/Code/comm/MpiRequest.h +++ b/Code/comm/MpiRequest.h @@ -11,21 +11,21 @@ #define HEMELB_NET_MPIREQUEST_H #include -#include "net/MpiError.h" +#include "comm/Request.h" +#include "comm/MpiError.h" namespace hemelb { - namespace net + namespace comm { - class MpiStatus; /** * */ - class MpiRequest + class MpiRequest : public Request { private: typedef std::vector ReqVec; - typedef std::vector StatVec; + // typedef std::vector StatVec; public: MpiRequest(); @@ -41,13 +41,13 @@ namespace hemelb } operator bool() const; - void Wait(); - void Wait(MpiStatus& stat); + virtual void Wait(); + // void Wait(MpiStatus& stat); static void WaitAll(ReqVec& reqs); - static void WaitAll(ReqVec& reqs, StatVec& stats); + // static void WaitAll(ReqVec& reqs, StatVec& stats); - bool Test(); + virtual bool Test(); static bool TestAll(ReqVec& reqs); private: diff --git a/Code/comm/Request.h b/Code/comm/Request.h new file mode 100644 index 000000000..38e380757 --- /dev/null +++ b/Code/comm/Request.h @@ -0,0 +1,55 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_COMM_REQUEST_H +#define HEMELB_COMM_REQUEST_H + +#include + +namespace hemelb +{ + namespace comm + { + // class Status; + /** + * + */ + class Request + { + public: + typedef std::vector ReqVec; + //typedef std::vector StatVec; + + Request(); + //Request(MPI_Request req); + + /** + * Allow implicit casts to MPI_Request + * @return The underlying MPI_Request + */ + // operator MPI_Request() const + // { + // return req; + // } + operator bool() const; + + virtual void Wait() = 0; + // void Wait(Status& stat); + + static void WaitAll(ReqVec& reqs); + // static void WaitAll(ReqVec& reqs, StatVec& stats); + + virtual bool Test() = 0; + static bool TestAll(ReqVec& reqs); + + }; + + } +} +#endif // HEMELB_COMM_REQUEST_H From 950cf89daee6814e223adc47f12b2129105cd2f1 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 19 Oct 2016 15:35:46 +0100 Subject: [PATCH 61/99] move comms to using std::shared_ptr --- Code/comm/Communicator.h | 50 ++++++++++++++++-------------- Code/comm/Communicator.hpp | 26 ++++++++-------- Code/comm/Group.h | 6 ++-- Code/{net => comm}/MapAllToAll.h | 25 +++++++-------- Code/comm/MpiCommunicator.cc | 52 +++++++++++++++++++++----------- Code/comm/MpiCommunicator.h | 34 +++++++++++---------- Code/comm/MpiFile.cc | 6 ++-- Code/comm/MpiFile.h | 10 +++--- Code/comm/MpiGroup.cc | 8 ++--- Code/comm/MpiGroup.h | 24 ++++++--------- Code/comm/Request.h | 3 +- 11 files changed, 134 insertions(+), 110 deletions(-) rename Code/{net => comm}/MapAllToAll.h (77%) diff --git a/Code/comm/Communicator.h b/Code/comm/Communicator.h index efc89566e..074a45130 100644 --- a/Code/comm/Communicator.h +++ b/Code/comm/Communicator.h @@ -22,6 +22,8 @@ namespace hemelb class Communicator { public: + typedef std::shared_ptr Ptr; + typedef std::shared_ptr ConstPtr; /** * Class has virtual methods so should have virtual d'tor. */ @@ -59,8 +61,10 @@ namespace hemelb * Duplicate the communicator - see MPI_COMM_DUP * @return */ - virtual Communicator* Duplicate() const = 0; + virtual Ptr Duplicate() const = 0; + virtual std::shared_ptr GetGroup() const = 0; + virtual Ptr Create(std::shared_ptr grp) const = 0; /** * Opens a file with MPI_File_open. A collective operation * @param filename @@ -68,11 +72,11 @@ namespace hemelb * @param info * @return */ - virtual MpiFile* OpenFile(const std::string& filename, int mode, - const MPI_Info info = MPI_INFO_NULL) const = 0; + virtual std::shared_ptr OpenFile(const std::string& filename, int mode, + const MPI_Info info = MPI_INFO_NULL) const = 0; virtual void Barrier() const = 0; - virtual Request* Ibarrier() const = 0; + virtual std::shared_ptr Ibarrier() const = 0; virtual bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const = 0; @@ -82,9 +86,9 @@ namespace hemelb void Broadcast(std::vector& vals, const int root) const; template - Request* Ibcast(T& val, const int root) const; + std::shared_ptr Ibcast(T& val, const int root) const; template - Request* Ibcast(std::vector& vals, const int root) const; + std::shared_ptr Ibcast(std::vector& vals, const int root) const; template T AllReduce(const T& val, const MPI_Op& op) const; @@ -92,10 +96,10 @@ namespace hemelb std::vector AllReduce(const std::vector& vals, const MPI_Op& op) const; template - Request* Iallreduce(const T& val, const MPI_Op& op, T& out) const; + std::shared_ptr Iallreduce(const T& val, const MPI_Op& op, T& out) const; template - Request* Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const; + std::shared_ptr Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const; template T Reduce(const T& val, const MPI_Op& op, const int root) const; @@ -130,33 +134,33 @@ namespace hemelb void Recv(T* val, int count, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; template - Request* Isend(const T& val, int dest, int tag=0) const; + std::shared_ptr Isend(const T& val, int dest, int tag=0) const; template - Request* Isend(const std::vector& vals, int dest, int tag=0) const; + std::shared_ptr Isend(const std::vector& vals, int dest, int tag=0) const; template - Request* Isend(const T* valPtr, int count, int dest, int tag=0) const; + std::shared_ptr Isend(const T* valPtr, int count, int dest, int tag=0) const; template - Request* Issend(const T& val, int dest, int tag=0) const; + std::shared_ptr Issend(const T& val, int dest, int tag=0) const; template - Request* Issend(const std::vector& vals, int dest, int tag=0) const; + std::shared_ptr Issend(const std::vector& vals, int dest, int tag=0) const; template - Request* Issend(const T* valPtr, int count, int dest, int tag=0) const; + std::shared_ptr Issend(const T* valPtr, int count, int dest, int tag=0) const; template - Request* Irecv(T& val, int source, int tag=0) const; + std::shared_ptr Irecv(T& val, int source, int tag=0) const; template - Request* Irecv(std::vector& vals, int source, int tag=0) const; + std::shared_ptr Irecv(std::vector& vals, int source, int tag=0) const; template - Request* Irecv(T* valPtr, int count, int source, int tag=0) const; + std::shared_ptr Irecv(T* valPtr, int count, int source, int tag=0) const; protected: virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; - virtual Request* IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; + virtual std::shared_ptr IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const = 0; - virtual Request* IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const = 0; - virtual Request* IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const = 0; + virtual std::shared_ptr IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const = 0; + virtual std::shared_ptr IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const = 0; virtual void ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const = 0; virtual void GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, void* recv, int recvcount, MPI_Datatype recvtype, @@ -172,11 +176,11 @@ namespace hemelb int dest, int tag) const = 0; virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, int src, int tag, MPI_Status* stat) const = 0; - virtual Request* IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + virtual std::shared_ptr IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest, int tag) const = 0; - virtual Request* IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + virtual std::shared_ptr IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest, int tag) const = 0; - virtual Request* IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + virtual std::shared_ptr IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, int source, int tag) const = 0; }; diff --git a/Code/comm/Communicator.hpp b/Code/comm/Communicator.hpp index 088f23201..74c810e7a 100644 --- a/Code/comm/Communicator.hpp +++ b/Code/comm/Communicator.hpp @@ -24,12 +24,12 @@ namespace hemelb } template - Request* Communicator::Ibcast(T& val, const int root) const + std::shared_ptr Communicator::Ibcast(T& val, const int root) const { return IbcastImpl(&val, 1, MpiDataType(), root); } template - Request* Communicator::Ibcast(std::vector& vals, const int root) const + std::shared_ptr Communicator::Ibcast(std::vector& vals, const int root) const { return IbcastImpl(vals.data(), vals.size(), MpiDataType(), root); } @@ -51,13 +51,13 @@ namespace hemelb } template - Request* Communicator::Iallreduce(const T& val, const MPI_Op& op, T& out) const + std::shared_ptr Communicator::Iallreduce(const T& val, const MPI_Op& op, T& out) const { return IallreduceImpl(&val, &out, 1, MpiDataType(), op); } template - Request* Communicator::Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const + std::shared_ptr Communicator::Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const { return IreduceImpl(&val, &out, 1, MpiDataType(), op, root); } @@ -193,51 +193,51 @@ namespace hemelb // Isend implementations template - Request* Communicator::Isend(const T* valPtr, int count, int dest, int tag) const + std::shared_ptr Communicator::Isend(const T* valPtr, int count, int dest, int tag) const { return IsendImpl(valPtr, count, MpiDataType(), dest, tag); } template - Request* Communicator::Isend(const T& val, int dest, int tag) const + std::shared_ptr Communicator::Isend(const T& val, int dest, int tag) const { return Isend(&val, 1, dest, tag); } template - Request* Communicator::Isend(const std::vector& vals, int dest, int tag) const + std::shared_ptr Communicator::Isend(const std::vector& vals, int dest, int tag) const { return Isend(vals.data(), vals.size(), dest, tag); } // Issend implementations template - Request* Communicator::Issend(const T* valPtr, int count, int dest, int tag) const + std::shared_ptr Communicator::Issend(const T* valPtr, int count, int dest, int tag) const { return IssendImpl(valPtr, count, MpiDataType(), dest, tag); } template - Request* Communicator::Issend(const T& val, int dest, int tag) const + std::shared_ptr Communicator::Issend(const T& val, int dest, int tag) const { return Issend(&val, 1, dest, tag); } template - Request* Communicator::Issend(const std::vector& vals, int dest, int tag) const + std::shared_ptr Communicator::Issend(const std::vector& vals, int dest, int tag) const { return Issend(vals.data(), vals.size(), dest, tag); } // Irecv implementations template - Request* Communicator::Irecv(T* valPtr, int count, int source, int tag) const + std::shared_ptr Communicator::Irecv(T* valPtr, int count, int source, int tag) const { return IrecvImpl(valPtr, count, MpiDataType(), source, tag); } template - Request* Communicator::Irecv(T& val, int source, int tag) const + std::shared_ptr Communicator::Irecv(T& val, int source, int tag) const { return Irecv(&val, 1, source, tag); } template - Request* Communicator::Irecv(std::vector& vals, int source, int tag) const + std::shared_ptr Communicator::Irecv(std::vector& vals, int source, int tag) const { return Irecv(&vals[0], vals.size(), source, tag); } diff --git a/Code/comm/Group.h b/Code/comm/Group.h index 45204a1ac..17c507b20 100644 --- a/Code/comm/Group.h +++ b/Code/comm/Group.h @@ -9,6 +9,7 @@ #include "units.h" #include +#include namespace hemelb { @@ -17,6 +18,7 @@ namespace hemelb class Group { public: + typedef std::shared_ptr Ptr; /** * Default c'tor - initialises equivalent to an empty group * (i.e. MPI_GROUP_NULL) @@ -40,13 +42,13 @@ namespace hemelb * @param ranksToExclude * @return */ - virtual Group* Exclude(const std::vector& ranksToExclude) = 0; + virtual Group::Ptr Exclude(const std::vector& ranksToExclude) const = 0; /** * Include the provided ranks * @param ranksToExclude * @return */ - virtual Group* Include(const std::vector& ranksToInclude) = 0; + virtual Group::Ptr Include(const std::vector& ranksToInclude) const = 0; }; } } diff --git a/Code/net/MapAllToAll.h b/Code/comm/MapAllToAll.h similarity index 77% rename from Code/net/MapAllToAll.h rename to Code/comm/MapAllToAll.h index 858cdaf9e..698529fa1 100644 --- a/Code/net/MapAllToAll.h +++ b/Code/comm/MapAllToAll.h @@ -11,22 +11,23 @@ #define HEMELB_NET_MAPALLTOALL_H #include -#include "net/mpi.h" +#include "comm/Communicator.h" +#include "comm/Request.h" namespace hemelb { - namespace net + namespace comm { template - void MapAllToAll(const MpiCommunicator& comm, + void MapAllToAll(Communicator::ConstPtr comm, const std::map& valsToSend, std::map& receivedVals, const int tag = 1234) { receivedVals.clear(); - std::vector sendReqs; - int localRank = comm.Rank(); + Request::ReqVec sendReqs; + int localRank = comm->Rank(); int nSends = valsToSend.size(); // Is localRank in the map of send destinations? if (valsToSend.find(localRank) != valsToSend.end()) @@ -48,36 +49,36 @@ namespace hemelb else { // Synchronous to ensure we know when this is matched - sendReqs[i] = comm.Issend(val, rank, tag); + sendReqs[i] = comm->Issend(val, rank, tag); i++; } } bool allProcsHaveHadAllSendsMatched = false; bool mySendsMatched = false; - MpiRequest barrierReq; + Request::Ptr barrierReq; // Allocate outside of loop to ensure compiler isn't reinitialising this // all the time. MPI_Status status; while(!mySendsMatched || !allProcsHaveHadAllSendsMatched) { - if (comm.Iprobe(MPI_ANY_SOURCE, tag, &status)) { + if (comm->Iprobe(MPI_ANY_SOURCE, tag, &status)) { // There is a message waiting - comm.Recv(receivedVals[status.MPI_SOURCE], status.MPI_SOURCE, status.MPI_TAG); + comm->Recv(receivedVals[status.MPI_SOURCE], status.MPI_SOURCE, status.MPI_TAG); } if (!mySendsMatched) { - if (MpiRequest::TestAll(sendReqs)) + if (Request::TestAll(sendReqs)) { mySendsMatched = true; - barrierReq = comm.Ibarrier(); + barrierReq = comm->Ibarrier(); } } else { - allProcsHaveHadAllSendsMatched = barrierReq.Test(); + allProcsHaveHadAllSendsMatched = barrierReq->Test(); } } } diff --git a/Code/comm/MpiCommunicator.cc b/Code/comm/MpiCommunicator.cc index 830c06404..b466f89c0 100644 --- a/Code/comm/MpiCommunicator.cc +++ b/Code/comm/MpiCommunicator.cc @@ -7,6 +7,7 @@ #include "comm/MpiError.h" #include "comm/MpiFile.h" #include "comm/MpiRequest.h" +#include "comm/MpiGroup.h" namespace hemelb { @@ -71,14 +72,29 @@ namespace hemelb HEMELB_MPI_CALL(MPI_Abort, (*commPtr, errCode)); } - Communicator* MpiCommunicator::Duplicate() const + Communicator::Ptr MpiCommunicator::Duplicate() const { MPI_Comm newComm; HEMELB_MPI_CALL(MPI_Comm_dup, (*commPtr, &newComm)); - return new MpiCommunicator(newComm, true); + return std::make_shared(newComm, true); + } + Group::Ptr MpiCommunicator::GetGroup() const + { + MPI_Group grp; + HEMELB_MPI_CALL(MPI_Comm_group, (*commPtr, &grp)); + return std::make_shared(grp, true); + } + + Communicator::Ptr MpiCommunicator::Create(std::shared_ptr grp) const + { + auto mpiGrp = std::dynamic_pointer_cast(grp); + + MPI_Comm newComm; + HEMELB_MPI_CALL(MPI_Comm_create, (*commPtr, *mpiGrp, &newComm)); + return std::make_shared(newComm, true); } - MpiFile* MpiCommunicator::OpenFile(const std::string& filename, int mode, + MpiFile::Ptr MpiCommunicator::OpenFile(const std::string& filename, int mode, const MPI_Info info) const { MPI_File ans; @@ -86,7 +102,7 @@ namespace hemelb MPI_File_open, (*this, filename.c_str(), mode, info, &ans) ); - return new MpiFile(this, ans); + return std::make_shared(shared_from_this(), ans); } void MpiCommunicator::Barrier() const @@ -94,11 +110,11 @@ namespace hemelb HEMELB_MPI_CALL(MPI_Barrier, (*commPtr)); } - Request* MpiCommunicator::Ibarrier() const + std::shared_ptr MpiCommunicator::Ibarrier() const { MPI_Request req; HEMELB_MPI_CALL(MPI_Ibarrier, (*commPtr, &req)); - return new MpiRequest(req); + return std::make_shared(req); } bool MpiCommunicator::Iprobe(int source, int tag, MPI_Status* stat) const @@ -121,7 +137,7 @@ namespace hemelb ); } - Request* MpiCommunicator::IbcastImpl(void* buf, int count, MPI_Datatype dt, + std::shared_ptr MpiCommunicator::IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const { MPI_Request req; @@ -129,7 +145,7 @@ namespace hemelb MPI_Ibcast, (buf, count, dt, root, *commPtr, &req) ); - return new MpiRequest(req); + return std::make_shared(req); } void MpiCommunicator::AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, @@ -141,7 +157,7 @@ namespace hemelb ); } - Request* MpiCommunicator::IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, + std::shared_ptr MpiCommunicator::IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const { MPI_Request req; @@ -149,10 +165,10 @@ namespace hemelb MPI_Iallreduce, (send, ans, count, dt, op, *commPtr, &req) ); - return new MpiRequest(req); + return std::make_shared(req); } - Request* MpiCommunicator::IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, + std::shared_ptr MpiCommunicator::IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const { MPI_Request req; @@ -160,7 +176,7 @@ namespace hemelb MPI_Ireduce, (send, ans, count, dt, op, root, *commPtr, &req) ); - return new MpiRequest(req); + return std::make_shared(req); } void MpiCommunicator::ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, @@ -236,7 +252,7 @@ namespace hemelb ); } - Request* MpiCommunicator::IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + std::shared_ptr MpiCommunicator::IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest, int tag) const { MPI_Request ans; @@ -245,10 +261,10 @@ namespace hemelb (sendbuf, sendcount, sendtype, dest, tag, *commPtr, &ans) ); - return new MpiRequest(ans); + return std::make_shared(ans); } - Request* MpiCommunicator::IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + std::shared_ptr MpiCommunicator::IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest, int tag) const { MPI_Request ans; @@ -257,10 +273,10 @@ namespace hemelb (sendbuf, sendcount, sendtype, dest, tag, *commPtr, &ans) ); - return new MpiRequest(ans); + return std::make_shared(ans); } - Request* MpiCommunicator::IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + std::shared_ptr MpiCommunicator::IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, int source, int tag) const { MPI_Request ans; @@ -269,7 +285,7 @@ namespace hemelb (recvbuf, recvcount, recvtype, source, tag, *commPtr, &ans) ); - return new MpiRequest(ans); + return std::make_shared(ans); } } diff --git a/Code/comm/MpiCommunicator.h b/Code/comm/MpiCommunicator.h index 29d56084d..9b33fb7c3 100644 --- a/Code/comm/MpiCommunicator.h +++ b/Code/comm/MpiCommunicator.h @@ -14,7 +14,7 @@ namespace hemelb { // MPI communicator - class MpiCommunicator : public Communicator + class MpiCommunicator : public Communicator, public std::enable_shared_from_this { public: /** @@ -23,6 +23,11 @@ namespace hemelb * @param communicator */ MpiCommunicator(); + /** + * Constructor to get data needed from an MPI communicator + * @param communicator + */ + MpiCommunicator(MPI_Comm communicator, bool willOwn); operator MPI_Comm() const; /** @@ -52,29 +57,26 @@ namespace hemelb * Duplicate the communicator - see MPI_COMM_DUP * @return */ - virtual Communicator* Duplicate() const; + virtual Communicator::Ptr Duplicate() const; - virtual MpiFile* OpenFile(const std::string& filename, int mode, + virtual std::shared_ptr GetGroup() const; + virtual Communicator::Ptr Create(std::shared_ptr grp) const; + + virtual std::shared_ptr OpenFile(const std::string& filename, int mode, const MPI_Info info = MPI_INFO_NULL) const; virtual void Barrier() const; - virtual Request* Ibarrier() const; + virtual std::shared_ptr Ibarrier() const; virtual bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const; private: - friend class MpiEnvironment; - /** - * Constructor to get data needed from an MPI communicator - * @param communicator - */ - MpiCommunicator(MPI_Comm communicator, bool willOwn); virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const; - virtual Request* IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const; + virtual std::shared_ptr IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const; virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const; - virtual Request* IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const; - virtual Request* IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const; + virtual std::shared_ptr IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const; + virtual std::shared_ptr IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const; virtual void ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const; virtual void GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, void* recv, int recvcount, MPI_Datatype recvtype, @@ -90,11 +92,11 @@ namespace hemelb int dest, int tag) const; virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, int src, int tag, MPI_Status* stat) const; - virtual Request* IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + virtual std::shared_ptr IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest, int tag) const; - virtual Request* IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + virtual std::shared_ptr IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, int dest, int tag) const; - virtual Request* IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + virtual std::shared_ptr IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, int source, int tag) const; std::shared_ptr commPtr; diff --git a/Code/comm/MpiFile.cc b/Code/comm/MpiFile.cc index 623688363..0a51c90f4 100644 --- a/Code/comm/MpiFile.cc +++ b/Code/comm/MpiFile.cc @@ -24,7 +24,7 @@ namespace hemelb } - MpiFile::MpiFile(const MpiCommunicator* parentComm, MPI_File fh) : + MpiFile::MpiFile(Communicator::ConstPtr parentComm, MPI_File fh) : comm(parentComm) { filePtr.reset(new MPI_File(fh), Deleter); @@ -44,9 +44,9 @@ namespace hemelb return *filePtr; } - const MpiCommunicator& MpiFile::GetCommunicator() const + Communicator::ConstPtr MpiFile::GetCommunicator() const { - return *comm; + return comm; } void MpiFile::SetView(MPI_Offset disp, MPI_Datatype etype, MPI_Datatype filetype, const std::string& datarep, MPI_Info info) diff --git a/Code/comm/MpiFile.h b/Code/comm/MpiFile.h index 232470bf7..b1d034ea8 100644 --- a/Code/comm/MpiFile.h +++ b/Code/comm/MpiFile.h @@ -9,7 +9,7 @@ #include #include "comm/MpiError.h" -#include "comm/MpiCommunicator.h" +#include "comm/Communicator.h" namespace hemelb { @@ -18,7 +18,10 @@ namespace hemelb class MpiFile { public: + typedef std::shared_ptr Ptr; + typedef std::shared_ptr ConstPtr; MpiFile(); + MpiFile(Communicator::ConstPtr parentComm, MPI_File fh); /** * Closes the file with MPI_File_close. @@ -34,7 +37,7 @@ namespace hemelb void SetView(MPI_Offset disp, MPI_Datatype etype, MPI_Datatype filetype, const std::string& datarep, MPI_Info info); - const MpiCommunicator& GetCommunicator() const; + Communicator::ConstPtr GetCommunicator() const; template void Read(std::vector& buffer, MPI_Status* stat = MPI_STATUS_IGNORE); @@ -48,9 +51,8 @@ namespace hemelb private: friend class MpiCommunicator; - MpiFile(const MpiCommunicator* parentComm, MPI_File fh); - const MpiCommunicator* comm; + Communicator::ConstPtr comm; std::shared_ptr filePtr; }; diff --git a/Code/comm/MpiGroup.cc b/Code/comm/MpiGroup.cc index 274643a45..5a1ab2202 100644 --- a/Code/comm/MpiGroup.cc +++ b/Code/comm/MpiGroup.cc @@ -41,20 +41,20 @@ namespace hemelb return size; } - Group* MpiGroup::Exclude(const std::vector& ranksToExclude) + Group::Ptr MpiGroup::Exclude(const std::vector& ranksToExclude) const { MPI_Group ans; HEMELB_MPI_CALL(MPI_Group_excl, (*groupPtr, ranksToExclude.size(), ranksToExclude.data(), &ans)); - return new MpiGroup(ans, true); + return std::make_shared(ans, true); } - Group* MpiGroup::Include(const std::vector& ranksToInclude) + Group::Ptr MpiGroup::Include(const std::vector& ranksToInclude) const { MPI_Group ans; HEMELB_MPI_CALL(MPI_Group_incl, (*groupPtr, ranksToInclude.size(), ranksToInclude.data(), &ans)); - return new MpiGroup(ans, true); + return std::make_shared(ans, true); } MpiGroup::MpiGroup(MPI_Group grp, bool own) diff --git a/Code/comm/MpiGroup.h b/Code/comm/MpiGroup.h index 1a7f3963a..d08b0ef4a 100644 --- a/Code/comm/MpiGroup.h +++ b/Code/comm/MpiGroup.h @@ -22,6 +22,14 @@ namespace hemelb * Default c'tor - initialises equivalent to MPI_GROUP_NULL */ MpiGroup(); + // Direct construction. + /** + * Construct from an MPI_Group + * @param grp + * @param own - Whether this instance is responsible for deleting the + * group when all copies are destroyed. + */ + MpiGroup(MPI_Group grp, bool own); /** * Returns the local rank within the group @@ -40,13 +48,13 @@ namespace hemelb * @param ranksToExclude * @return */ - virtual Group* Exclude(const std::vector& ranksToExclude); + virtual Group::Ptr Exclude(const std::vector& ranksToExclude) const; /** * Include the provided ranks * @param ranksToExclude * @return */ - virtual Group* Include(const std::vector& ranksToInclude); + virtual Group::Ptr Include(const std::vector& ranksToInclude) const; /** * Implicit cast to the underlying MPI_group @@ -58,18 +66,6 @@ namespace hemelb } private: - // Allow Communicators access to the c'tor. - friend class MpiCommunicator; - - // Direct construction. - /** - * Construct from an MPI_Group - * @param grp - * @param own - Whether this instance is responsible for deleting the - * group when all copies are destroyed. - */ - MpiGroup(MPI_Group grp, bool own); - std::shared_ptr groupPtr; }; } diff --git a/Code/comm/Request.h b/Code/comm/Request.h index 38e380757..6b48ded66 100644 --- a/Code/comm/Request.h +++ b/Code/comm/Request.h @@ -23,7 +23,8 @@ namespace hemelb class Request { public: - typedef std::vector ReqVec; + typedef std::shared_ptr Ptr; + typedef std::vector ReqVec; //typedef std::vector StatVec; Request(); From ff9a9202813858f11e75d7dbc6b8d1ace855d27b Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 19 Oct 2016 15:39:25 +0100 Subject: [PATCH 62/99] minor fixes and move geometry to new comms --- Code/extraction/LocalPropertyOutput.h | 2 +- Code/geometry/GeometryReader.cc | 63 ++++++----- Code/geometry/GeometryReader.h | 11 +- Code/geometry/LatticeData.cc | 32 +++--- Code/geometry/LatticeData.h | 6 +- Code/geometry/SiteData.cc | 2 +- Code/geometry/SiteData.h | 4 +- .../decomposition/BasicDecomposition.cc | 9 +- .../decomposition/BasicDecomposition.h | 6 +- .../decomposition/OptimisedDecomposition.cc | 107 +++++++++--------- .../decomposition/OptimisedDecomposition.h | 6 +- Code/geometry/needs/Needs.cc | 20 ++-- Code/geometry/needs/Needs.h | 12 +- .../neighbouring/NeighbouringDataManager.cc | 14 +-- .../neighbouring/RequiredSiteInformation.cc | 2 +- Code/steering/basic/SteeringComponentB.cc | 4 +- 16 files changed, 151 insertions(+), 149 deletions(-) diff --git a/Code/extraction/LocalPropertyOutput.h b/Code/extraction/LocalPropertyOutput.h index f2330aedb..5c62e360e 100644 --- a/Code/extraction/LocalPropertyOutput.h +++ b/Code/extraction/LocalPropertyOutput.h @@ -74,7 +74,7 @@ namespace hemelb /** * The MPI file to write into. */ - comm::MpiFile* outputFile; + comm::MpiFile::Ptr outputFile; /** * The data source to use for file output. diff --git a/Code/geometry/GeometryReader.cc b/Code/geometry/GeometryReader.cc index a156f69ba..0b2f01f4f 100644 --- a/Code/geometry/GeometryReader.cc +++ b/Code/geometry/GeometryReader.cc @@ -18,10 +18,11 @@ #include "geometry/GeometryReader.h" #include "lb/lattices/D3Q27.h" #include "net/net.h" -#include "net/IOCommunicator.h" #include "log/Logger.h" #include "util/utilityFunctions.h" #include "constants.h" +#include "comm/Group.h" + namespace hemelb { namespace geometry @@ -29,7 +30,7 @@ namespace hemelb GeometryReader::GeometryReader(const bool reserveSteeringCore, const lb::lattices::LatticeInfo& latticeInfo, - reporting::Timers &atimings, const net::IOCommunicator& ioComm) : + reporting::Timers &atimings, comm::Communicator::ConstPtr ioComm) : latticeInfo(latticeInfo), hemeLbComms(ioComm), timings(atimings) { // This rank should participate in the domain decomposition if @@ -38,15 +39,15 @@ namespace hemelb // - there's only one processor (so core 0 has to participate) // Create our own group, without the root node if we're not running with it. - if (reserveSteeringCore && ioComm.Size() > 1) + if (reserveSteeringCore && ioComm->Size() > 1) { - participateInTopology = !ioComm.OnIORank(); + participateInTopology = !ioComm->OnIORank(); std::vector lExclusions(1); lExclusions[0] = 0; - net::MpiGroup computeGroup = hemeLbComms.Group().Exclude(lExclusions); + comm::Group::Ptr computeGroup = hemeLbComms->GetGroup()->Exclude(lExclusions); // Create a communicator just for the domain decomposition. - computeComms = ioComm.Create(computeGroup); + computeComms = ioComm->Create(computeGroup); // Note that on the steering core, this is a null communicator. } else @@ -84,13 +85,13 @@ namespace hemelb ); // Open the file. - file = net::MpiFile::Open(hemeLbComms, dataFilePath, MPI_MODE_RDONLY, fileInfo); + file = hemeLbComms->OpenFile(dataFilePath, MPI_MODE_RDONLY, fileInfo); log::Logger::Log("Opened config file %s", dataFilePath.c_str()); // TODO: Why is there this fflush? fflush( NULL); // Set the view to the file. - file.SetView(0, MPI_CHAR, MPI_CHAR, "native", fileInfo); + file->SetView(0, MPI_CHAR, MPI_CHAR, "native", fileInfo); log::Logger::Log("Reading file preamble"); Geometry geometry = ReadPreamble(); @@ -99,7 +100,7 @@ namespace hemelb ReadHeader(geometry.GetBlockCount()); // Close the file - only the ranks participating in the topology need to read it again. - file.Close(); + file->Close(); timings[hemelb::reporting::Timers::initialDecomposition].Start(); log::Logger::Log("Beginning initial decomposition"); @@ -136,9 +137,9 @@ namespace hemelb { // Reopen in the file just between the nodes in the topology decomposition. Read in blocks // local to this node. - file = net::MpiFile::Open(computeComms, dataFilePath, MPI_MODE_RDONLY, fileInfo); + file = computeComms->OpenFile(dataFilePath, MPI_MODE_RDONLY, fileInfo); - ReadInBlocksWithHalo(geometry, principalProcForEachBlock, computeComms.Rank()); + ReadInBlocksWithHalo(geometry, principalProcForEachBlock, computeComms->Rank()); if (ShouldValidate()) { @@ -163,7 +164,7 @@ namespace hemelb { ValidateGeometry(geometry); } - file.Close(); + file->Close(); } // Finish up - close the file, set the timings, deallocate memory. @@ -177,12 +178,12 @@ namespace hemelb std::vector GeometryReader::ReadOnAllTasks(unsigned nBytes) { std::vector buffer(nBytes); - const net::MpiCommunicator& comm = file.GetCommunicator(); - if (comm.Rank() == HEADER_READING_RANK) + auto comm = file->GetCommunicator(); + if (comm->Rank() == HEADER_READING_RANK) { - file.Read(buffer); + file->Read(buffer); } - comm.Broadcast(buffer, HEADER_READING_RANK); + comm->Broadcast(buffer, HEADER_READING_RANK); return buffer; } @@ -324,7 +325,7 @@ namespace hemelb Needs needs(geometry.GetBlockCount(), readBlock, - util::NumericalFunctions::min(READING_GROUP_SIZE, computeComms.Size()), + util::NumericalFunctions::min(READING_GROUP_SIZE, computeComms->Size()), computeComms, ShouldValidate()); @@ -368,18 +369,18 @@ namespace hemelb net::Net net = net::Net(computeComms); - if (readingCore == computeComms.Rank()) + if (readingCore == computeComms->Rank()) { timings[hemelb::reporting::Timers::readBlock].Start(); // Read the data. compressedBlockData.resize(bytesPerCompressedBlock[blockNumber]); - file.ReadAt(offsetSoFar, compressedBlockData); + file->ReadAt(offsetSoFar, compressedBlockData); // Spread it. for (std::vector::const_iterator receiver = procsWantingThisBlock.begin(); receiver != procsWantingThisBlock.end(); receiver++) { - if (*receiver != computeComms.Rank()) + if (*receiver != computeComms->Rank()) { net.RequestSendV(compressedBlockData, *receiver); @@ -590,7 +591,7 @@ namespace hemelb proc_t GeometryReader::GetReadingCoreForBlock(site_t blockNumber) { return proc_t(blockNumber % util::NumericalFunctions::min(READING_GROUP_SIZE, - computeComms.Size())); + computeComms->Size())); } /** @@ -649,13 +650,13 @@ namespace hemelb // Reduce using a minimum to find the actual processor for each site (ignoring the // invalid entries). - std::vector procForSiteRecv = computeComms.AllReduce(myProcForSite, MPI_MIN); - std::vector siteDataRecv = computeComms.AllReduce(dummySiteData, MPI_MIN); + std::vector procForSiteRecv = computeComms->AllReduce(myProcForSite, MPI_MIN); + std::vector siteDataRecv = computeComms->AllReduce(dummySiteData, MPI_MIN); for (site_t site = 0; site < geometry.GetSitesPerBlock(); ++site) { - if (procForSiteRecv[site] == ConvertTopologyRankToGlobalRank(computeComms.Rank()) - && (myProcForSite[site] != ConvertTopologyRankToGlobalRank(computeComms.Rank()))) + if (procForSiteRecv[site] == ConvertTopologyRankToGlobalRank(computeComms->Rank()) + && (myProcForSite[site] != ConvertTopologyRankToGlobalRank(computeComms->Rank()))) { log::Logger::Log("Other cores think this core has site %li on block %li but it disagrees.", site, @@ -804,7 +805,7 @@ namespace hemelb // going to be moved to the current proc. idx_t moveIndex = 0; - for (proc_t fromProc = 0; fromProc < computeComms.Size(); ++fromProc) + for (proc_t fromProc = 0; fromProc < computeComms->Size(); ++fromProc) { for (idx_t moveNumber = 0; moveNumber < movesPerProc[fromProc]; ++moveNumber) { @@ -812,15 +813,15 @@ namespace hemelb idx_t toProc = movesList[3 * moveIndex + 2]; ++moveIndex; - if (toProc == (idx_t) computeComms.Rank()) + if (toProc == (idx_t) computeComms->Rank()) { - newProcForEachBlock[block] = computeComms.Rank(); + newProcForEachBlock[block] = computeComms->Rank(); } } } // Reread the blocks into the GlobalLatticeData now. - ReadInBlocksWithHalo(geometry, newProcForEachBlock, computeComms.Rank()); + ReadInBlocksWithHalo(geometry, newProcForEachBlock, computeComms->Rank()); } void GeometryReader::ImplementMoves(Geometry& geometry, @@ -857,7 +858,7 @@ namespace hemelb idx_t moveIndex = 0; // For each source proc, go through as many moves as it had. - for (proc_t fromProc = 0; fromProc < computeComms.Size(); ++fromProc) + for (proc_t fromProc = 0; fromProc < computeComms->Size(); ++fromProc) { for (idx_t moveNumber = 0; moveNumber < movesFromEachProc[fromProc]; ++moveNumber) { @@ -899,7 +900,7 @@ namespace hemelb { // If the global rank is not equal to the topology rank, we are not using rank 0 for // LBM. - return (hemeLbComms.Rank() == computeComms.Rank()) + return (hemeLbComms->Rank() == computeComms->Rank()) ? topologyRankIn : (topologyRankIn + 1); } diff --git a/Code/geometry/GeometryReader.h b/Code/geometry/GeometryReader.h index 6a5d0c40d..0c91f9a40 100644 --- a/Code/geometry/GeometryReader.h +++ b/Code/geometry/GeometryReader.h @@ -13,7 +13,6 @@ #include "io/writers/xdr/XdrReader.h" #include "lb/lattices/LatticeInfo.h" #include "lb/LbmParameters.h" -#include "net/mpi.h" #include "geometry/ParmetisHeader.h" #include "reporting/Timers.h" #include "util/Vector3D.h" @@ -21,7 +20,7 @@ #include "geometry/Geometry.h" #include "geometry/needs/Needs.h" -#include "net/MpiFile.h" +#include "comm/MpiFile.h" namespace hemelb { @@ -34,7 +33,7 @@ namespace hemelb typedef util::Vector3D BlockLocation; GeometryReader(const bool reserveSteeringCore, const lb::lattices::LatticeInfo&, - reporting::Timers &timings, const net::IOCommunicator& ioComm); + reporting::Timers &timings, comm::Communicator::ConstPtr ioComm); ~GeometryReader(); Geometry LoadAndDecompose(const std::string& dataFilePath); @@ -164,11 +163,11 @@ namespace hemelb //! Info about the connectivity of the lattice. const lb::lattices::LatticeInfo& latticeInfo; //! File accessed to read in the geometry data. - net::MpiFile file; + comm::MpiFile::Ptr file; //! Information about the file, to give cues and hints to MPI. - const net::IOCommunicator& hemeLbComms; //! HemeLB's main communicator - net::MpiCommunicator computeComms; //! Communication info for all ranks that will need a slice of the geometry (i.e. all non-steering cores) + comm::Communicator::ConstPtr hemeLbComms; //! HemeLB's main communicator + comm::Communicator::ConstPtr computeComms; //! Communication info for all ranks that will need a slice of the geometry (i.e. all non-steering cores) //! True iff this rank is participating in the domain decomposition. bool participateInTopology; diff --git a/Code/geometry/LatticeData.cc b/Code/geometry/LatticeData.cc index 01266dc06..dd14bef47 100644 --- a/Code/geometry/LatticeData.cc +++ b/Code/geometry/LatticeData.cc @@ -9,7 +9,7 @@ #include "debug/Debugger.h" #include "log/Logger.h" -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" #include "geometry/BlockTraverser.h" #include "geometry/LatticeData.h" #include "geometry/neighbouring/NeighbouringLatticeData.h" @@ -19,7 +19,7 @@ namespace hemelb { namespace geometry { - LatticeData::LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const net::IOCommunicator& comms_) : + LatticeData::LatticeData(const lb::lattices::LatticeInfo& latticeInfo, comm::Communicator::ConstPtr comms_) : latticeInfo(latticeInfo), neighbouringData(new neighbouring::NeighbouringLatticeData(latticeInfo)), comms(comms_) { } @@ -29,7 +29,7 @@ namespace hemelb delete neighbouringData; } - LatticeData::LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const Geometry& readResult, const net::IOCommunicator& comms_) : + LatticeData::LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const Geometry& readResult, comm::Communicator::ConstPtr comms_) : latticeInfo(latticeInfo), neighbouringData(new neighbouring::NeighbouringLatticeData(latticeInfo)), comms(comms_) { SetBasicDetails(readResult.GetBlockDimensions(), @@ -39,7 +39,7 @@ namespace hemelb // if debugging then output beliefs regarding geometry and neighbour list if (log::Logger::ShouldDisplay()) { - proc_t localRank = comms.Rank(); + proc_t localRank = comms->Rank(); for (std::vector::iterator itNeighProc = neighbouringProcs.begin(); itNeighProc != neighbouringProcs.end(); ++itNeighProc) { @@ -82,7 +82,7 @@ namespace hemelb std::vector domainEdgeWallDistance[COLLISION_TYPES]; std::vector midDomainWallDistance[COLLISION_TYPES]; - proc_t localRank = comms.Rank(); + proc_t localRank = comms->Rank(); // Iterate over all blocks in site units for (BlockTraverser blockTraverser(*this); blockTraverser.CurrentLocationValid(); blockTraverser.TraverseOne()) { @@ -275,9 +275,9 @@ namespace hemelb void LatticeData::CollectFluidSiteDistribution() { hemelb::log::Logger::Log("Gathering lattice info."); - fluidSitesOnEachProcessor = comms.AllGather(localFluidSites); + fluidSitesOnEachProcessor = comms->AllGather(localFluidSites); totalFluidSites = 0; - for (proc_t ii = 0; ii < comms.Size(); ++ii) + for (proc_t ii = 0; ii < comms->Size(); ++ii) { totalFluidSites += fluidSitesOnEachProcessor[ii]; } @@ -305,7 +305,7 @@ namespace hemelb siteSet.TraverseOne()) { if (block.GetProcessorRankForSite(siteSet.GetCurrentIndex()) - == comms.Rank()) + == comms->Rank()) { util::Vector3D globalCoords = blockSet.GetCurrentLocation() * GetBlockSize() + siteSet.GetCurrentLocation(); @@ -320,8 +320,8 @@ namespace hemelb } - std::vector siteMins = comms.AllReduce(localMins, MPI_MIN); - std::vector siteMaxes = comms.AllReduce(localMaxes, MPI_MAX); + std::vector siteMins = comms->AllReduce(localMins, MPI_MIN); + std::vector siteMaxes = comms->AllReduce(localMaxes, MPI_MAX); for (unsigned ii = 0; ii < 3; ++ii) { @@ -335,7 +335,7 @@ namespace hemelb // Allocate the index in which to put the distribution functions received from the other // process. std::vector > sharedDistributionLocationForEachProc = - std::vector >(comms.Size()); + std::vector >(comms->Size()); site_t totalSharedDistributionsSoFar = 0; // Set the remaining neighbouring processor data. for (size_t neighbourId = 0; neighbourId < neighbouringProcs.size(); neighbourId++) @@ -353,7 +353,7 @@ namespace hemelb void LatticeData::InitialiseNeighbourLookup(std::vector >& sharedFLocationForEachProc) { - const proc_t localRank = comms.Rank(); + const proc_t localRank = comms->Rank(); neighbourIndices.resize(latticeInfo.GetNumVectors() * localFluidSites); for (BlockTraverser blockTraverser(*this); blockTraverser.CurrentLocationValid(); blockTraverser.TraverseOne()) { @@ -433,7 +433,7 @@ namespace hemelb void LatticeData::InitialisePointToPointComms(std::vector >& sharedFLocationForEachProc) { - proc_t localRank = comms.Rank(); + proc_t localRank = comms->Rank(); // point-to-point communications are performed to match data to be // sent to/receive from different partitions; in this way, the // communication of the locations of the interface-dependent fluid @@ -465,7 +465,7 @@ namespace hemelb void LatticeData::InitialiseReceiveLookup(std::vector >& sharedFLocationForEachProc) { - proc_t localRank = comms.Rank(); + proc_t localRank = comms->Rank(); streamingIndicesForReceivedDistributions.resize(totalSharedFs); site_t f_count = GetLocalFluidSiteCount() * latticeInfo.GetNumVectors(); site_t sharedSitesSeen = 0; @@ -594,7 +594,7 @@ namespace hemelb // get the rank of the processor that owns the site procId = block.GetProcessorRankForSite(localSiteIndex); - if (procId != comms.Rank()) + if (procId != comms->Rank()) return false; if (procId == SITE_OR_BLOCK_SOLID) // means that the site is solid return false; @@ -704,7 +704,7 @@ namespace hemelb int LatticeData::GetLocalRank() const { - return comms.Rank(); + return comms->Rank(); } } diff --git a/Code/geometry/LatticeData.h b/Code/geometry/LatticeData.h index 121b0e803..9ac7879df 100644 --- a/Code/geometry/LatticeData.h +++ b/Code/geometry/LatticeData.h @@ -39,7 +39,7 @@ namespace hemelb template friend class lb::LBM; //! Let the LBM have access to internals so it can initialise the distribution arrays. template friend class Site; //! Let the inner classes have access to site-related data that's otherwise private. - LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const Geometry& readResult, const net::IOCommunicator& comms); + LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const Geometry& readResult, comm::Communicator::ConstPtr comms); virtual ~LatticeData(); @@ -337,7 +337,7 @@ namespace hemelb * class for the purpose of testing. * @return */ - LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const net::IOCommunicator& comms); + LatticeData(const lb::lattices::LatticeInfo& latticeInfo, comm::Communicator::ConstPtr comms); void SetBasicDetails(util::Vector3D blocks, site_t blockSize); @@ -574,7 +574,7 @@ namespace hemelb std::vector neighbourIndices; //! Data about neighbouring fluid sites. std::vector streamingIndicesForReceivedDistributions; //! The indices to stream to for distributions received from other processors. neighbouring::NeighbouringLatticeData *neighbouringData; - const net::IOCommunicator& comms; + comm::Communicator::ConstPtr comms; }; } } diff --git a/Code/geometry/SiteData.cc b/Code/geometry/SiteData.cc index d6258106f..fc75106bd 100644 --- a/Code/geometry/SiteData.cc +++ b/Code/geometry/SiteData.cc @@ -8,7 +8,7 @@ namespace hemelb { - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() diff --git a/Code/geometry/SiteData.h b/Code/geometry/SiteData.h index ccac22b15..0f6f7d8d5 100644 --- a/Code/geometry/SiteData.h +++ b/Code/geometry/SiteData.h @@ -8,11 +8,11 @@ #define HEMELB_GEOMETRY_SITEDATA_H #include "geometry/SiteDataBare.h" -#include "net/MpiDataType.h" +#include "comm/MpiDataType.h" namespace hemelb { - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); diff --git a/Code/geometry/decomposition/BasicDecomposition.cc b/Code/geometry/decomposition/BasicDecomposition.cc index ed74d3bcf..0f86b361b 100644 --- a/Code/geometry/decomposition/BasicDecomposition.cc +++ b/Code/geometry/decomposition/BasicDecomposition.cc @@ -5,7 +5,6 @@ // license in the file LICENSE. #include "geometry/decomposition/BasicDecomposition.h" -#include "net/mpi.h" namespace hemelb { @@ -16,7 +15,7 @@ namespace hemelb BasicDecomposition::BasicDecomposition(const Geometry& geometry, const lb::lattices::LatticeInfo& latticeInfo, - const net::MpiCommunicator& communicator, + comm::Communicator::ConstPtr communicator, const std::vector& fluidSitesOnEachBlock) : geometry(geometry), latticeInfo(latticeInfo), communicator(communicator), fluidSitesOnEachBlock(fluidSitesOnEachBlock) @@ -42,7 +41,7 @@ namespace hemelb DivideBlocks(procAssignedToEachBlock, unvisitedFluidBlockCount, geometry, - communicator.Size(), + communicator->Size(), fluidSitesOnEachBlock); } @@ -50,7 +49,7 @@ namespace hemelb { log::Logger::Log("Validating procForEachBlock"); - std::vector procForEachBlockRecv = communicator.AllReduce(procAssignedToEachBlock, MPI_MAX); + std::vector procForEachBlockRecv = communicator->AllReduce(procAssignedToEachBlock, MPI_MAX); for (site_t block = 0; block < geometry.GetBlockCount(); ++block) { @@ -74,7 +73,7 @@ namespace hemelb // required on each unit. proc_t currentUnit = 0; - site_t targetBlocksPerUnit = (site_t) ceil((double) unassignedBlocks / (double) (communicator.Size())); + site_t targetBlocksPerUnit = (site_t) ceil((double) unassignedBlocks / (double) (communicator->Size())); // Create an array to monitor whether each block has been assigned yet. std::vector blockAssigned(geometry.GetBlockCount(), false); diff --git a/Code/geometry/decomposition/BasicDecomposition.h b/Code/geometry/decomposition/BasicDecomposition.h index 43f1a85f5..02fb48ca8 100644 --- a/Code/geometry/decomposition/BasicDecomposition.h +++ b/Code/geometry/decomposition/BasicDecomposition.h @@ -9,7 +9,7 @@ #include "geometry/Geometry.h" #include "lb/lattices/LatticeInfo.h" -#include "net/MpiCommunicator.h" +#include "comm/Communicator.h" #include "units.h" #include "util/Vector3D.h" @@ -36,7 +36,7 @@ namespace hemelb */ BasicDecomposition(const Geometry& geometry, const lb::lattices::LatticeInfo& latticeInfo, - const net::MpiCommunicator& communicator, + comm::Communicator::ConstPtr communicator, const std::vector& fluidSitesOnEachBlock); /** @@ -111,7 +111,7 @@ namespace hemelb const Geometry& geometry; //! The geometry being decomposed. const lb::lattices::LatticeInfo& latticeInfo; //! The lattice to decompose for. - const net::MpiCommunicator& communicator; //! The communicator object being decomposed over. + comm::Communicator::ConstPtr communicator; //! The communicator object being decomposed over. const std::vector& fluidSitesOnEachBlock; //! The number of fluid sites on each block in the geometry. }; } /* namespace decomposition */ diff --git a/Code/geometry/decomposition/OptimisedDecomposition.cc b/Code/geometry/decomposition/OptimisedDecomposition.cc index 4089c14d6..72d71f5fa 100644 --- a/Code/geometry/decomposition/OptimisedDecomposition.cc +++ b/Code/geometry/decomposition/OptimisedDecomposition.cc @@ -9,6 +9,8 @@ #include "lb/lattices/D3Q27.h" #include "log/Logger.h" #include "net/net.h" +#include "comm/MpiCommunicator.h" +#include "Exception.h" namespace hemelb { @@ -17,7 +19,7 @@ namespace hemelb namespace decomposition { OptimisedDecomposition::OptimisedDecomposition( - reporting::Timers& timers, net::MpiCommunicator& comms, const Geometry& geometry, + reporting::Timers& timers, comm::Communicator::ConstPtr comms, const Geometry& geometry, const lb::lattices::LatticeInfo& latticeInfo, const std::vector& procForEachBlock, const std::vector& fluidSitesOnEachBlock) : timers(timers), comms(comms), geometry(geometry), latticeInfo(latticeInfo), @@ -42,7 +44,7 @@ namespace hemelb } // Populate the adjacency data arrays (for ParMetis) and validate if appropriate - idx_t localVertexCount = vtxDistribn[comms.Rank() + 1] - vtxDistribn[comms.Rank()]; + idx_t localVertexCount = vtxDistribn[comms->Rank() + 1] - vtxDistribn[comms->Rank()]; PopulateAdjacencyData(localVertexCount); @@ -111,7 +113,7 @@ namespace hemelb // comm* is a pointer to the MPI communicator of the processes involved // Initialise the partition vector. - partitionVector = std::vector(localVertexCount, comms.Rank()); + partitionVector = std::vector(localVertexCount, comms->Rank()); // Weight all vertices evenly. //std::vector < idx_t > vertexWeight(localVertexCount, 1); @@ -120,7 +122,7 @@ namespace hemelb PopulateVertexWeightData(localVertexCount); // Set the weights of each partition to be even, and to sum to 1. - idx_t desiredPartitionSize = comms.Size(); + idx_t desiredPartitionSize = comms->Size(); std::vector domainWeights(desiredPartitionSize, (real_t) (1.0) / ((real_t) (desiredPartitionSize))); @@ -156,7 +158,8 @@ namespace hemelb adjacenciesPerVertex.reserve(1); localAdjacencies.reserve(1); vertexWeights.reserve(1); - MPI_Comm communicator = comms; + auto mpi_comms = std::dynamic_pointer_cast(comms); + MPI_Comm communicator = *mpi_comms; ParMETIS_V3_PartKway(&vtxDistribn[0], &adjacenciesPerVertex[0], &localAdjacencies[0], @@ -174,10 +177,10 @@ namespace hemelb &communicator); log::Logger::Log("ParMetis returned."); - if (comms.Rank() == comms.Size() - 1) + if (comms->Rank() == comms->Size() - 1) { log::Logger::Log("ParMetis cut %d edges.", edgesCut); - if (edgesCut < 1 && comms.Size() > 2) + if (edgesCut < 1 && comms->Size() > 2) { throw Exception() << "The decomposition using ParMetis returned an edge cut of 0 even though there are multiple processes. " @@ -204,7 +207,7 @@ namespace hemelb blockK); // Only consider sites on this processor. - if (procForEachBlock[blockNumber] != comms.Rank()) + if (procForEachBlock[blockNumber] != comms->Rank()) { continue; } @@ -300,14 +303,14 @@ namespace hemelb WallSiteCounter, IOSiteCounter, WallIOSiteCounter, - comms.Rank(), + comms->Rank(), TotalSites, TotalCoreWeight); } void OptimisedDecomposition::PopulateSiteDistribution() { - vtxDistribn.resize(comms.Size() + 1, 0); + vtxDistribn.resize(comms->Size() + 1, 0); // Firstly, count the sites per processor. Do this off-by-one // to be compatible with ParMetis. for (site_t block = 0; block < geometry.GetBlockCount(); ++block) @@ -319,7 +322,7 @@ namespace hemelb } // Now make the count cumulative, again off-by-one. - for (proc_t rank = 0; rank < comms.Size(); ++rank) + for (proc_t rank = 0; rank < comms->Size(); ++rank) { vtxDistribn[rank + 1] += vtxDistribn[rank]; } @@ -363,7 +366,7 @@ namespace hemelb blockJ, blockK); // ... considering only the ones which live on this proc... - if (procForEachBlock[blockNumber] != comms.Rank()) + if (procForEachBlock[blockNumber] != comms->Rank()) { continue; } @@ -450,14 +453,14 @@ namespace hemelb // Right. Let's count how many sites we're going to have to move. Count the local number of // sites to be moved, and collect the site id and the destination processor. std::vector moveData; - const idx_t myLowest = vtxDistribn[comms.Rank()]; - const idx_t myHighest = vtxDistribn[comms.Rank() + 1] - 1; + const idx_t myLowest = vtxDistribn[comms->Rank()]; + const idx_t myHighest = vtxDistribn[comms->Rank() + 1] - 1; // For each local fluid site... for (idx_t ii = 0; ii <= (myHighest - myLowest); ++ii) { // ... if it's going elsewhere... - if (partitionVector[ii] != comms.Rank()) + if (partitionVector[ii] != comms->Rank()) { // ... get its id on the local processor... idx_t localFluidSiteId = myLowest + ii; @@ -513,7 +516,7 @@ namespace hemelb // If we've ended up on an impossible block, or one that doesn't live on this rank, // inform the user. if (fluidSiteBlock >= geometry.GetBlockCount() - || procForEachBlock[fluidSiteBlock] != comms.Rank()) + || procForEachBlock[fluidSiteBlock] != comms->Rank()) { log::Logger::Log("Partition element %i wrongly assigned to block %u of %i (block on processor %i)", ii, @@ -561,7 +564,7 @@ namespace hemelb // We also need to force some data upon blocks, i.e. when they're receiving data from a new // block they didn't previously want to know about. std::map > blockForcedUponX; - std::vector numberOfBlocksIForceUponX(comms.Size(), 0); + std::vector numberOfBlocksIForceUponX(comms->Size(), 0); for (idx_t moveNumber = 0; moveNumber < (idx_t) (moveData.size()); moveNumber += 3) { proc_t target_proc = moveData[moveNumber + 2]; @@ -580,13 +583,13 @@ namespace hemelb // Now find how many blocks are being forced upon us from every other core. log::Logger::Log("Moving forcing block numbers"); - std::vector blocksForcedOnMe = comms.AllToAll(numberOfBlocksIForceUponX); + std::vector blocksForcedOnMe = comms->AllToAll(numberOfBlocksIForceUponX); timers[hemelb::reporting::Timers::moveForcingNumbers].Stop(); timers[hemelb::reporting::Timers::moveForcingData].Start(); // Now get all the blocks being forced upon me. std::map > blocksForcedOnMeByEachProc; - for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { if (blocksForcedOnMe[otherProc] > 0) { @@ -606,7 +609,7 @@ namespace hemelb log::Logger::Log("Moving forcing block ids"); netForMoveSending.Dispatch(); // Now go through every block forced upon me and add it to the list of ones I want. - for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { if (blocksForcedOnMe[otherProc] > 0) { @@ -680,7 +683,7 @@ namespace hemelb log::Logger::Log("Calculating block requirements"); timers[hemelb::reporting::Timers::blockRequirements].Start(); // Populate numberOfBlocksRequiredFrom - for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { numberOfBlocksRequiredFrom[otherProc] = blockIdsIRequireFromX.count(otherProc) == 0 ? 0 : @@ -691,12 +694,12 @@ namespace hemelb } // Now perform the exchange s.t. each core knows how many blocks are required of it from // each other core. - numberOfBlocksXRequiresFromMe = comms.AllToAll(numberOfBlocksRequiredFrom); + numberOfBlocksXRequiresFromMe = comms->AllToAll(numberOfBlocksRequiredFrom); // Awesome. Now we need to get a list of all the blocks wanted from each core by each other // core. net::Net netForMoveSending(comms); - for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { blockIdsXRequiresFromMe[otherProc] = std::vector(numberOfBlocksXRequiresFromMe[otherProc]); @@ -723,13 +726,13 @@ namespace hemelb // block has no moves. for (site_t blockId = 0; blockId < geometry.GetBlockCount(); ++blockId) { - if (procForEachBlock[blockId] == comms.Rank()) + if (procForEachBlock[blockId] == comms->Rank()) { movesForEachLocalBlock[blockId] = 0; } } - for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { for (site_t blockNum = 0; blockNum < site_t(blockIdsXRequiresFromMe[otherProc].size()); @@ -763,7 +766,7 @@ namespace hemelb } net::Net netForMoveSending(comms); - for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); it != blockIdsIRequireFromX[otherProc].end(); ++it) @@ -809,7 +812,7 @@ namespace hemelb net::Net netForMoveSending(comms); - for (proc_t otherProc = 0; otherProc < proc_t(comms.Size()); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { allMoves[otherProc] = 0; for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); @@ -864,7 +867,7 @@ namespace hemelb */ void OptimisedDecomposition::PopulateMovesList() { - allMoves = std::vector(comms.Size()); + allMoves = std::vector(comms->Size()); // Create a map for looking up block Ids: the map is from the contiguous site index // of the last fluid site on the block, to the block id. @@ -908,8 +911,8 @@ namespace hemelb // Now we want to spread this info around so that each core knows which blocks each other // requires from it. - std::vector numberOfBlocksRequiredFrom(comms.Size(), 0); - std::vector numberOfBlocksXRequiresFromMe(comms.Size(), 0); + std::vector numberOfBlocksRequiredFrom(comms->Size(), 0); + std::vector numberOfBlocksXRequiresFromMe(comms->Size(), 0); std::map > blockIdsXRequiresFromMe; GetBlockRequirements(numberOfBlocksRequiredFrom, blockIdsIRequireFromX, @@ -944,14 +947,14 @@ namespace hemelb // Right. Let's count how many sites we're going to have to move. Count the local number of // sites to be moved, and collect the site id and the destination processor. std::vector moveData; - const idx_t myLowest = vtxDistribn[comms.Rank()]; - const idx_t myHighest = vtxDistribn[comms.Rank() + 1] - 1; + const idx_t myLowest = vtxDistribn[comms->Rank()]; + const idx_t myHighest = vtxDistribn[comms->Rank() + 1] - 1; // For each local fluid site... for (idx_t ii = 0; ii <= (myHighest - myLowest); ++ii) { // ... if it's going elsewhere... - if (partitionVector[ii] != comms.Rank()) + if (partitionVector[ii] != comms->Rank()) { // ... get its id on the local processor... idx_t localFluidSiteId = myLowest + ii; @@ -990,7 +993,7 @@ namespace hemelb /* Below is a modified version of PopulateMovesList(). */ void OptimisedDecomposition::LoadDecomposition() { - allMoves = std::vector(comms.Size()); + allMoves = std::vector(comms->Size()); // Create a map for looking up block Ids: the map is from the contiguous site index // of the last fluid site on the block, to the block id. @@ -1034,8 +1037,8 @@ namespace hemelb // Now we want to spread this info around so that each core knows which blocks each other // requires from it. - std::vector numberOfBlocksRequiredFrom(comms.Size(), 0); - std::vector numberOfBlocksXRequiresFromMe(comms.Size(), 0); + std::vector numberOfBlocksRequiredFrom(comms->Size(), 0); + std::vector numberOfBlocksXRequiresFromMe(comms->Size(), 0); std::map > blockIdsXRequiresFromMe; GetBlockRequirements(numberOfBlocksRequiredFrom, blockIdsIRequireFromX, @@ -1077,9 +1080,9 @@ namespace hemelb { log::Logger::Log("Validating the vertex distribution."); // vtxDistribn should be the same on all cores. - std::vector vtxDistribnRecv = comms.AllReduce(vtxDistribn, MPI_MIN); + std::vector vtxDistribnRecv = comms->AllReduce(vtxDistribn, MPI_MIN); - for (proc_t rank = 0; rank < comms.Size() + 1; ++rank) + for (proc_t rank = 0; rank < comms->Size() + 1; ++rank) { if (vtxDistribn[rank] != vtxDistribnRecv[rank]) { @@ -1101,12 +1104,12 @@ namespace hemelb log::Logger::Log("Validating the graph adjacency structure"); // Create an array of lists to store all of this node's adjacencies, arranged by the // proc the adjacent vertex is on. - std::vector > adjByNeighProc(comms.Size(), + std::vector > adjByNeighProc(comms->Size(), std::multimap()); // The adjacency data should correspond across all cores. for (idx_t index = 0; index < localVertexCount; ++index) { - idx_t vertex = vtxDistribn[comms.Rank()] + index; + idx_t vertex = vtxDistribn[comms->Rank()] + index; // Iterate over each adjacency (of each vertex). for (idx_t adjNumber = 0; adjNumber < (adjacenciesPerVertex[index + 1] - adjacenciesPerVertex[index]); @@ -1115,7 +1118,7 @@ namespace hemelb idx_t adjacentVertex = localAdjacencies[adjacenciesPerVertex[index] + adjNumber]; proc_t adjacentProc = -1; // Calculate the proc of the neighbouring vertex. - for (proc_t proc = 0; proc < comms.Size(); ++proc) + for (proc_t proc = 0; proc < comms->Size(); ++proc) { if (vtxDistribn[proc] <= adjacentVertex && vtxDistribn[proc + 1] > adjacentVertex) { @@ -1139,18 +1142,18 @@ namespace hemelb } // Create variables for the neighbour data to go into. - std::vector counts(comms.Size()); - std::vector > data(comms.Size()); + std::vector counts(comms->Size()); + std::vector > data(comms->Size()); log::Logger::Log("Validating neighbour data"); // Now spread and compare the adjacency information. Larger ranks send data to smaller // ranks which receive the data and compare it. - for (proc_t neigh = 0; neigh < proc_t(comms.Size()); ++neigh) + for (proc_t neigh = 0; neigh < proc_t(comms->Size()); ++neigh) { SendAdjacencyDataToLowerRankedProc(neigh, counts[neigh], data[neigh], adjByNeighProc[neigh]); - if (neigh < comms.Rank()) + if (neigh < comms->Rank()) { // Sending arrays don't perform comparison. continue; @@ -1167,11 +1170,11 @@ namespace hemelb std::vector& neighboursAdjacencyData, std::multimap& expectedAdjacencyData) { - if (neighbouringProc < comms.Rank()) + if (neighbouringProc < comms->Rank()) { // Send the array length. neighboursAdjacencyCount = 2 * expectedAdjacencyData.size(); - comms.Send(neighboursAdjacencyCount, neighbouringProc, 42); + comms->Send(neighboursAdjacencyCount, neighbouringProc, 42); // Create a sendable array (std::lists aren't organised in a sendable format). neighboursAdjacencyData.resize(neighboursAdjacencyCount); unsigned int adjacencyIndex = 0; @@ -1183,15 +1186,15 @@ namespace hemelb ++adjacencyIndex; } // Send the data to the neighbouringProc. - comms.Send(neighboursAdjacencyData, neighbouringProc, 43); + comms->Send(neighboursAdjacencyData, neighbouringProc, 43); } else // If this is a greater rank number than the neighbouringProc, receive the data. - if (neighbouringProc > comms.Rank()) + if (neighbouringProc > comms->Rank()) { - comms.Recv(neighboursAdjacencyCount, neighbouringProc, 42); + comms->Recv(neighboursAdjacencyCount, neighbouringProc, 42); neighboursAdjacencyData.resize(neighboursAdjacencyCount); - comms.Recv(neighboursAdjacencyData, neighbouringProc, 43); + comms->Recv(neighboursAdjacencyData, neighbouringProc, 43); } else // Neigh == mTopologyRank, i.e. neighbouring vertices on the same proc // Duplicate the data. @@ -1268,7 +1271,7 @@ namespace hemelb // Reduce finding the maximum across all nodes. Note that we have to use the maximum // because some cores will have -1 for a block (indicating that it has no neighbours on // that block. - std::vector firstSiteIndexPerBlockRecv = comms.AllReduce(firstSiteIndexPerBlock, + std::vector firstSiteIndexPerBlockRecv = comms->AllReduce(firstSiteIndexPerBlock, MPI_MAX); for (site_t block = 0; block < geometry.GetBlockCount(); ++block) diff --git a/Code/geometry/decomposition/OptimisedDecomposition.h b/Code/geometry/decomposition/OptimisedDecomposition.h index 8ad4e134f..0853ab664 100644 --- a/Code/geometry/decomposition/OptimisedDecomposition.h +++ b/Code/geometry/decomposition/OptimisedDecomposition.h @@ -12,7 +12,7 @@ #include "lb/lattices/LatticeInfo.h" #include "geometry/ParmetisHeader.h" #include "reporting/Timers.h" -#include "net/MpiCommunicator.h" +#include "comm/Communicator.h" #include "geometry/SiteData.h" #include "geometry/GeometryBlock.h" @@ -25,7 +25,7 @@ namespace hemelb class OptimisedDecomposition { public: - OptimisedDecomposition(reporting::Timers& timers, net::MpiCommunicator& comms, + OptimisedDecomposition(reporting::Timers& timers, comm::Communicator::ConstPtr comms, const Geometry& geometry, const lb::lattices::LatticeInfo& latticeInfo, const std::vector& procForEachBlock, @@ -206,7 +206,7 @@ namespace hemelb std::map > moveDataForEachBlock); reporting::Timers& timers; //! Timers for reporting. - net::MpiCommunicator& comms; //! Communicator + comm::Communicator::ConstPtr comms; //! Communicator const Geometry& geometry; //! The geometry being optimised. const lb::lattices::LatticeInfo& latticeInfo; //! The lattice info to optimise for. const std::vector& procForEachBlock; //! The processor assigned to each block at the moment diff --git a/Code/geometry/needs/Needs.cc b/Code/geometry/needs/Needs.cc index bf1bcb561..44dee944c 100644 --- a/Code/geometry/needs/Needs.cc +++ b/Code/geometry/needs/Needs.cc @@ -15,7 +15,7 @@ namespace hemelb Needs::Needs(const site_t blockCount, const std::vector& readBlock, const proc_t readingGroupSize, - net::MpiCommunicator& comm, + comm::Communicator::ConstPtr comm, bool shouldValidate_) : procsWantingBlocksBuffer(blockCount), communicator(comm), readingGroupSize(readingGroupSize), shouldValidate(shouldValidate_) { @@ -36,8 +36,8 @@ namespace hemelb for (proc_t readingCore = 0; readingCore < readingGroupSize; readingCore++) { blocksNeededSize[readingCore] = blocksNeededHere[readingCore].size(); - auto tmp = communicator.Gather(blocksNeededSize[readingCore], readingCore); - if (readingCore == communicator.Rank()) + auto tmp = communicator->Gather(blocksNeededSize[readingCore], readingCore); + if (readingCore == communicator->Rank()) blocksNeededSizes = std::move(tmp); } @@ -45,16 +45,16 @@ namespace hemelb std::vector blocksNeededOn; for (proc_t readingCore = 0; readingCore < readingGroupSize; readingCore++) { - auto tmp = communicator.GatherV(blocksNeededHere[readingCore], blocksNeededSizes, readingCore); - if (readingCore == communicator.Rank()) + auto tmp = communicator->GatherV(blocksNeededHere[readingCore], blocksNeededSizes, readingCore); + if (readingCore == communicator->Rank()) blocksNeededOn = std::move(tmp); } - if (communicator.Rank() < readingGroupSize) + if (communicator->Rank() < readingGroupSize) { int needsPassed = 0; // Transpose the blocks needed on cores matrix - for (proc_t sendingCore = 0; sendingCore < communicator.Size(); sendingCore++) + for (proc_t sendingCore = 0; sendingCore < communicator->Size(); sendingCore++) { for (int needForThisSendingCore = 0; needForThisSendingCore < blocksNeededSizes[sendingCore]; ++needForThisSendingCore) @@ -79,12 +79,12 @@ namespace hemelb { int neededHere = readBlock[block]; proc_t readingCore = GetReadingCoreForBlock(block); - std::vector procsWantingThisBlockBuffer = communicator.Gather(neededHere, readingCore); + std::vector procsWantingThisBlockBuffer = communicator->Gather(neededHere, readingCore); - if (communicator.Rank() == readingCore) + if (communicator->Rank() == readingCore) { - for (proc_t needingProcOld = 0; needingProcOld < communicator.Size(); needingProcOld++) + for (proc_t needingProcOld = 0; needingProcOld < communicator->Size(); needingProcOld++) { bool found = false; for (std::vector::iterator needingProc = procsWantingBlocksBuffer[block].begin(); diff --git a/Code/geometry/needs/Needs.h b/Code/geometry/needs/Needs.h index c49d0da1f..a878823ef 100644 --- a/Code/geometry/needs/Needs.h +++ b/Code/geometry/needs/Needs.h @@ -8,7 +8,7 @@ #define HEMELB_GEOMETRY_NEEDS_NEEDS_H #include #include "units.h" -#include "net/MpiCommunicator.h" +#include "comm/Communicator.h" namespace hemelb { @@ -33,10 +33,10 @@ namespace hemelb * @param comm MPI communicator. */ Needs(const site_t blockCount, - const std::vector& readBlock, - const proc_t readingGroupSize, - net::MpiCommunicator& comm, - bool shouldValidate); // Temporarily during the refactor, constructed just to abstract the block sharing bit + const std::vector& readBlock, + const proc_t readingGroupSize, + comm::Communicator::ConstPtr comm, + bool shouldValidate); // Temporarily during the refactor, constructed just to abstract the block sharing bit /*** * Which processors need a given block? @@ -58,7 +58,7 @@ namespace hemelb proc_t GetReadingCoreForBlock(const site_t blockNumber) const; private: std::vector > procsWantingBlocksBuffer; - const net::MpiCommunicator & communicator; + comm::Communicator::ConstPtr communicator; const proc_t readingGroupSize; bool shouldValidate; void Validate(const site_t blockCount, const std::vector& readBlock); diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.cc b/Code/geometry/neighbouring/NeighbouringDataManager.cc index 01f710939..161f71468 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.cc +++ b/Code/geometry/neighbouring/NeighbouringDataManager.cc @@ -9,7 +9,7 @@ #include "geometry/neighbouring/NeighbouringDataManager.h" #include "geometry/LatticeData.h" #include "log/Logger.h" -#include "net/MapAllToAll.h" +#include "comm/MapAllToAll.h" namespace hemelb { @@ -167,17 +167,17 @@ namespace hemelb // This will store the numbers of sites other ranks need from this rank CountMap countOfNeedsOnEachProcFromMe; // This is collective - const net::MpiCommunicator& comms = net.GetCommunicator(); - net::MapAllToAll(comms, countOfNeedsIHaveFromEachProc, countOfNeedsOnEachProcFromMe, 1234); + comm::Communicator::ConstPtr comms = net.GetCommunicator(); + comm::MapAllToAll(comms, countOfNeedsIHaveFromEachProc, countOfNeedsOnEachProcFromMe, 1234); - std::vector requestQueue; + comm::Request::ReqVec requestQueue; // Now, for every rank, which I need something from, send the ids of those for (CountMap::const_iterator countIt = countOfNeedsIHaveFromEachProc.begin(); countIt != countOfNeedsIHaveFromEachProc.end(); ++countIt) { int other = countIt->first; - requestQueue.push_back(comms.Isend(needsIHaveFromEachProc[other], other)); + requestQueue.push_back(comms->Isend(needsIHaveFromEachProc[other], other)); } // And for every rank, which needs something from me, receive those ids @@ -189,10 +189,10 @@ namespace hemelb int size = countIt->second; IdVec& otherNeeds = needsEachProcHasFromMe[other]; otherNeeds.resize(size); - requestQueue.push_back(comms.Irecv(otherNeeds, other)); + requestQueue.push_back(comms->Irecv(otherNeeds, other)); } - net::MpiRequest::WaitAll(requestQueue); + comm::Request::WaitAll(requestQueue); needsHaveBeenShared = true; } } diff --git a/Code/geometry/neighbouring/RequiredSiteInformation.cc b/Code/geometry/neighbouring/RequiredSiteInformation.cc index 6601cf626..4eea16c0f 100644 --- a/Code/geometry/neighbouring/RequiredSiteInformation.cc +++ b/Code/geometry/neighbouring/RequiredSiteInformation.cc @@ -5,7 +5,7 @@ // license in the file LICENSE. #include "geometry/neighbouring/RequiredSiteInformation.h" -#include "net/mpi.h" + namespace hemelb { namespace geometry diff --git a/Code/steering/basic/SteeringComponentB.cc b/Code/steering/basic/SteeringComponentB.cc index a9dc2cc05..c6ae73989 100644 --- a/Code/steering/basic/SteeringComponentB.cc +++ b/Code/steering/basic/SteeringComponentB.cc @@ -36,7 +36,7 @@ namespace hemelb void SteeringComponent::PreSend() { - if (collectiveComm.Rank() != RootRank) + if (collectiveComm->Rank() != RootRank) return; // Create a buffer for the data received. @@ -84,7 +84,7 @@ namespace hemelb } void SteeringComponent::Send() { - collectiveReq = collectiveComm.Ibcast(privateSteeringParams, RootRank); + collectiveReq = collectiveComm->Ibcast(privateSteeringParams, RootRank); } void SteeringComponent::PostReceive() From 709affbe964aacfec7c23860a788734d274f459e Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 19 Oct 2016 16:06:27 +0100 Subject: [PATCH 63/99] tweaks the colloids --- Code/colloids/ColloidController.cc | 2 +- Code/colloids/ColloidController.h | 4 ++-- Code/colloids/ParticleSet.cc | 2 +- Code/colloids/ParticleSet.h | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Code/colloids/ColloidController.cc b/Code/colloids/ColloidController.cc index ac116a441..f1c975894 100644 --- a/Code/colloids/ColloidController.cc +++ b/Code/colloids/ColloidController.cc @@ -34,7 +34,7 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, const std::string& outputPath, - const comm::Communicator* ioComms_, + comm::Communicator::ConstPtr ioComms_, reporting::Timers& timers) : ioComms(ioComms_), simulationState(simulationState), timers(timers) { diff --git a/Code/colloids/ColloidController.h b/Code/colloids/ColloidController.h index 64a235db5..9448c56be 100644 --- a/Code/colloids/ColloidController.h +++ b/Code/colloids/ColloidController.h @@ -34,7 +34,7 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, const std::string& outputPath, - const comm::Communicator* ioComms_, + comm::Communicator::ConstPtr ioComms_, reporting::Timers& timers); /** destructor - releases resources allocated by this class */ @@ -50,7 +50,7 @@ namespace hemelb private: /** Main code communicator */ - const comm::Communicator* ioComms; + comm::Communicator::ConstPtr ioComms; const lb::SimulationState& simulationState; diff --git a/Code/colloids/ParticleSet.cc b/Code/colloids/ParticleSet.cc index f0709f403..e9fdcd1c9 100644 --- a/Code/colloids/ParticleSet.cc +++ b/Code/colloids/ParticleSet.cc @@ -43,7 +43,7 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, std::vector& neighbourProcessors, - const comm::Communicator* ioComms_, + comm::Communicator::ConstPtr ioComms_, const std::string& outputPath) : ioComms(ioComms_), localRank(ioComms->Rank()), latDatLBM(latDatLBM), propertyCache(propertyCache), path(outputPath), net(ioComms) { diff --git a/Code/colloids/ParticleSet.h b/Code/colloids/ParticleSet.h index 09b382bc3..cb86353a3 100644 --- a/Code/colloids/ParticleSet.h +++ b/Code/colloids/ParticleSet.h @@ -29,7 +29,7 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, std::vector& neighbourProcessors, - const comm::Communicator* ioComms_, + comm::Communicator::ConstPtr ioComms_, const std::string& outputPath); /** destructor - de-allocates all Particle objects created by this Set */ @@ -60,7 +60,7 @@ namespace hemelb const void OutputInformation(const LatticeTimeStep timestep); private: - const comm::Communicator* ioComms; + comm::Communicator::ConstPtr ioComms; /** cached copy of local rank (obtained from topology) */ const proc_t localRank; @@ -106,7 +106,7 @@ namespace hemelb /** * MPI File handle to write with */ - comm::MpiFile* file; + comm::MpiFile::Ptr file; }; } } From a5b599c3c27e48549e45cde5c9d0c70b9f812174 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Wed, 19 Oct 2016 16:07:23 +0100 Subject: [PATCH 64/99] move net to the new comms - involves temp friend access for p2p mixins --- Code/comm/Communicator.h | 14 ++++++++ Code/comm/MpiCommunicator.cc | 10 ++++++ Code/comm/MpiCommunicator.h | 2 ++ Code/net/BaseNet.cc | 5 +-- Code/net/BaseNet.h | 13 ++++--- Code/net/CMakeLists.txt | 8 ----- Code/net/CollectiveAction.cc | 8 ++--- Code/net/CollectiveAction.h | 8 ++--- Code/net/ProcComms.h | 3 +- Code/net/StoredRequest.h | 2 +- Code/net/mixins/InterfaceDelegationNet.h | 6 ++-- Code/net/mixins/StoringNet.cc | 2 +- Code/net/mixins/StoringNet.h | 2 +- .../mixins/pointpoint/CoalescePointPoint.cc | 34 ++++++++----------- .../mixins/pointpoint/CoalescePointPoint.h | 9 +++-- .../mixins/pointpoint/ImmediatePointPoint.cc | 11 ++---- .../mixins/pointpoint/ImmediatePointPoint.h | 2 +- .../mixins/pointpoint/SeparatedPointPoint.cc | 30 +++++++--------- .../mixins/pointpoint/SeparatedPointPoint.h | 7 ++-- Code/net/net.h | 2 +- 20 files changed, 92 insertions(+), 86 deletions(-) diff --git a/Code/comm/Communicator.h b/Code/comm/Communicator.h index 074a45130..3d36fe9aa 100644 --- a/Code/comm/Communicator.h +++ b/Code/comm/Communicator.h @@ -12,6 +12,13 @@ namespace hemelb { + // TODO: remove when net is dead + namespace net + { + class CoalescePointPoint; + class SeparatedPointPoint; + class ImmediatePointPoint; + } namespace comm { class Group; @@ -156,6 +163,11 @@ namespace hemelb protected: + // TODO: remove when net is dead. + friend class net::CoalescePointPoint; + friend class net::SeparatedPointPoint; + friend class net::ImmediatePointPoint; + virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; virtual std::shared_ptr IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const = 0; @@ -174,6 +186,8 @@ namespace hemelb void* recv, int recvcount, MPI_Datatype recvtype) const = 0; virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, int dest, int tag) const = 0; + virtual void SsendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const = 0; virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, int src, int tag, MPI_Status* stat) const = 0; virtual std::shared_ptr IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, diff --git a/Code/comm/MpiCommunicator.cc b/Code/comm/MpiCommunicator.cc index b466f89c0..906e88be1 100644 --- a/Code/comm/MpiCommunicator.cc +++ b/Code/comm/MpiCommunicator.cc @@ -242,6 +242,16 @@ namespace hemelb dest, tag, *commPtr) ); } + + void MpiCommunicator::SsendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const + { + HEMELB_MPI_CALL( + MPI_Ssend, + (sendbuf, sendcount, sendtype, + dest, tag, *commPtr) + ); + } void MpiCommunicator::RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, int src, int tag, MPI_Status* stat) const { diff --git a/Code/comm/MpiCommunicator.h b/Code/comm/MpiCommunicator.h index 9b33fb7c3..9ed93067e 100644 --- a/Code/comm/MpiCommunicator.h +++ b/Code/comm/MpiCommunicator.h @@ -90,6 +90,8 @@ namespace hemelb void* recv, int recvcount, MPI_Datatype recvtype) const; virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, int dest, int tag) const; + virtual void SsendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const; virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, int src, int tag, MPI_Status* stat) const; virtual std::shared_ptr IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, diff --git a/Code/net/BaseNet.cc b/Code/net/BaseNet.cc index 46109924a..e4b54adbc 100644 --- a/Code/net/BaseNet.cc +++ b/Code/net/BaseNet.cc @@ -17,7 +17,8 @@ #include "net/BaseNet.h" #include "util/utilityFunctions.h" #include "util/Vector3D.h" -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" + namespace hemelb { namespace net @@ -30,7 +31,7 @@ namespace hemelb Wait(); } - BaseNet::BaseNet(const MpiCommunicator &commObject) : + BaseNet::BaseNet(comm::Communicator::ConstPtr commObject) : BytesSent(0), SyncPointsCounted(0), communicator(commObject) { } diff --git a/Code/net/BaseNet.h b/Code/net/BaseNet.h index b636cb12f..4b8d52d1e 100644 --- a/Code/net/BaseNet.h +++ b/Code/net/BaseNet.h @@ -13,8 +13,7 @@ #include #include "constants.h" -#include "net/mpi.h" -#include "net/MpiCommunicator.h" +#include "comm/Communicator.h" namespace hemelb { @@ -24,7 +23,7 @@ namespace hemelb class BaseNet { public: - BaseNet(const MpiCommunicator &communicator); + BaseNet(comm::Communicator::ConstPtr communicator); virtual ~BaseNet() { @@ -44,18 +43,18 @@ namespace hemelb */ void Dispatch(); - inline const MpiCommunicator &GetCommunicator() const + inline comm::Communicator::ConstPtr GetCommunicator() const { return communicator; } inline int Rank() const { - return communicator.Rank(); + return communicator->Rank(); } inline int Size() const { - return communicator.Size(); + return communicator->Size(); } protected: virtual void SendPointToPoint()=0; @@ -71,7 +70,7 @@ namespace hemelb std::vector & GetDisplacementsBuffer(); std::vector & GetCountsBuffer(); - const MpiCommunicator &communicator; + comm::Communicator::ConstPtr communicator; private: /*** * Buffers which can be used to store displacements and counts for cleaning up interfaces diff --git a/Code/net/CMakeLists.txt b/Code/net/CMakeLists.txt index 63f3b0503..0d85f0141 100644 --- a/Code/net/CMakeLists.txt +++ b/Code/net/CMakeLists.txt @@ -4,12 +4,8 @@ # file AUTHORS. This software is provided under the terms of the # license in the file LICENSE. add_library(hemelb_net - MpiDataType.cc MpiEnvironment.cc MpiError.cc - MpiCommunicator.cc MpiGroup.cc MpiFile.cc - MpiRequest.cc MpiStatus.cc IteratedAction.cc BaseNet.cc CollectiveAction.cc -IOCommunicator.cc mixins/pointpoint/CoalescePointPoint.cc mixins/pointpoint/SeparatedPointPoint.cc mixins/pointpoint/ImmediatePointPoint.cc @@ -19,7 +15,3 @@ configure_file ( "${PROJECT_SOURCE_DIR}/net/BuildInfo.h.in" "${PROJECT_BINARY_DIR}/net/BuildInfo.h" ) -configure_file ( - "${PROJECT_SOURCE_DIR}/net/MpiConstness.h.in" - "${PROJECT_BINARY_DIR}/net/MpiConstness.h" - ) \ No newline at end of file diff --git a/Code/net/CollectiveAction.cc b/Code/net/CollectiveAction.cc index 3ae06bf48..13d3044af 100644 --- a/Code/net/CollectiveAction.cc +++ b/Code/net/CollectiveAction.cc @@ -14,8 +14,8 @@ namespace hemelb { namespace net { - CollectiveAction::CollectiveAction(const MpiCommunicator& comm, reporting::Timer& wTimer) : - isCollectiveRunning(false), mustWait(false), collectiveComm(comm.Duplicate()), waitTimer(wTimer), collectiveReq() + CollectiveAction::CollectiveAction(comm::Communicator::ConstPtr comm, reporting::Timer& wTimer) : + isCollectiveRunning(false), mustWait(false), collectiveComm(comm->Duplicate()), waitTimer(wTimer), collectiveReq() { } @@ -65,13 +65,13 @@ namespace hemelb { waitTimer.Start(); if (mustWait) { - collectiveReq.Wait(); + collectiveReq->Wait(); mustWait = false; isCollectiveRunning = false; } else { - bool done = collectiveReq.Test(); + bool done = collectiveReq->Test(); if (done) isCollectiveRunning = false; } diff --git a/Code/net/CollectiveAction.h b/Code/net/CollectiveAction.h index 53ef84fae..e36b81787 100644 --- a/Code/net/CollectiveAction.h +++ b/Code/net/CollectiveAction.h @@ -12,7 +12,7 @@ #include "net/IteratedAction.h" #include "reporting/Timers.h" -#include "net/mpi.h" +#include "comm/Request.h" namespace hemelb { @@ -30,7 +30,7 @@ namespace hemelb } protected: - CollectiveAction(const MpiCommunicator& comm, reporting::Timer& waitTimer); + CollectiveAction(comm::Communicator::ConstPtr comm, reporting::Timer& waitTimer); /** * Initiate the collective. */ @@ -47,7 +47,7 @@ namespace hemelb /** * Private communicator for non-blocking collectives. */ - MpiCommunicator collectiveComm; + comm::Communicator::Ptr collectiveComm; /** * Timings for the wait etc. */ @@ -55,7 +55,7 @@ namespace hemelb /** * Request object for the collective */ - MpiRequest collectiveReq; + comm::Request::Ptr collectiveReq; }; } diff --git a/Code/net/ProcComms.h b/Code/net/ProcComms.h index 017431210..7031c81c6 100644 --- a/Code/net/ProcComms.h +++ b/Code/net/ProcComms.h @@ -6,8 +6,9 @@ #ifndef HEMELB_NET_PROCCOMMS_H #define HEMELB_NET_PROCCOMMS_H + #include "constants.h" -#include "net/mpi.h" +#include "comm/MpiDataType.h" #include "net/StoredRequest.h" #include #include diff --git a/Code/net/StoredRequest.h b/Code/net/StoredRequest.h index 6336a3450..f527de45f 100644 --- a/Code/net/StoredRequest.h +++ b/Code/net/StoredRequest.h @@ -8,7 +8,7 @@ #define HEMELB_NET_STOREDREQUEST_H #include #include "constants.h" -#include "net/mpi.h" +#include "comm/MpiDataType.h" namespace hemelb { namespace net diff --git a/Code/net/mixins/InterfaceDelegationNet.h b/Code/net/mixins/InterfaceDelegationNet.h index b9d0f01c2..d903c6b2d 100644 --- a/Code/net/mixins/InterfaceDelegationNet.h +++ b/Code/net/mixins/InterfaceDelegationNet.h @@ -18,7 +18,7 @@ namespace hemelb class InterfaceDelegationNet : public virtual BaseNet { public: - InterfaceDelegationNet(const MpiCommunicator& comms) : + InterfaceDelegationNet(comm::Communicator::ConstPtr comms) : BaseNet(comms) { } @@ -50,13 +50,13 @@ namespace hemelb template void RequestSend(const T* pointer, int count, proc_t rank) { - RequestSendImpl(const_cast(pointer), count, rank, MpiDataType()); + RequestSendImpl(const_cast(pointer), count, rank, comm::MpiDataType()); } template void RequestReceive(T* pointer, int count, proc_t rank) { - RequestReceiveImpl(pointer, count, rank, MpiDataType()); + RequestReceiveImpl(pointer, count, rank, comm::MpiDataType()); } }; diff --git a/Code/net/mixins/StoringNet.cc b/Code/net/mixins/StoringNet.cc index 9e8529e16..44dc6c617 100644 --- a/Code/net/mixins/StoringNet.cc +++ b/Code/net/mixins/StoringNet.cc @@ -9,7 +9,7 @@ namespace hemelb { namespace net { - StoringNet::StoringNet(const MpiCommunicator& comms) : + StoringNet::StoringNet(comm::Communicator::ConstPtr comms) : BaseNet(comms) { } diff --git a/Code/net/mixins/StoringNet.h b/Code/net/mixins/StoringNet.h index 1536fc628..80a77fb45 100644 --- a/Code/net/mixins/StoringNet.h +++ b/Code/net/mixins/StoringNet.h @@ -16,7 +16,7 @@ namespace hemelb class StoringNet : public virtual BaseNet { public: - StoringNet(const MpiCommunicator& comms); + StoringNet(comm::Communicator::ConstPtr comms); virtual void RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); virtual void RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); diff --git a/Code/net/mixins/pointpoint/CoalescePointPoint.cc b/Code/net/mixins/pointpoint/CoalescePointPoint.cc index 77e62cdf0..661fc8e68 100644 --- a/Code/net/mixins/pointpoint/CoalescePointPoint.cc +++ b/Code/net/mixins/pointpoint/CoalescePointPoint.cc @@ -16,8 +16,7 @@ namespace hemelb { if (requests.size() < count) { - requests.resize(count, MPI_Request()); - statuses.resize(count, MPI_Status()); + requests.resize(count); } } @@ -31,14 +30,11 @@ namespace hemelb for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); ++it) { - - MPI_Irecv(it->second.front().Pointer, - 1, - it->second.Type, - it->first, - 10, - communicator, - &requests[m]); + requests[m] = communicator->IrecvImpl(it->second.front().Pointer, + 1, + it->second.Type, + it->first, + 10); ++m; } @@ -87,13 +83,12 @@ namespace hemelb MPI_Type_size(it->second.Type, &TypeSizeStorage); //DTMP: BytesSent += TypeSizeStorage; //DTMP: - MPI_Isend(it->second.front().Pointer, - 1, - it->second.Type, - it->first, - 10, - communicator, - &requests[receiveProcessorComms.size() + m]); + requests[receiveProcessorComms.size() + m] = + communicator->IsendImpl(it->second.front().Pointer, + 1, + it->second.Type, + it->first, + 10); ++m; } @@ -123,9 +118,8 @@ namespace hemelb void CoalescePointPoint::WaitPointToPoint() { - - MPI_Waitall((int) (sendProcessorComms.size() + receiveProcessorComms.size()), &requests[0], &statuses[0]); - + comm::Request::WaitAll(requests); + for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); ++it) { diff --git a/Code/net/mixins/pointpoint/CoalescePointPoint.h b/Code/net/mixins/pointpoint/CoalescePointPoint.h index 69844b6be..3dd326648 100644 --- a/Code/net/mixins/pointpoint/CoalescePointPoint.h +++ b/Code/net/mixins/pointpoint/CoalescePointPoint.h @@ -8,6 +8,8 @@ #define HEMELB_NET_MIXINS_POINTPOINT_COALESCEPOINTPOINT_H #include "net/BaseNet.h" #include "net/mixins/StoringNet.h" +#include "comm/Request.h" + namespace hemelb { namespace net @@ -16,7 +18,7 @@ namespace hemelb { public: - CoalescePointPoint(const MpiCommunicator& comms) : + CoalescePointPoint(comm::Communicator::ConstPtr comms) : BaseNet(comms), StoringNet(comms), sendReceivePrepped(false) { } @@ -37,8 +39,9 @@ namespace hemelb // initialisation and during each iteration). Code using these must make sure // there are enough available. We do this in a way to minimise the number created // on each core, but also to minimise creation / deletion overheads. - std::vector requests; - std::vector statuses; + //std::vector requests; + comm::Request::ReqVec requests; + //std::vector statuses; }; } } diff --git a/Code/net/mixins/pointpoint/ImmediatePointPoint.cc b/Code/net/mixins/pointpoint/ImmediatePointPoint.cc index 9fa1f7ad6..10473fd1a 100644 --- a/Code/net/mixins/pointpoint/ImmediatePointPoint.cc +++ b/Code/net/mixins/pointpoint/ImmediatePointPoint.cc @@ -13,17 +13,12 @@ namespace hemelb { void ImmediatePointPoint::RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type) { - HEMELB_MPI_CALL( - MPI_Ssend, - (pointer, count, type, rank, 10, communicator) - ); + communicator->SsendImpl(pointer, count, type, rank, 10); } void ImmediatePointPoint::RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type) { - HEMELB_MPI_CALL( - MPI_Recv, - (pointer, count, type, rank, 10, communicator, MPI_STATUS_IGNORE) - ); + communicator->RecvImpl(pointer, count, type, rank, 10, MPI_STATUS_IGNORE); + } void ImmediatePointPoint::ReceivePointToPoint() diff --git a/Code/net/mixins/pointpoint/ImmediatePointPoint.h b/Code/net/mixins/pointpoint/ImmediatePointPoint.h index 967b56126..ed257c7a4 100644 --- a/Code/net/mixins/pointpoint/ImmediatePointPoint.h +++ b/Code/net/mixins/pointpoint/ImmediatePointPoint.h @@ -19,7 +19,7 @@ namespace hemelb { public: - ImmediatePointPoint(const MpiCommunicator& comms) : + ImmediatePointPoint(comm::Communicator::ConstPtr comms) : BaseNet(comms), StoringNet(comms) { } diff --git a/Code/net/mixins/pointpoint/SeparatedPointPoint.cc b/Code/net/mixins/pointpoint/SeparatedPointPoint.cc index 280699204..336176382 100644 --- a/Code/net/mixins/pointpoint/SeparatedPointPoint.cc +++ b/Code/net/mixins/pointpoint/SeparatedPointPoint.cc @@ -16,8 +16,7 @@ namespace hemelb { if (requests.size() < count) { - requests.resize(count, MPI_Request()); - statuses.resize(count, MPI_Status()); + requests.resize(count); } } @@ -31,14 +30,11 @@ namespace hemelb { for (ProcComms::iterator request = it->second.begin(); request != it->second.end(); request++) { - - MPI_Irecv(request->Pointer, - request->Count, - request->Type, - it->first, - 10, - communicator, - &requests[m]); + requests[m] = communicator->IrecvImpl(request->Pointer, + request->Count, + request->Type, + it->first, + 10); ++m; } } @@ -82,13 +78,11 @@ namespace hemelb { for (ProcComms::iterator request = it->second.begin(); request != it->second.end(); request++) { - MPI_Isend(request->Pointer, - request->Count, - request->Type, - it->first, - 10, - communicator, - &requests[count_receives+m]); + requests[count_receives+m] = communicator->IsendImpl(request->Pointer, + request->Count, + request->Type, + it->first, + 10); ++m; } } @@ -103,7 +97,7 @@ namespace hemelb void SeparatedPointPoint::WaitPointToPoint() { - MPI_Waitall(static_cast(count_sends+count_receives), &requests[0], &statuses[0]); + comm::Request::WaitAll(requests); receiveProcessorComms.clear(); sendProcessorComms.clear(); diff --git a/Code/net/mixins/pointpoint/SeparatedPointPoint.h b/Code/net/mixins/pointpoint/SeparatedPointPoint.h index 239e76e2a..d68abc369 100644 --- a/Code/net/mixins/pointpoint/SeparatedPointPoint.h +++ b/Code/net/mixins/pointpoint/SeparatedPointPoint.h @@ -8,6 +8,8 @@ #define HEMELB_NET_MIXINS_POINTPOINT_SEPARATEDPOINTPOINT_H #include "net/BaseNet.h" #include "net/mixins/StoringNet.h" +#include "comm/Request.h" + namespace hemelb { namespace net @@ -16,7 +18,7 @@ namespace hemelb { public: - SeparatedPointPoint(const MpiCommunicator& comms) : + SeparatedPointPoint(comm::Communicator::ConstPtr comms) : BaseNet(comms), StoringNet(comms), sendReceivePrepped(false), count_sends(0), count_receives(0) { @@ -38,8 +40,7 @@ namespace hemelb // initialisation and during each iteration). Code using these must make sure // there are enough available. We do this in a way to minimise the number created // on each core, but also to minimise creation / deletion overheads. - std::vector requests; - std::vector statuses; + comm::Request::ReqVec requests; unsigned int count_sends; unsigned int count_receives; }; diff --git a/Code/net/net.h b/Code/net/net.h index ccd6f491d..3adeb023c 100644 --- a/Code/net/net.h +++ b/Code/net/net.h @@ -18,7 +18,7 @@ namespace hemelb public InterfaceDelegationNet { public: - Net(const MpiCommunicator &communicator) : + Net(comm::Communicator::ConstPtr communicator) : BaseNet(communicator), StoringNet(communicator), PointPointImpl(communicator), InterfaceDelegationNet(communicator) { From d3cc2a86921101608dbae3e6a9afae1f96c18efc Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 20 Oct 2016 10:31:22 +0100 Subject: [PATCH 65/99] move remaining non-test stuff over --- Code/SimulationMaster.cc | 20 ++++++++++---------- Code/SimulationMaster.h | 4 ++-- Code/extraction/LocalPropertyOutput.cc | 2 +- Code/extraction/LocalPropertyOutput.h | 4 ++-- Code/extraction/PropertyActor.cc | 2 +- Code/extraction/PropertyActor.h | 2 +- Code/extraction/PropertyWriter.cc | 2 +- Code/extraction/PropertyWriter.h | 2 +- Code/lb/EntropyTester.h | 4 ++-- Code/lb/IncompressibilityChecker.cc | 4 ++-- Code/lb/StabilityTester.hpp | 2 +- Code/lb/iolets/BoundaryComms.cc | 22 +++++++++++++++++----- Code/lb/iolets/BoundaryCommunicator.cc | 11 ++++++++--- Code/lb/iolets/BoundaryCommunicator.h | 10 +++++++--- Code/lb/iolets/BoundaryValues.cc | 6 +++--- Code/lb/iolets/BoundaryValues.h | 4 ++-- Code/lb/iolets/InOutLetCosine.cc | 1 - Code/lb/iolets/InOutLetMultiscale.cc | 3 +-- Code/lb/lb.h | 2 +- Code/main.cc | 14 ++++++-------- Code/reporting/Timers.h | 4 ++-- 21 files changed, 71 insertions(+), 54 deletions(-) diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index 655594ba1..8039562a5 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -18,9 +18,9 @@ #include "io/xml/XmlAbstractionLayer.h" #include "colloids/ColloidController.h" #include "net/BuildInfo.h" -#include "net/IOCommunicator.h" #include "colloids/BodyForces.h" #include "colloids/BoundaryConditions.h" +#include "comm/MpiEnvironment.h" #include #include @@ -32,7 +32,7 @@ * Initialises member variables including the network topology * object. */ -SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, const hemelb::net::IOCommunicator& ioComm) : +SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, hemelb::comm::Communicator::ConstPtr ioComm) : ioComms(ioComm), timings(ioComm), build_info(), communicationNet(ioComm) { timings[hemelb::reporting::Timers::total].Start(); @@ -111,7 +111,7 @@ SimulationMaster::~SimulationMaster() */ bool SimulationMaster::IsCurrentProcTheIOProc() { - return ioComms.OnIORank(); + return ioComms->OnIORank(); } /** @@ -119,7 +119,7 @@ bool SimulationMaster::IsCurrentProcTheIOProc() */ int SimulationMaster::GetProcessorCount() { - return ioComms.Size(); + return ioComms->Size(); } /** @@ -193,7 +193,7 @@ void SimulationMaster::Initialise() timings[hemelb::reporting::Timers::colloidInitialisation].Stop(); // Initialise and begin the steering. - if (ioComms.OnIORank()) + if (ioComms->OnIORank()) { network = new hemelb::steering::Network(steeringSessionId, timings); } @@ -251,7 +251,7 @@ void SimulationMaster::Initialise() propertyDataSource = new hemelb::extraction::LbDataSourceIterator(latticeBoltzmannModel->GetPropertyCache(), *latticeData, - ioComms.Rank(), + ioComms->Rank(), *unitConverter); if (simConfig->PropertyOutputCount() > 0) @@ -301,7 +301,7 @@ void SimulationMaster::Initialise() stepManager->RegisterIteratedActorSteps(*propertyExtractor, 1); } - if (ioComms.OnIORank()) + if (ioComms->OnIORank()) { stepManager->RegisterIteratedActorSteps(*network, 1); } @@ -346,11 +346,11 @@ void SimulationMaster::RunSimulation() { // We need to keep the master rank in sync with the workers // Here, each rank notifies that it is beginning a time step - auto syncReq = ioComms.Ibarrier(); + auto syncReq = ioComms->Ibarrier(); DoTimeStep(); - syncReq.Wait(); + syncReq->Wait(); // Now all ranks have at least started this time step if (simulationState->IsTerminating()) @@ -459,7 +459,7 @@ void SimulationMaster::Abort() // This gives us something to work from when we have an error - we get the rank // that calls abort, and we get a stack-trace from the exception having been thrown. hemelb::log::Logger::Log("Aborting"); - hemelb::net::MpiEnvironment::Abort(1); + hemelb::comm::MpiEnvironment::Abort(1); exit(1); } diff --git a/Code/SimulationMaster.h b/Code/SimulationMaster.h index 9806432ab..a35dda748 100644 --- a/Code/SimulationMaster.h +++ b/Code/SimulationMaster.h @@ -29,7 +29,7 @@ class SimulationMaster { public: - SimulationMaster(hemelb::configuration::CommandLine &options, const hemelb::net::IOCommunicator& ioComms); + SimulationMaster(hemelb::configuration::CommandLine &options, hemelb::comm::Communicator::ConstPtr ioComms); virtual ~SimulationMaster(); void Abort(); @@ -55,7 +55,7 @@ class SimulationMaster hemelb::geometry::LatticeData* latticeData; hemelb::lb::LBM* latticeBoltzmannModel; hemelb::geometry::neighbouring::NeighbouringDataManager *neighbouringDataManager; - const hemelb::net::IOCommunicator& ioComms; + hemelb::comm::Communicator::ConstPtr ioComms; private: void Initialise(); diff --git a/Code/extraction/LocalPropertyOutput.cc b/Code/extraction/LocalPropertyOutput.cc index 5396c1be6..b4c0b4b5e 100644 --- a/Code/extraction/LocalPropertyOutput.cc +++ b/Code/extraction/LocalPropertyOutput.cc @@ -18,7 +18,7 @@ namespace hemelb { LocalPropertyOutput::LocalPropertyOutput(IterableDataSource& dataSource, const PropertyOutputFile* outputSpec, - const comm::Communicator* ioComms) : + comm::Communicator::ConstPtr ioComms) : comms(ioComms), dataSource(dataSource), outputSpec(outputSpec) { // Open the file as write-only, create it if it doesn't exist, don't create if the file diff --git a/Code/extraction/LocalPropertyOutput.h b/Code/extraction/LocalPropertyOutput.h index 5c62e360e..352670981 100644 --- a/Code/extraction/LocalPropertyOutput.h +++ b/Code/extraction/LocalPropertyOutput.h @@ -30,7 +30,7 @@ namespace hemelb * @return */ LocalPropertyOutput(IterableDataSource& dataSource, const PropertyOutputFile* outputSpec, - const comm::Communicator* ioComms); + comm::Communicator::ConstPtr ioComms); /** * Tidies up the LocalPropertyOutput (close files etc). @@ -70,7 +70,7 @@ namespace hemelb */ double GetOffset(OutputField::FieldType field) const; - const comm::Communicator* comms; + comm::Communicator::ConstPtr comms; /** * The MPI file to write into. */ diff --git a/Code/extraction/PropertyActor.cc b/Code/extraction/PropertyActor.cc index 43d047018..fd87f621c 100644 --- a/Code/extraction/PropertyActor.cc +++ b/Code/extraction/PropertyActor.cc @@ -14,7 +14,7 @@ namespace hemelb const std::vector& propertyOutputs, IterableDataSource& dataSource, reporting::Timers& timers, - const comm::Communicator* ioComms) : + comm::Communicator::ConstPtr ioComms) : simulationState(simulationState), timers(timers) { propertyWriter = new PropertyWriter(dataSource, propertyOutputs, ioComms); diff --git a/Code/extraction/PropertyActor.h b/Code/extraction/PropertyActor.h index bb82a69d7..55c3e2ee3 100644 --- a/Code/extraction/PropertyActor.h +++ b/Code/extraction/PropertyActor.h @@ -31,7 +31,7 @@ namespace hemelb const std::vector& propertyOutputs, IterableDataSource& dataSource, reporting::Timers& timers, - const comm::Communicator* ioComms); + comm::Communicator::ConstPtr ioComms); ~PropertyActor(); diff --git a/Code/extraction/PropertyWriter.cc b/Code/extraction/PropertyWriter.cc index 7c016f779..7386848e4 100644 --- a/Code/extraction/PropertyWriter.cc +++ b/Code/extraction/PropertyWriter.cc @@ -12,7 +12,7 @@ namespace hemelb { PropertyWriter::PropertyWriter(IterableDataSource& dataSource, const std::vector& propertyOutputs, - const comm::Communicator* ioComms) + comm::Communicator::ConstPtr ioComms) { for (unsigned outputNumber = 0; outputNumber < propertyOutputs.size(); ++outputNumber) { diff --git a/Code/extraction/PropertyWriter.h b/Code/extraction/PropertyWriter.h index 0a4a1f9ed..19fffd5ae 100644 --- a/Code/extraction/PropertyWriter.h +++ b/Code/extraction/PropertyWriter.h @@ -23,7 +23,7 @@ namespace hemelb * @param propertyOutputs * @return */ - PropertyWriter(IterableDataSource& dataSource, const std::vector& propertyOutputs, const comm::Communicator* ioComms); + PropertyWriter(IterableDataSource& dataSource, const std::vector& propertyOutputs, comm::Communicator::ConstPtr ioComms); /** * Destructor; deallocates memory used to store property info. diff --git a/Code/lb/EntropyTester.h b/Code/lb/EntropyTester.h index c42074990..4d9e545c0 100644 --- a/Code/lb/EntropyTester.h +++ b/Code/lb/EntropyTester.h @@ -135,7 +135,7 @@ namespace hemelb } void Send(void) { - collectiveReq = collectiveComm.Ireduce(localHTheorem, MPI_MAX, RootRank, globalHTheorem); + collectiveReq = collectiveComm->Ireduce(localHTheorem, MPI_MAX, RootRank, globalHTheorem); } /** * Take the combined stability information (an int, with a value of hemelb::lb::Unstable @@ -143,7 +143,7 @@ namespace hemelb */ void PostReceive() { - if (collectiveComm.Rank() == RootRank && globalHTheorem == DISOBEYED) + if (collectiveComm->Rank() == RootRank && globalHTheorem == DISOBEYED) { log::Logger::Log("H Theorem violated."); } diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index 142b2ac75..63879c318 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -7,7 +7,7 @@ namespace hemelb { - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() @@ -114,7 +114,7 @@ namespace hemelb void IncompressibilityChecker::Send(void) { // Begin collective. - collectiveReq = collectiveComm.Iallreduce(localDensity, reduction, globalDensity); + collectiveReq = collectiveComm->Iallreduce(localDensity, reduction, globalDensity); } bool IncompressibilityChecker::IsDensityDiffWithinRange() const diff --git a/Code/lb/StabilityTester.hpp b/Code/lb/StabilityTester.hpp index b1b66cf02..445aae279 100644 --- a/Code/lb/StabilityTester.hpp +++ b/Code/lb/StabilityTester.hpp @@ -103,7 +103,7 @@ namespace hemelb void StabilityTester::Send(void) { // Begin collective. - collectiveReq = collectiveComm.Iallreduce(localStability, MPI_MIN, globalStability); + collectiveReq = collectiveComm->Iallreduce(localStability, MPI_MIN, globalStability); } /** diff --git a/Code/lb/iolets/BoundaryComms.cc b/Code/lb/iolets/BoundaryComms.cc index 99826a75f..decb6441f 100644 --- a/Code/lb/iolets/BoundaryComms.cc +++ b/Code/lb/iolets/BoundaryComms.cc @@ -6,7 +6,7 @@ #include "lb/iolets/BoundaryComms.h" #include "lb/iolets/BoundaryValues.h" -#include "net/IOCommunicator.h" +#include "comm/MpiCommunicator.h" #include "util/utilityFunctions.h" namespace hemelb @@ -46,6 +46,8 @@ namespace hemelb void BoundaryComms::Wait() { + // TODO: refactor the raw MPI calls into the wrapped stuff + if (hasBoundary) { HEMELB_MPI_CALL( @@ -56,6 +58,8 @@ namespace hemelb void BoundaryComms::WaitAllComms() { + // TODO: refactor the raw MPI calls into the wrapped stuff + // Now wait for all to complete if (bcComm.IsCurrentProcTheBCProc()) { @@ -80,16 +84,19 @@ namespace hemelb // It is up to the caller to make sure only BCproc calls send void BoundaryComms::Send(distribn_t* density) { + // TODO: refactor the raw MPI calls into the wrapped stuff + auto mpi = std::dynamic_pointer_cast(bcComm.GetComm()); + MPI_Comm mpi_comm = *mpi; for (int proc = 0; proc < nProcs; proc++) { HEMELB_MPI_CALL( MPI_Isend, ( density, 1, - net::MpiDataType(*density), + comm::MpiDataType(*density), procsList[proc], 100, - bcComm, + mpi_comm, &sendRequest[proc] )); } @@ -97,16 +104,19 @@ namespace hemelb void BoundaryComms::Receive(distribn_t* density) { + // TODO: refactor the raw MPI calls into the wrapped stuff + auto mpi = std::dynamic_pointer_cast(bcComm.GetComm()); + MPI_Comm mpi_comm = *mpi; if (hasBoundary) { HEMELB_MPI_CALL( MPI_Irecv, ( density, 1, - net::MpiDataType(*density), + comm::MpiDataType(*density), bcComm.GetBCProcRank(), 100, - bcComm, + mpi_comm, &receiveRequest )); } @@ -114,6 +124,8 @@ namespace hemelb void BoundaryComms::FinishSend() { + // TODO: refactor the raw MPI calls into the wrapped stuff + // Don't move on to next step with BC proc until all messages have been sent // Precautionary measure to make sure proc doesn't overwrite, before message is sent if (bcComm.IsCurrentProcTheBCProc()) diff --git a/Code/lb/iolets/BoundaryCommunicator.cc b/Code/lb/iolets/BoundaryCommunicator.cc index 18cedaf24..1aefa207e 100644 --- a/Code/lb/iolets/BoundaryCommunicator.cc +++ b/Code/lb/iolets/BoundaryCommunicator.cc @@ -13,14 +13,19 @@ namespace hemelb namespace iolets { - BoundaryCommunicator::BoundaryCommunicator(const net::MpiCommunicator& parent) : - MpiCommunicator(parent.Duplicate()) + BoundaryCommunicator::BoundaryCommunicator(comm::Communicator::ConstPtr parent) : + comm(parent->Duplicate()) { } + comm::Communicator::ConstPtr BoundaryCommunicator::GetComm() const + { + return comm; + } + bool BoundaryCommunicator::IsCurrentProcTheBCProc() const { - return Rank() == GetBCProcRank(); + return comm->Rank() == GetBCProcRank(); } int BoundaryCommunicator::GetBCProcRank() const { diff --git a/Code/lb/iolets/BoundaryCommunicator.h b/Code/lb/iolets/BoundaryCommunicator.h index d0f596ab2..d19c6da05 100644 --- a/Code/lb/iolets/BoundaryCommunicator.h +++ b/Code/lb/iolets/BoundaryCommunicator.h @@ -7,7 +7,7 @@ #ifndef HEMELB_LB_IOLETS_BOUNDARYCOMMUNICATOR_H #define HEMELB_LB_IOLETS_BOUNDARYCOMMUNICATOR_H -#include "net/MpiCommunicator.h" +#include "comm/Communicator.h" namespace hemelb { @@ -15,12 +15,16 @@ namespace hemelb { namespace iolets { - class BoundaryCommunicator : public net::MpiCommunicator + class BoundaryCommunicator { public: - BoundaryCommunicator(const net::MpiCommunicator& parent); + BoundaryCommunicator(comm::Communicator::ConstPtr parent); bool IsCurrentProcTheBCProc() const; int GetBCProcRank() const; + comm::Communicator::ConstPtr GetComm() const; + private: + comm::Communicator::Ptr comm; + }; } } diff --git a/Code/lb/iolets/BoundaryValues.cc b/Code/lb/iolets/BoundaryValues.cc index 7a36d029c..12c59a5de 100644 --- a/Code/lb/iolets/BoundaryValues.cc +++ b/Code/lb/iolets/BoundaryValues.cc @@ -21,7 +21,7 @@ namespace hemelb geometry::LatticeData* latticeData, const std::vector &incoming_iolets, SimulationState* simulationState, - const net::MpiCommunicator& comms, + comm::Communicator::ConstPtr comms, const util::UnitConverter& units) : net::IteratedAction(), ioletType(ioletType), totalIoletCount(incoming_iolets.size()), localIoletCount(0), state(simulationState), unitConverter(units), bcComms(comms) @@ -105,8 +105,8 @@ namespace hemelb // For each inlet/outlet there is an array of length equal to total number of procs. // Each stores true/false value. True if proc of rank equal to the index contains // the given inlet/outlet. - - std::vector processorsNeedingIoletFlags = bcComms.Gather(isIOletOnThisProc, bcComms.GetBCProcRank()); + + std::vector processorsNeedingIoletFlags = bcComms.GetComm()->Gather(isIOletOnThisProc, bcComms.GetBCProcRank()); if (bcComms.IsCurrentProcTheBCProc()) { diff --git a/Code/lb/iolets/BoundaryValues.h b/Code/lb/iolets/BoundaryValues.h index 8041ff773..0f4001bda 100644 --- a/Code/lb/iolets/BoundaryValues.h +++ b/Code/lb/iolets/BoundaryValues.h @@ -7,7 +7,7 @@ #ifndef HEMELB_LB_IOLETS_BOUNDARYVALUES_H #define HEMELB_LB_IOLETS_BOUNDARYVALUES_H -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" #include "net/IteratedAction.h" #include "lb/iolets/InOutLet.h" #include "geometry/LatticeData.h" @@ -27,7 +27,7 @@ namespace hemelb geometry::LatticeData* latticeData, const std::vector &iolets, SimulationState* simulationState, - const net::MpiCommunicator& comms, + comm::Communicator::ConstPtr comms, const util::UnitConverter& units); ~BoundaryValues(); diff --git a/Code/lb/iolets/InOutLetCosine.cc b/Code/lb/iolets/InOutLetCosine.cc index 356800166..fdf7a2e34 100644 --- a/Code/lb/iolets/InOutLetCosine.cc +++ b/Code/lb/iolets/InOutLetCosine.cc @@ -6,7 +6,6 @@ #include "lb/iolets/InOutLetCosine.h" #include "configuration/SimConfig.h" -#include "net/IOCommunicator.h" namespace hemelb { diff --git a/Code/lb/iolets/InOutLetMultiscale.cc b/Code/lb/iolets/InOutLetMultiscale.cc index bcfd76af5..18dca2d1b 100644 --- a/Code/lb/iolets/InOutLetMultiscale.cc +++ b/Code/lb/iolets/InOutLetMultiscale.cc @@ -6,7 +6,6 @@ #include "lb/iolets/InOutLetMultiscale.h" #include "configuration/SimConfig.h" -#include "net/IOCommunicator.h" #include "lb/iolets/BoundaryComms.h" #include "lb/iolets/BoundaryValues.h" @@ -123,7 +122,7 @@ namespace hemelb pressure_array[1] = minPressure.GetPayload(); pressure_array[2] = maxPressure.GetPayload(); - net::Net commsNet(bcComms); + net::Net commsNet(bcComms.GetComm()); const std::vector& procList = comms->GetListOfProcs(); //TODO: CHECK + IMPROVE! diff --git a/Code/lb/lb.h b/Code/lb/lb.h index e6dd90a4a..68bceb37f 100644 --- a/Code/lb/lb.h +++ b/Code/lb/lb.h @@ -9,7 +9,7 @@ #include "net/net.h" #include "net/IteratedAction.h" -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" #include "lb/SimulationState.h" #include "lb/iolets/BoundaryValues.h" #include "lb/MacroscopicPropertyCache.h" diff --git a/Code/main.cc b/Code/main.cc index b2bdc0bdb..90d35a53c 100644 --- a/Code/main.cc +++ b/Code/main.cc @@ -4,8 +4,7 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/mpi.h" -#include "net/IOCommunicator.h" +#include "comm/MpiEnvironment.h" #include "configuration/CommandLine.h" #include "SimulationMaster.h" @@ -16,23 +15,22 @@ int main(int argc, char *argv[]) // standard output // Bring up MPI - hemelb::net::MpiEnvironment mpi(argc, argv); + hemelb::comm::MpiEnvironment mpi(argc, argv); hemelb::log::Logger::Init(); try { - hemelb::net::MpiCommunicator commWorld = hemelb::net::MpiCommunicator::World(); - - hemelb::net::IOCommunicator hemelbCommunicator(commWorld); + auto commWorld = mpi.World(); + try { // Parse command line hemelb::configuration::CommandLine options = hemelb::configuration::CommandLine(argc, argv); // Start the debugger (if requested) - hemelb::debug::Debugger::Init(options.GetDebug(), argv[0], commWorld); + hemelb::debug::Debugger::Init(options.GetDebug(), argv[0], commWorld.get()); // Prepare main simulation object... - SimulationMaster master = SimulationMaster(options, hemelbCommunicator); + SimulationMaster master = SimulationMaster(options, commWorld); // ..and run it. master.RunSimulation(); diff --git a/Code/reporting/Timers.h b/Code/reporting/Timers.h index a39936437..cf641d6ea 100644 --- a/Code/reporting/Timers.h +++ b/Code/reporting/Timers.h @@ -127,7 +127,7 @@ namespace hemelb */ static const std::string timerNames[TimersBase::numberOfTimers]; - TimersBase(const comm::Communicator* commPtr) : comms(commPtr), + TimersBase(comm::Communicator::ConstPtr commPtr) : comms(commPtr), timers(numberOfTimers), maxes(numberOfTimers), mins(numberOfTimers), means(numberOfTimers) { } @@ -202,7 +202,7 @@ namespace hemelb void Report(ctemplate::TemplateDictionary& dictionary); private: - const comm::Communicator* comms; + comm::Communicator::ConstPtr comms; std::vector timers; //! The set of timers std::vector maxes; //! Max across processes std::vector mins; //! Min across processes From 536e2153055055aa00fc30cfe0c261d5b6118036 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 20 Oct 2016 10:32:09 +0100 Subject: [PATCH 66/99] sort out lists of requests --- Code/comm/Communicator.h | 4 +- Code/comm/Group.h | 6 -- Code/comm/MapAllToAll.h | 8 +- Code/comm/MpiCommunicator.cc | 5 ++ Code/comm/MpiCommunicator.h | 15 ++-- Code/comm/MpiEnvironment.cc | 4 +- Code/comm/MpiEnvironment.h | 2 +- Code/comm/MpiRequest.cc | 90 ++++++++----------- Code/comm/MpiRequest.h | 30 ++++--- Code/comm/Request.h | 40 ++++----- .../neighbouring/NeighbouringDataManager.cc | 10 ++- .../mixins/pointpoint/CoalescePointPoint.cc | 31 ++++--- .../mixins/pointpoint/CoalescePointPoint.h | 2 +- .../mixins/pointpoint/SeparatedPointPoint.cc | 28 +++--- .../mixins/pointpoint/SeparatedPointPoint.h | 2 +- 15 files changed, 138 insertions(+), 139 deletions(-) diff --git a/Code/comm/Communicator.h b/Code/comm/Communicator.h index 3d36fe9aa..f70f7df71 100644 --- a/Code/comm/Communicator.h +++ b/Code/comm/Communicator.h @@ -23,6 +23,7 @@ namespace hemelb { class Group; class Request; + class RequestList; class MpiFile; // Base class for communicators (MPI, null, and mock) @@ -34,7 +35,7 @@ namespace hemelb /** * Class has virtual methods so should have virtual d'tor. */ - virtual ~Communicator(); + virtual ~Communicator() {}; /** * Returns the local rank on the communicator @@ -82,6 +83,7 @@ namespace hemelb virtual std::shared_ptr OpenFile(const std::string& filename, int mode, const MPI_Info info = MPI_INFO_NULL) const = 0; + virtual std::shared_ptr MakeRequestList() const = 0; virtual void Barrier() const = 0; virtual std::shared_ptr Ibarrier() const = 0; diff --git a/Code/comm/Group.h b/Code/comm/Group.h index 17c507b20..fb9667d8a 100644 --- a/Code/comm/Group.h +++ b/Code/comm/Group.h @@ -19,12 +19,6 @@ namespace hemelb { public: typedef std::shared_ptr Ptr; - /** - * Default c'tor - initialises equivalent to an empty group - * (i.e. MPI_GROUP_NULL) - */ - Group(); - /** * Returns the local rank within the group * @return diff --git a/Code/comm/MapAllToAll.h b/Code/comm/MapAllToAll.h index 698529fa1..7d26c5bb4 100644 --- a/Code/comm/MapAllToAll.h +++ b/Code/comm/MapAllToAll.h @@ -26,7 +26,7 @@ namespace hemelb { receivedVals.clear(); - Request::ReqVec sendReqs; + auto sendReqs = comm->MakeRequestList(); int localRank = comm->Rank(); int nSends = valsToSend.size(); // Is localRank in the map of send destinations? @@ -34,7 +34,7 @@ namespace hemelb // If so, reduce the number of sends by 1. nSends -= 1; // Set up a container for all the Status objs - sendReqs.resize(nSends); + sendReqs->resize(nSends); typename std::map::const_iterator iter = valsToSend.begin(), end = valsToSend.end(); @@ -49,7 +49,7 @@ namespace hemelb else { // Synchronous to ensure we know when this is matched - sendReqs[i] = comm->Issend(val, rank, tag); + sendReqs->set(i, std::move(*comm->Issend(val, rank, tag))); i++; } } @@ -70,7 +70,7 @@ namespace hemelb if (!mySendsMatched) { - if (Request::TestAll(sendReqs)) + if (sendReqs->TestAll()) { mySendsMatched = true; barrierReq = comm->Ibarrier(); diff --git a/Code/comm/MpiCommunicator.cc b/Code/comm/MpiCommunicator.cc index 906e88be1..01c771999 100644 --- a/Code/comm/MpiCommunicator.cc +++ b/Code/comm/MpiCommunicator.cc @@ -105,6 +105,11 @@ namespace hemelb return std::make_shared(shared_from_this(), ans); } + RequestList::Ptr MpiCommunicator::MakeRequestList() const + { + return std::make_shared(); + } + void MpiCommunicator::Barrier() const { HEMELB_MPI_CALL(MPI_Barrier, (*commPtr)); diff --git a/Code/comm/MpiCommunicator.h b/Code/comm/MpiCommunicator.h index 9ed93067e..51ede781b 100644 --- a/Code/comm/MpiCommunicator.h +++ b/Code/comm/MpiCommunicator.h @@ -57,14 +57,15 @@ namespace hemelb * Duplicate the communicator - see MPI_COMM_DUP * @return */ - virtual Communicator::Ptr Duplicate() const; - - virtual std::shared_ptr GetGroup() const; - virtual Communicator::Ptr Create(std::shared_ptr grp) const; - - virtual std::shared_ptr OpenFile(const std::string& filename, int mode, - const MPI_Info info = MPI_INFO_NULL) const; + virtual Communicator::Ptr Duplicate() const; + + virtual std::shared_ptr GetGroup() const; + virtual Communicator::Ptr Create(std::shared_ptr grp) const; + virtual std::shared_ptr OpenFile(const std::string& filename, int mode, + const MPI_Info info = MPI_INFO_NULL) const; + virtual std::shared_ptr MakeRequestList() const; + virtual void Barrier() const; virtual std::shared_ptr Ibarrier() const; diff --git a/Code/comm/MpiEnvironment.cc b/Code/comm/MpiEnvironment.cc index f7a5ea652..1aad4ea73 100644 --- a/Code/comm/MpiEnvironment.cc +++ b/Code/comm/MpiEnvironment.cc @@ -33,9 +33,9 @@ namespace hemelb } } - Communicator* MpiEnvironment::World() + Communicator::Ptr MpiEnvironment::World() { - return new MpiCommunicator(MPI_COMM_WORLD, false); + return std::make_shared(MPI_COMM_WORLD, false); } bool MpiEnvironment::Initialized() diff --git a/Code/comm/MpiEnvironment.h b/Code/comm/MpiEnvironment.h index f430db9e4..80aa89993 100644 --- a/Code/comm/MpiEnvironment.h +++ b/Code/comm/MpiEnvironment.h @@ -38,7 +38,7 @@ namespace hemelb */ ~MpiEnvironment(); - static Communicator* World(); + static Communicator::Ptr World(); /** * Query if MPI is initialised * @return diff --git a/Code/comm/MpiRequest.cc b/Code/comm/MpiRequest.cc index f4bc264fd..f5ccf4907 100644 --- a/Code/comm/MpiRequest.cc +++ b/Code/comm/MpiRequest.cc @@ -32,67 +32,53 @@ namespace hemelb HEMELB_MPI_CALL(MPI_Wait, (&req, MPI_STATUS_IGNORE)); } - void MpiRequest::WaitAll(ReqVec& reqs) - { - size_t n = reqs.size(); - MPI_Request* rawreqs = new MPI_Request[n]; - try - { - for (size_t i = 0; i < n; ++i) - { - rawreqs[i] = reqs[i]; - } - - HEMELB_MPI_CALL( - MPI_Waitall, - (n, rawreqs, MPI_STATUSES_IGNORE) - ); - } - catch (const std::exception& e) - { - delete[] rawreqs; - throw; - } - - delete[] rawreqs; - } - bool MpiRequest::Test() { int flag; HEMELB_MPI_CALL(MPI_Test, (&req, &flag, MPI_STATUS_IGNORE)); return flag; } - - bool MpiRequest::TestAll(ReqVec& reqs) + + size_t MpiRequestList::size() const + { + return reqs.size(); + } + + void MpiRequestList::resize(size_t i) + { + reqs.resize(i); + } + + void MpiRequestList::push_back(Request&& r) + { + MpiRequest&& mr = dynamic_cast(r); + reqs.push_back(mr.req); + mr.req = MPI_REQUEST_NULL; + } + + void MpiRequestList::set(size_t i, Request&& r) { + MpiRequest&& mr = dynamic_cast(r); + reqs[i] = mr.req; + mr.req = MPI_REQUEST_NULL; + } + + void MpiRequestList::WaitAll() + { + HEMELB_MPI_CALL( + MPI_Waitall, + (reqs.size(), reqs.data(), MPI_STATUSES_IGNORE) + ); + } + + bool MpiRequestList::TestAll() { int flag; - size_t n = reqs.size(); - if (n == 0) - return true; - - MPI_Request* rawreqs = new MPI_Request[n]; - try - { - for (size_t i = 0; i < n; ++i) - { - rawreqs[i] = reqs[i]; - } - - HEMELB_MPI_CALL( - MPI_Testall, - (n, rawreqs, &flag, MPI_STATUSES_IGNORE) - ); - } - catch (const std::exception& e) - { - delete[] rawreqs; - throw; - } - - delete[] rawreqs; + HEMELB_MPI_CALL( + MPI_Testall, + (reqs.size(), reqs.data(), &flag, MPI_STATUSES_IGNORE) + ); return flag; } - + } } diff --git a/Code/comm/MpiRequest.h b/Code/comm/MpiRequest.h index 8d2af08fb..a6dcbfbb2 100644 --- a/Code/comm/MpiRequest.h +++ b/Code/comm/MpiRequest.h @@ -23,14 +23,14 @@ namespace hemelb */ class MpiRequest : public Request { - private: - typedef std::vector ReqVec; - // typedef std::vector StatVec; - public: MpiRequest(); MpiRequest(MPI_Request req); - + MpiRequest(MpiRequest&& req); + MpiRequest& operator=(MpiRequest&& req); + MpiRequest(const MpiRequest&) = delete; + MpiRequest& operator=(const MpiRequest&) = delete; + /** * Allow implicit casts to MPI_Request * @return The underlying MPI_Request @@ -42,18 +42,28 @@ namespace hemelb operator bool() const; virtual void Wait(); - // void Wait(MpiStatus& stat); - - static void WaitAll(ReqVec& reqs); - // static void WaitAll(ReqVec& reqs, StatVec& stats); virtual bool Test(); - static bool TestAll(ReqVec& reqs); private: + friend class MpiRequestList; MPI_Request req; }; + class MpiRequestList : public RequestList + { + public: + virtual size_t size() const; + virtual void resize(size_t i); + virtual void push_back(Request&&); + virtual void set(size_t i, Request&&); + + virtual void WaitAll(); + virtual bool TestAll(); + private: + std::vector reqs; + }; + } } #endif // HEMELB_NET_MPIREQUEST_H diff --git a/Code/comm/Request.h b/Code/comm/Request.h index 6b48ded66..909a3e346 100644 --- a/Code/comm/Request.h +++ b/Code/comm/Request.h @@ -16,7 +16,7 @@ namespace hemelb { namespace comm { - // class Status; + class RequestList; /** * */ @@ -24,33 +24,31 @@ namespace hemelb { public: typedef std::shared_ptr Ptr; - typedef std::vector ReqVec; - //typedef std::vector StatVec; - Request(); - //Request(MPI_Request req); - - /** - * Allow implicit casts to MPI_Request - * @return The underlying MPI_Request - */ - // operator MPI_Request() const - // { - // return req; - // } + Request() = default; + Request(Request&& req) = default; + Request& operator=(Request&& req) = default; + Request(const Request&) = delete; + Request& operator=(const Request&) = delete; operator bool() const; virtual void Wait() = 0; - // void Wait(Status& stat); - - static void WaitAll(ReqVec& reqs); - // static void WaitAll(ReqVec& reqs, StatVec& stats); - virtual bool Test() = 0; - static bool TestAll(ReqVec& reqs); - }; + class RequestList + { + public: + typedef std::shared_ptr Ptr; + + virtual size_t size() const = 0; + virtual void resize(size_t i) = 0; + virtual void push_back(Request&&) = 0; + virtual void set(size_t i, Request&&) = 0; + + virtual void WaitAll() = 0; + virtual bool TestAll() = 0; + }; } } #endif // HEMELB_COMM_REQUEST_H diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.cc b/Code/geometry/neighbouring/NeighbouringDataManager.cc index 161f71468..698b7a292 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.cc +++ b/Code/geometry/neighbouring/NeighbouringDataManager.cc @@ -170,14 +170,16 @@ namespace hemelb comm::Communicator::ConstPtr comms = net.GetCommunicator(); comm::MapAllToAll(comms, countOfNeedsIHaveFromEachProc, countOfNeedsOnEachProcFromMe, 1234); - comm::Request::ReqVec requestQueue; + //comm::Request::ReqVec requestQueue; + auto requestQueue = comms->MakeRequestList(); + // Now, for every rank, which I need something from, send the ids of those for (CountMap::const_iterator countIt = countOfNeedsIHaveFromEachProc.begin(); countIt != countOfNeedsIHaveFromEachProc.end(); ++countIt) { int other = countIt->first; - requestQueue.push_back(comms->Isend(needsIHaveFromEachProc[other], other)); + requestQueue->push_back(std::move(*comms->Isend(needsIHaveFromEachProc[other], other))); } // And for every rank, which needs something from me, receive those ids @@ -189,10 +191,10 @@ namespace hemelb int size = countIt->second; IdVec& otherNeeds = needsEachProcHasFromMe[other]; otherNeeds.resize(size); - requestQueue.push_back(comms->Irecv(otherNeeds, other)); + requestQueue->push_back(std::move(*comms->Irecv(otherNeeds, other))); } - comm::Request::WaitAll(requestQueue); + requestQueue->WaitAll(); needsHaveBeenShared = true; } } diff --git a/Code/net/mixins/pointpoint/CoalescePointPoint.cc b/Code/net/mixins/pointpoint/CoalescePointPoint.cc index 661fc8e68..b7c706e9b 100644 --- a/Code/net/mixins/pointpoint/CoalescePointPoint.cc +++ b/Code/net/mixins/pointpoint/CoalescePointPoint.cc @@ -14,9 +14,9 @@ namespace hemelb void CoalescePointPoint::EnsureEnoughRequests(size_t count) { - if (requests.size() < count) + if (requests->size() < count) { - requests.resize(count); + requests->resize(count); } } @@ -30,11 +30,11 @@ namespace hemelb for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); ++it) { - requests[m] = communicator->IrecvImpl(it->second.front().Pointer, - 1, - it->second.Type, - it->first, - 10); + requests->set(m, std::move(*communicator->IrecvImpl(it->second.front().Pointer, + 1, + it->second.Type, + it->first, + 10))); ++m; } @@ -83,14 +83,13 @@ namespace hemelb MPI_Type_size(it->second.Type, &TypeSizeStorage); //DTMP: BytesSent += TypeSizeStorage; //DTMP: - requests[receiveProcessorComms.size() + m] = - communicator->IsendImpl(it->second.front().Pointer, - 1, - it->second.Type, - it->first, - 10); - - ++m; + requests->set(receiveProcessorComms.size() + m, + std::move(*communicator->IsendImpl(it->second.front().Pointer, + 1, + it->second.Type, + it->first, + 10))); + ++m; } } @@ -118,7 +117,7 @@ namespace hemelb void CoalescePointPoint::WaitPointToPoint() { - comm::Request::WaitAll(requests); + requests->WaitAll(); for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); ++it) diff --git a/Code/net/mixins/pointpoint/CoalescePointPoint.h b/Code/net/mixins/pointpoint/CoalescePointPoint.h index 3dd326648..c391ec8f5 100644 --- a/Code/net/mixins/pointpoint/CoalescePointPoint.h +++ b/Code/net/mixins/pointpoint/CoalescePointPoint.h @@ -40,7 +40,7 @@ namespace hemelb // there are enough available. We do this in a way to minimise the number created // on each core, but also to minimise creation / deletion overheads. //std::vector requests; - comm::Request::ReqVec requests; + comm::RequestList::Ptr requests; //std::vector statuses; }; } diff --git a/Code/net/mixins/pointpoint/SeparatedPointPoint.cc b/Code/net/mixins/pointpoint/SeparatedPointPoint.cc index 336176382..4f28b1482 100644 --- a/Code/net/mixins/pointpoint/SeparatedPointPoint.cc +++ b/Code/net/mixins/pointpoint/SeparatedPointPoint.cc @@ -14,9 +14,9 @@ namespace hemelb void SeparatedPointPoint::EnsureEnoughRequests(size_t count) { - if (requests.size() < count) + if (requests->size() < count) { - requests.resize(count); + requests->resize(count); } } @@ -30,11 +30,12 @@ namespace hemelb { for (ProcComms::iterator request = it->second.begin(); request != it->second.end(); request++) { - requests[m] = communicator->IrecvImpl(request->Pointer, - request->Count, - request->Type, - it->first, - 10); + requests->set(m, + std::move(*communicator->IrecvImpl(request->Pointer, + request->Count, + request->Type, + it->first, + 10))); ++m; } } @@ -78,11 +79,12 @@ namespace hemelb { for (ProcComms::iterator request = it->second.begin(); request != it->second.end(); request++) { - requests[count_receives+m] = communicator->IsendImpl(request->Pointer, - request->Count, - request->Type, - it->first, - 10); + requests->set(count_receives+m, + std::move(*communicator->IsendImpl(request->Pointer, + request->Count, + request->Type, + it->first, + 10))); ++m; } } @@ -97,7 +99,7 @@ namespace hemelb void SeparatedPointPoint::WaitPointToPoint() { - comm::Request::WaitAll(requests); + requests->WaitAll(); receiveProcessorComms.clear(); sendProcessorComms.clear(); diff --git a/Code/net/mixins/pointpoint/SeparatedPointPoint.h b/Code/net/mixins/pointpoint/SeparatedPointPoint.h index d68abc369..4125b0934 100644 --- a/Code/net/mixins/pointpoint/SeparatedPointPoint.h +++ b/Code/net/mixins/pointpoint/SeparatedPointPoint.h @@ -40,7 +40,7 @@ namespace hemelb // initialisation and during each iteration). Code using these must make sure // there are enough available. We do this in a way to minimise the number created // on each core, but also to minimise creation / deletion overheads. - comm::Request::ReqVec requests; + comm::RequestList::Ptr requests; unsigned int count_sends; unsigned int count_receives; }; From 860d7053d7c1fad5c9d2078536a578e183ffcabf Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 20 Oct 2016 10:32:26 +0100 Subject: [PATCH 67/99] sort the build cmake --- Code/CMakeLists.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 08d9a55f1..85870a32e 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -146,7 +146,7 @@ include(mpi) if(MPI_STANDARD_VERSION VERSION_LESS 3) message(FATAL_ERROR "Must have MPI 3.0 or greater") endif() -set(HAVE_CONSTCORRECTMPI TRUE) + include(dependencies) #-------------Resources ----------------------- @@ -179,6 +179,7 @@ add_executable(${HEMELB_EXECUTABLE} main.cc ${root_sources}) include_directories(${PROJECT_SOURCE_DIR}) set(package_subdirs configuration + comm extraction reporting steering @@ -224,6 +225,7 @@ if (HEMELB_BUILD_MULTISCALE) add_executable(multiscale_hemelb mainMultiscale.cc ${root_sources}) include_directories(${PROJECT_SOURCE_DIR}) set(package_subdirs + comm configuration extraction reporting From fc6d5a52a4c415cb3985e3353a2fb59cd2972ec7 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 20 Oct 2016 15:17:44 +0100 Subject: [PATCH 68/99] fix debugger --- Code/debug/Debugger.cc | 4 ++-- Code/debug/Debugger.h | 6 +++--- Code/debug/OSX/OsxDebugger.cc | 2 +- Code/debug/OSX/OsxDebugger.h | 2 +- Code/debug/common/ActiveDebugger.cc | 2 +- Code/debug/common/ActiveDebugger.h | 2 +- Code/debug/none/NullDebugger.cc | 2 +- Code/debug/none/NullDebugger.h | 2 +- Code/main.cc | 2 +- Code/net/mpi.h | 19 ------------------- 10 files changed, 12 insertions(+), 31 deletions(-) delete mode 100644 Code/net/mpi.h diff --git a/Code/debug/Debugger.cc b/Code/debug/Debugger.cc index 6611c5183..283d4f105 100644 --- a/Code/debug/Debugger.cc +++ b/Code/debug/Debugger.cc @@ -13,7 +13,7 @@ namespace hemelb namespace debug { - Debugger* Debugger::Init(bool active, const char * const executable, const comm::Communicator* comm) + Debugger* Debugger::Init(bool active, const char * const executable, comm::Communicator::ConstPtr comm) { /* Static member function that implements the singleton pattern. * Use the namespace function PlatformDebuggerFactory to @@ -40,7 +40,7 @@ namespace hemelb // Init static members Debugger* Debugger::singleton = NULL; - Debugger::Debugger(const char* const executable, const comm::Communicator* comm) : + Debugger::Debugger(const char* const executable, comm::Communicator::ConstPtr comm) : mExecutable(executable), mCommunicator(comm) { } diff --git a/Code/debug/Debugger.h b/Code/debug/Debugger.h index 705fa99d9..15839a273 100644 --- a/Code/debug/Debugger.h +++ b/Code/debug/Debugger.h @@ -22,20 +22,20 @@ namespace hemelb */ public: // the singleton pattern - static Debugger* Init(bool active, const char *const, const comm::Communicator* comm); + static Debugger* Init(bool active, const char *const, comm::Communicator::ConstPtr comm); static Debugger* Get(void); virtual void BreakHere(void) = 0; virtual void Print(const char* iFormat, ...) = 0; protected: - Debugger(const char* const executable, const comm::Communicator* comm); + Debugger(const char* const executable, comm::Communicator::ConstPtr comm); virtual ~Debugger(); virtual void Attach() = 0; std::string mExecutable; - const comm::Communicator* mCommunicator; + comm::Communicator::ConstPtr mCommunicator; // Singleton pattern static Debugger* singleton; diff --git a/Code/debug/OSX/OsxDebugger.cc b/Code/debug/OSX/OsxDebugger.cc index 245a6d5e0..709469976 100644 --- a/Code/debug/OSX/OsxDebugger.cc +++ b/Code/debug/OSX/OsxDebugger.cc @@ -12,7 +12,7 @@ namespace hemelb { namespace debug { - OsxDebugger::OsxDebugger(const char* const executable, const comm::Communicator* comm) : + OsxDebugger::OsxDebugger(const char* const executable, comm::Communicator::ConstPtr comm) : ActiveDebugger(executable, comm) { } diff --git a/Code/debug/OSX/OsxDebugger.h b/Code/debug/OSX/OsxDebugger.h index e570bd4b8..494587631 100644 --- a/Code/debug/OSX/OsxDebugger.h +++ b/Code/debug/OSX/OsxDebugger.h @@ -22,7 +22,7 @@ namespace hemelb const std::string GetPlatformScript(void) const; // C'tor... - OsxDebugger(const char* const executable, const comm::Communicator* comm); + OsxDebugger(const char* const executable, comm::Communicator::ConstPtr comm); // ... which the factory function needs to be able to get at. friend class Debugger; diff --git a/Code/debug/common/ActiveDebugger.cc b/Code/debug/common/ActiveDebugger.cc index da9d3e365..7a0c5c3cd 100644 --- a/Code/debug/common/ActiveDebugger.cc +++ b/Code/debug/common/ActiveDebugger.cc @@ -26,7 +26,7 @@ namespace hemelb namespace debug { - ActiveDebugger::ActiveDebugger(const char* const executable, const comm::Communicator* comm) : + ActiveDebugger::ActiveDebugger(const char* const executable, comm::Communicator::ConstPtr comm) : Debugger(executable, comm), mAmAttached(false), mPIds() { } diff --git a/Code/debug/common/ActiveDebugger.h b/Code/debug/common/ActiveDebugger.h index 5aac28404..ab1441965 100644 --- a/Code/debug/common/ActiveDebugger.h +++ b/Code/debug/common/ActiveDebugger.h @@ -33,7 +33,7 @@ namespace hemelb typedef std::vector VoS;// Vector of Strings // C'tor - ActiveDebugger(const char* const executable, const comm::Communicator* comm); + ActiveDebugger(const char* const executable, comm::Communicator::ConstPtr comm); bool mAmAttached;// Indicate attachment state VoI mPIds;// vector of process IDs diff --git a/Code/debug/none/NullDebugger.cc b/Code/debug/none/NullDebugger.cc index 876aa128b..f010d2bcc 100644 --- a/Code/debug/none/NullDebugger.cc +++ b/Code/debug/none/NullDebugger.cc @@ -22,7 +22,7 @@ namespace hemelb { } - NullDebugger::NullDebugger(const char* const executable, const comm::Communicator* comm) : + NullDebugger::NullDebugger(const char* const executable, comm::Communicator::ConstPtr comm) : Debugger(executable, comm) { } diff --git a/Code/debug/none/NullDebugger.h b/Code/debug/none/NullDebugger.h index 21e7ab2b6..8cfeabf73 100644 --- a/Code/debug/none/NullDebugger.h +++ b/Code/debug/none/NullDebugger.h @@ -21,7 +21,7 @@ namespace hemelb protected: void Attach(void); - NullDebugger(const char* const executable, const comm::Communicator* comm); + NullDebugger(const char* const executable, comm::Communicator::ConstPtr); friend class Debugger; }; diff --git a/Code/main.cc b/Code/main.cc index 90d35a53c..ad677fa13 100644 --- a/Code/main.cc +++ b/Code/main.cc @@ -27,7 +27,7 @@ int main(int argc, char *argv[]) hemelb::configuration::CommandLine options = hemelb::configuration::CommandLine(argc, argv); // Start the debugger (if requested) - hemelb::debug::Debugger::Init(options.GetDebug(), argv[0], commWorld.get()); + hemelb::debug::Debugger::Init(options.GetDebug(), argv[0], commWorld); // Prepare main simulation object... SimulationMaster master = SimulationMaster(options, commWorld); diff --git a/Code/net/mpi.h b/Code/net/mpi.h deleted file mode 100644 index 57774c78a..000000000 --- a/Code/net/mpi.h +++ /dev/null @@ -1,19 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MPI_H -#define HEMELB_NET_MPI_H - -#include -#include "net/MpiDataType.h" -#include "net/MpiEnvironment.h" -#include "net/MpiCommunicator.h" -#include "net/MpiGroup.h" -#include "net/MpiStatus.h" -#include "net/MpiRequest.h" -#include "net/MpiError.h" - -#endif // HEMELB_NET_MPI_H From 34e3484173a64d04061f330c84f8ed668264bd9a Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 20 Oct 2016 15:43:26 +0100 Subject: [PATCH 69/99] update the multiscale sim master and add AllGatherV to comms --- Code/comm/Communicator.h | 4 ++ Code/comm/Communicator.hpp | 22 ++++++++++ Code/comm/MpiCommunicator.cc | 10 +++++ Code/comm/MpiCommunicator.h | 2 + Code/multiscale/MultiscaleSimulationMaster.h | 45 +++----------------- 5 files changed, 45 insertions(+), 38 deletions(-) diff --git a/Code/comm/Communicator.h b/Code/comm/Communicator.h index f70f7df71..249a977bf 100644 --- a/Code/comm/Communicator.h +++ b/Code/comm/Communicator.h @@ -121,6 +121,8 @@ namespace hemelb template std::vector GatherV(const std::vector senddata, const std::vector recvcounts, const int root) const; + template + std::vector AllGatherV(const std::vector senddata, const std::vector recvcounts) const; template std::vector AllGather(const T& val) const; @@ -184,6 +186,8 @@ namespace hemelb int root) const = 0; virtual void AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, void* recv, int recvcount, MPI_Datatype recvtype) const = 0; + virtual void AllgathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype) const = 0; virtual void AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, void* recv, int recvcount, MPI_Datatype recvtype) const = 0; virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, diff --git a/Code/comm/Communicator.hpp b/Code/comm/Communicator.hpp index 74c810e7a..21bd1c36f 100644 --- a/Code/comm/Communicator.hpp +++ b/Code/comm/Communicator.hpp @@ -136,6 +136,28 @@ namespace hemelb root); return ans; } + + template + std::vector Communicator::AllGatherV(const std::vector senddata, + const std::vector recvcounts) const + { + const int np = Size(); + const int sendcount = senddata.size(); + std::vector displs; + // Compute the displacements from the counts + displs.resize(np); + int total = 0; + for(size_t i = 0; i < np; ++i) { + displs[i] = total; + total += recvcounts[i]; + } + // set up recv buffer + std::vector ans(total); + + AllgathervImpl(senddata.data(), sendcount, MpiDataType(), + ans.data(), recvcounts.data(), displs.data(), MpiDataType()); + return ans; + } template std::vector Communicator::AllGather(const T& val) const diff --git a/Code/comm/MpiCommunicator.cc b/Code/comm/MpiCommunicator.cc index 01c771999..1e6455672 100644 --- a/Code/comm/MpiCommunicator.cc +++ b/Code/comm/MpiCommunicator.cc @@ -216,6 +216,16 @@ namespace hemelb root, *commPtr) ); } + void MpiCommunicator::AllgathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype) const + { + HEMELB_MPI_CALL( + MPI_Allgatherv, + (sendbuf, sendcount, sendtype, + recvbuf, recvcounts, displs, recvtype, + *commPtr) + ); + } void MpiCommunicator::AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, void* recv, int recvcount, MPI_Datatype recvtype) const { diff --git a/Code/comm/MpiCommunicator.h b/Code/comm/MpiCommunicator.h index 51ede781b..6c1cdcbce 100644 --- a/Code/comm/MpiCommunicator.h +++ b/Code/comm/MpiCommunicator.h @@ -87,6 +87,8 @@ namespace hemelb int root) const; virtual void AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, void* recv, int recvcount, MPI_Datatype recvtype) const; + virtual void AllgathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype) const; virtual void AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, void* recv, int recvcount, MPI_Datatype recvtype) const; virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, diff --git a/Code/multiscale/MultiscaleSimulationMaster.h b/Code/multiscale/MultiscaleSimulationMaster.h index 8325f2b16..8330cc64b 100644 --- a/Code/multiscale/MultiscaleSimulationMaster.h +++ b/Code/multiscale/MultiscaleSimulationMaster.h @@ -23,7 +23,7 @@ namespace hemelb { public: MultiscaleSimulationMaster(hemelb::configuration::CommandLine &options, - const net::IOCommunicator& ioComm, + comm::Communicator::ConstPtr ioComm, Intercommunicator & aintercomms) : SimulationMaster(options, ioComm), intercomms(aintercomms), multiscaleIoletType("inoutlet") @@ -66,7 +66,7 @@ namespace hemelb std::vector GlobalIoletCount; GlobalIoletCount.push_back(inletValues->GetLocalIoletCount()); GlobalIoletCount.push_back(outletValues->GetLocalIoletCount()); - ioComms.Broadcast(GlobalIoletCount, 0); + ioComms->Broadcast(GlobalIoletCount, 0); std::vector > invertedInletBoundaryList(GlobalIoletCount[0]); std::vector > invertedOutletBoundaryList(GlobalIoletCount[1]); @@ -332,49 +332,18 @@ namespace hemelb std::vector > ExchangeAndCompleteInverseBoundaryList( std::vector > inList) { - std::vector > outList; - int *recvSizes = new int[ioComms.Size()]; - int *recvDispls = new int[ioComms.Size()]; + std::vector > outList(inList.size()); /* TODO: ASSUMPTION: * inList.size() is equal everywhere. This is not necessarily the case. * Use an AllReduce MAX and resize inList accordingly to make the remnant * of the code work here for unequal inList sizes. */ - for (unsigned int i = 0; i < inList.size(); i++) { - int sendSize = ((int) inList[i].size()); - site_t *sendList = new site_t[inList[i].size()]; - for (unsigned int j = 0; j < inList[i].size(); j++) - { - sendList[j] = inList[i][j]; - } - HEMELB_MPI_CALL( MPI_Allgather, - ( &sendSize, 1, MPI_INT, recvSizes, 1, MPI_INT, ioComms )); - - int64_t totalSize = 0; - - int np = ioComms.Size(); - int64_t offset = 0; - - for (int j = 0; j < np; j++) - { - totalSize += recvSizes[j]; - recvDispls[j] = offset; - offset += recvSizes[j]; - } - - site_t *recvList = new site_t[totalSize]; //inList[i].size() - - HEMELB_MPI_CALL( MPI_Allgatherv, - ( sendList, inList[i].size(), MPI_LONG_LONG, recvList, recvSizes, recvDispls, MPI_LONG_LONG, ioComms )); - - std::vector subList; - for (int j = 0; j < totalSize; j++) - { - subList.push_back(recvList[j]); - } - outList.push_back(subList); + int sendSize = inList[i].size(); + auto recvSizes = ioComms->AllGather(sendSize); + auto recvList = ioComms->AllGatherV(inList[i], recvSizes); + outList.emplace_back(std::move(recvList)); } return outList; From 6d9648d76de400bac2a8ebf2c3a330964479c28f Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 20 Oct 2016 15:48:15 +0100 Subject: [PATCH 70/99] fix allocation bug in net --- Code/net/mixins/pointpoint/CoalescePointPoint.h | 4 ++-- Code/net/mixins/pointpoint/SeparatedPointPoint.h | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Code/net/mixins/pointpoint/CoalescePointPoint.h b/Code/net/mixins/pointpoint/CoalescePointPoint.h index c391ec8f5..2bd51e31b 100644 --- a/Code/net/mixins/pointpoint/CoalescePointPoint.h +++ b/Code/net/mixins/pointpoint/CoalescePointPoint.h @@ -18,8 +18,8 @@ namespace hemelb { public: - CoalescePointPoint(comm::Communicator::ConstPtr comms) : - BaseNet(comms), StoringNet(comms), sendReceivePrepped(false) + CoalescePointPoint(comm::Communicator::ConstPtr comms) : + BaseNet(comms), StoringNet(comms), sendReceivePrepped(false), requests(comms->MakeRequestList()) { } ~CoalescePointPoint(); diff --git a/Code/net/mixins/pointpoint/SeparatedPointPoint.h b/Code/net/mixins/pointpoint/SeparatedPointPoint.h index 4125b0934..ecf7a3525 100644 --- a/Code/net/mixins/pointpoint/SeparatedPointPoint.h +++ b/Code/net/mixins/pointpoint/SeparatedPointPoint.h @@ -19,8 +19,9 @@ namespace hemelb public: SeparatedPointPoint(comm::Communicator::ConstPtr comms) : - BaseNet(comms), StoringNet(comms), sendReceivePrepped(false), count_sends(0), - count_receives(0) + BaseNet(comms), StoringNet(comms), sendReceivePrepped(false), + requests(comms->MakeRequestList()), count_sends(0), + count_receives(0) { } ~SeparatedPointPoint(); From 93ba9ff89c36d674971621c6c5fc3e3f5451f0ef Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 21 Oct 2016 10:23:05 +0100 Subject: [PATCH 71/99] make comm::RequestList accept shared pointers --- Code/comm/MapAllToAll.h | 2 +- Code/comm/MpiRequest.cc | 18 +++++++++-------- Code/comm/MpiRequest.h | 4 ++-- Code/comm/Request.h | 4 ++-- .../neighbouring/NeighbouringDataManager.cc | 4 ++-- .../mixins/pointpoint/CoalescePointPoint.cc | 20 +++++++++---------- .../mixins/pointpoint/SeparatedPointPoint.cc | 20 +++++++++---------- 7 files changed, 37 insertions(+), 35 deletions(-) diff --git a/Code/comm/MapAllToAll.h b/Code/comm/MapAllToAll.h index 7d26c5bb4..eebe86b68 100644 --- a/Code/comm/MapAllToAll.h +++ b/Code/comm/MapAllToAll.h @@ -49,7 +49,7 @@ namespace hemelb else { // Synchronous to ensure we know when this is matched - sendReqs->set(i, std::move(*comm->Issend(val, rank, tag))); + sendReqs->set(i, comm->Issend(val, rank, tag)); i++; } } diff --git a/Code/comm/MpiRequest.cc b/Code/comm/MpiRequest.cc index f5ccf4907..295a1afd7 100644 --- a/Code/comm/MpiRequest.cc +++ b/Code/comm/MpiRequest.cc @@ -49,17 +49,19 @@ namespace hemelb reqs.resize(i); } - void MpiRequestList::push_back(Request&& r) + void MpiRequestList::push_back(Request::Ptr r) { - MpiRequest&& mr = dynamic_cast(r); - reqs.push_back(mr.req); - mr.req = MPI_REQUEST_NULL; + auto mr = std::dynamic_pointer_cast(r); + //MpiRequest&& mr = dynamic_cast(r); + reqs.push_back(mr->req); + mr->req = MPI_REQUEST_NULL; } - void MpiRequestList::set(size_t i, Request&& r) { - MpiRequest&& mr = dynamic_cast(r); - reqs[i] = mr.req; - mr.req = MPI_REQUEST_NULL; + void MpiRequestList::set(size_t i, Request::Ptr r) { + auto mr = std::dynamic_pointer_cast(r); + //MpiRequest&& mr = dynamic_cast(r); + reqs[i] = mr->req; + mr->req = MPI_REQUEST_NULL; } void MpiRequestList::WaitAll() diff --git a/Code/comm/MpiRequest.h b/Code/comm/MpiRequest.h index a6dcbfbb2..e0314a87d 100644 --- a/Code/comm/MpiRequest.h +++ b/Code/comm/MpiRequest.h @@ -55,8 +55,8 @@ namespace hemelb public: virtual size_t size() const; virtual void resize(size_t i); - virtual void push_back(Request&&); - virtual void set(size_t i, Request&&); + virtual void push_back(Request::Ptr); + virtual void set(size_t i, Request::Ptr); virtual void WaitAll(); virtual bool TestAll(); diff --git a/Code/comm/Request.h b/Code/comm/Request.h index 909a3e346..a50152af5 100644 --- a/Code/comm/Request.h +++ b/Code/comm/Request.h @@ -43,8 +43,8 @@ namespace hemelb virtual size_t size() const = 0; virtual void resize(size_t i) = 0; - virtual void push_back(Request&&) = 0; - virtual void set(size_t i, Request&&) = 0; + virtual void push_back(Request::Ptr) = 0; + virtual void set(size_t i, Request::Ptr) = 0; virtual void WaitAll() = 0; virtual bool TestAll() = 0; diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.cc b/Code/geometry/neighbouring/NeighbouringDataManager.cc index 698b7a292..ac20359db 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.cc +++ b/Code/geometry/neighbouring/NeighbouringDataManager.cc @@ -179,7 +179,7 @@ namespace hemelb ++countIt) { int other = countIt->first; - requestQueue->push_back(std::move(*comms->Isend(needsIHaveFromEachProc[other], other))); + requestQueue->push_back(comms->Isend(needsIHaveFromEachProc[other], other)); } // And for every rank, which needs something from me, receive those ids @@ -191,7 +191,7 @@ namespace hemelb int size = countIt->second; IdVec& otherNeeds = needsEachProcHasFromMe[other]; otherNeeds.resize(size); - requestQueue->push_back(std::move(*comms->Irecv(otherNeeds, other))); + requestQueue->push_back(comms->Irecv(otherNeeds, other)); } requestQueue->WaitAll(); diff --git a/Code/net/mixins/pointpoint/CoalescePointPoint.cc b/Code/net/mixins/pointpoint/CoalescePointPoint.cc index b7c706e9b..632e69d8a 100644 --- a/Code/net/mixins/pointpoint/CoalescePointPoint.cc +++ b/Code/net/mixins/pointpoint/CoalescePointPoint.cc @@ -30,11 +30,11 @@ namespace hemelb for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); ++it) { - requests->set(m, std::move(*communicator->IrecvImpl(it->second.front().Pointer, - 1, - it->second.Type, - it->first, - 10))); + requests->set(m, communicator->IrecvImpl(it->second.front().Pointer, + 1, + it->second.Type, + it->first, + 10)); ++m; } @@ -84,11 +84,11 @@ namespace hemelb BytesSent += TypeSizeStorage; //DTMP: requests->set(receiveProcessorComms.size() + m, - std::move(*communicator->IsendImpl(it->second.front().Pointer, - 1, - it->second.Type, - it->first, - 10))); + communicator->IsendImpl(it->second.front().Pointer, + 1, + it->second.Type, + it->first, + 10)); ++m; } } diff --git a/Code/net/mixins/pointpoint/SeparatedPointPoint.cc b/Code/net/mixins/pointpoint/SeparatedPointPoint.cc index 4f28b1482..2d2dd433d 100644 --- a/Code/net/mixins/pointpoint/SeparatedPointPoint.cc +++ b/Code/net/mixins/pointpoint/SeparatedPointPoint.cc @@ -31,11 +31,11 @@ namespace hemelb for (ProcComms::iterator request = it->second.begin(); request != it->second.end(); request++) { requests->set(m, - std::move(*communicator->IrecvImpl(request->Pointer, - request->Count, - request->Type, - it->first, - 10))); + communicator->IrecvImpl(request->Pointer, + request->Count, + request->Type, + it->first, + 10)); ++m; } } @@ -80,11 +80,11 @@ namespace hemelb for (ProcComms::iterator request = it->second.begin(); request != it->second.end(); request++) { requests->set(count_receives+m, - std::move(*communicator->IsendImpl(request->Pointer, - request->Count, - request->Type, - it->first, - 10))); + communicator->IsendImpl(request->Pointer, + request->Count, + request->Type, + it->first, + 10)); ++m; } } From 357ae7be4f6545f4c53b51f3e94a0cec8fe95e81 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Sun, 23 Oct 2016 11:39:26 +0100 Subject: [PATCH 72/99] update the unit tests --- Code/unittests/FourCubeLatticeData.h | 4 +- Code/unittests/geometry/NeedsTests.h | 144 ++++++-------- .../NeighbouringDataManagerTests.h | 6 +- .../helpers/FourCubeBasedTestFixture.h | 1 - Code/unittests/helpers/HasCommsTestFixture.h | 14 +- Code/unittests/helpers/MockNetHelper.h | 186 ++++++++++++++---- Code/unittests/main.cc | 6 +- .../multiscale/MockIntercommunicator.h | 3 +- Code/unittests/net/MpiTests.h | 32 +-- Code/unittests/net/NetMock.h | 8 +- Code/unittests/net/RecordingNet.h | 7 +- Code/unittests/reporting/Mocks.h | 4 +- Code/unittests/reporting/ReporterTests.h | 6 +- Code/unittests/reporting/TimerTests.h | 10 +- 14 files changed, 241 insertions(+), 190 deletions(-) diff --git a/Code/unittests/FourCubeLatticeData.h b/Code/unittests/FourCubeLatticeData.h index 3bf2353a3..854a09f0d 100644 --- a/Code/unittests/FourCubeLatticeData.h +++ b/Code/unittests/FourCubeLatticeData.h @@ -58,7 +58,7 @@ namespace hemelb * * @return */ - static FourCubeLatticeData* Create(const net::IOCommunicator& comm, site_t sitesPerBlockUnit = 6, proc_t rankCount = 1) + static FourCubeLatticeData* Create(comm::Communicator::ConstPtr comm, site_t sitesPerBlockUnit = 6, proc_t rankCount = 1) { hemelb::geometry::Geometry readResult(util::Vector3D::Ones(), sitesPerBlockUnit); @@ -228,7 +228,7 @@ namespace hemelb } protected: - FourCubeLatticeData(hemelb::geometry::Geometry& readResult, const net::IOCommunicator& comms) : + FourCubeLatticeData(hemelb::geometry::Geometry& readResult, comm::Communicator::ConstPtr comms) : hemelb::geometry::LatticeData(lb::lattices::D3Q15::GetLatticeInfo(), readResult, comms) { diff --git a/Code/unittests/geometry/NeedsTests.h b/Code/unittests/geometry/NeedsTests.h index 1005aba0e..cc6b98c06 100644 --- a/Code/unittests/geometry/NeedsTests.h +++ b/Code/unittests/geometry/NeedsTests.h @@ -46,73 +46,31 @@ namespace hemelb SetupMocks(6, 2, 5, 0); CPPUNIT_ASSERT_EQUAL(communicatorMock->Size(),5); // Start to record the expected communications calls. - // First will come, sending to the reading cores, each of the lengths. - // I would expect to send to the other reading core, my count of needed cores - int core_0_requires_from_0_count = 1; - int core_0_requires_from_1_count = 1; - - netMock->RequireSend(&core_0_requires_from_0_count, 1, 0, "Count"); - netMock->RequireSend(&core_0_requires_from_1_count, 1, 1, "Count"); - - // And I would expect the reading core to post a receive from each of the other cores, - // asking for its count of needed blocks from this reading core - int core_0_requires_count = 1; - int core_1_requires_count = 2; - int core_2_requires_count = 1; - int core_3_requires_count = 2; - int core_4_requires_count = 1; - - netMock->RequireReceive(&core_0_requires_count, 1, 0, "Count"); - netMock->RequireReceive(&core_1_requires_count, 1, 1, "Count"); - netMock->RequireReceive(&core_2_requires_count, 1, 2, "Count"); - netMock->RequireReceive(&core_3_requires_count, 1, 3, "Count"); - netMock->RequireReceive(&core_4_requires_count, 1, 4, "Count"); - - // Then, I would expect to send to myself, my needs - std::vector core_0_requires_from_0; - core_0_requires_from_0.push_back(0); - netMock->RequireSend(&core_0_requires_from_0[0], 1, 0, "Needs"); - - // Then, I would expect to send to the other reading core, my needs - std::vector core_0_requires_from_1; - core_0_requires_from_1.push_back(1); - netMock->RequireSend(&core_0_requires_from_1[0], 1, 1, "Needs"); - - // Then, I should receieve from myself, my own requirements - // Then, I would expect to receive the lists of needed blocks themselves. - // Core 0, I expect it to need blocks 0,1 - std::vector core_0_requires; - core_0_requires.push_back(0); - netMock->RequireReceive(&core_0_requires[0], 1, 0, "Needs"); - - // Then, I would expect to receive the lists of needed blocks themselves. - // From core 1, the other reading core, I expect it to need blocks 0,1,2 - std::vector core_1_requires; - core_1_requires.push_back(0); - - core_1_requires.push_back(2); - netMock->RequireReceive(&core_1_requires[0], 2, 1, "Needs"); - // From core 2, I expect it to need blocks 1,2,3 - std::vector core_2_requires; - - core_2_requires.push_back(2); - - netMock->RequireReceive(&core_2_requires[0], 1, 2, "Needs"); - // Then, I would expect to receive the lists of needed blocks themselves. - // From core 3, I expect it to need blocks 2,3,4 - std::vector core_3_requires; - core_3_requires.push_back(2); - - core_3_requires.push_back(4); - netMock->RequireReceive(&core_3_requires[0], 2, 3, "Needs"); - // Then, I would expect to receive the lists of needed blocks themselves. - // From core 4, I expect it to need blocks 3,4,5 - std::vector core_4_requires; - - core_4_requires.push_back(4); - - netMock->RequireReceive(&core_4_requires[0], 1, 4, "Needs"); + // First we gather the counts of blocks needed on each reading core + auto mock = std::dynamic_pointer_cast(communicatorMock); + const std::vector block_size_reading_core_0 = {1, 2, 1, 2, 1}; + // We don't really care about what the other ranks need from RC1 + const std::vector block_size_reading_core_1 = {1, 0, 2, 0, 0}; + + mock->AddGatherResult(block_size_reading_core_0); + mock->AddGatherResult(block_size_reading_core_1); + + // Now we list the blocks needed from reading core 0 (us) + // Note counts match above + const std::vector blocks_needed_from_reading_core_0 = { + 0, + 0, 2, + 2, + 2, 4, + 4 + }; + mock->AddGatherVResult(blocks_needed_from_reading_core_0); + // Blocks needed from RC1 + const std::vector blocks_needed_from_reading_core_1 = {1}; + mock->AddGatherVResult(blocks_needed_from_reading_core_1); + ShareMockNeeds(); + // Finally, I would expect the resulting array of needs on core one to be as planned: std::vector needing_block_0; needing_block_0.push_back(0); @@ -136,34 +94,42 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(needing_block_3, mockedNeeds->ProcessorsNeedingBlock(3)); CPPUNIT_ASSERT_EQUAL(needing_block_4, mockedNeeds->ProcessorsNeedingBlock(4)); CPPUNIT_ASSERT_EQUAL(needing_block_5, mockedNeeds->ProcessorsNeedingBlock(5)); - + // I guess this means that no comms went through the net netMock->ExpectationsAllCompleted(); } void TestNonReading() { + // We are core 2 - a no reading one SetupMocks(6, 2, 5, 2); - - // Start to record the expected communications calls. - // First will come, sending to the reading cores, each of the lengths. - // So I would expect the non-reading core to post a send to each of the reading cores, its count of needed blocks - int core_2_requires_from_0_count = 1; - int core_2_requires_from_1_count = 2; - - netMock->RequireSend(&core_2_requires_from_0_count, 1, 0); - netMock->RequireSend(&core_2_requires_from_1_count, 1, 1); - - // Then, I would expect to send my list of needed blocks - // From core 2, the other reading core, I expect it to need blocks 1,2,3 - std::vector core_2_requires_from_0; - std::vector core_2_requires_from_1; - - core_2_requires_from_1.push_back(1); - core_2_requires_from_0.push_back(2); - core_2_requires_from_1.push_back(3); - - netMock->RequireSend(&core_2_requires_from_0[0], 1, 0); - netMock->RequireSend(&core_2_requires_from_1[0], 2, 1); + + CPPUNIT_ASSERT_EQUAL(communicatorMock->Size(),5); + + // Record the expected communications calls. + auto mock = std::dynamic_pointer_cast(communicatorMock); + + // First we gather the counts of blocks needed on each reading core + // For RC0 we know this from above + const std::vector block_size_reading_core_0 = {1, 2, 1, 2, 1}; + // We don't really care about what the other ranks need from RC1 + const std::vector block_size_reading_core_1 = {0, 0, 2, 0, 0}; + + mock->AddGatherResult(block_size_reading_core_0); + mock->AddGatherResult(block_size_reading_core_1); + + // Now we list the blocks needed from RC0 + // Note counts match above + const std::vector blocks_needed_from_reading_core_0 = { + 0, + 0, 2, + 2, + 2, 4, + 4 + }; + mock->AddGatherVResult(blocks_needed_from_reading_core_0); + // Blocks needed from RC1 + const std::vector blocks_needed_from_reading_core_1 = {1 ,3}; + mock->AddGatherVResult(blocks_needed_from_reading_core_1); ShareMockNeeds(); // Finally, I would expect the resulting array of needs to be empty @@ -204,7 +170,7 @@ namespace hemelb mockedNeeds = new Needs(blockCount, inputNeededBlocks, readingCores, - *netMock, + communicatorMock, false); } diff --git a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h index 7e1316469..77a8c5171 100644 --- a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h +++ b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h @@ -10,6 +10,7 @@ #include "net/phased/NetConcern.h" #include "geometry/neighbouring/NeighbouringDataManager.h" #include "unittests/helpers/MockNetHelper.h" +#include "comm/MpiEnvironment.h" namespace hemelb { @@ -51,10 +52,9 @@ namespace hemelb { delete manager; delete netMock; - delete communicatorMock; - communicatorMock = new net::MpiCommunicator(net::MpiCommunicator::World()); - netMock = new net::NetMock(*communicatorMock); + communicatorMock = comm::MpiEnvironment::World(); + netMock = new net::NetMock(communicatorMock); manager = new NeighbouringDataManager(*latDat, *data, *netMock); } diff --git a/Code/unittests/helpers/FourCubeBasedTestFixture.h b/Code/unittests/helpers/FourCubeBasedTestFixture.h index 992a110cf..08cbd5987 100644 --- a/Code/unittests/helpers/FourCubeBasedTestFixture.h +++ b/Code/unittests/helpers/FourCubeBasedTestFixture.h @@ -10,7 +10,6 @@ #include "configuration/SimConfig.h" #include "lb/collisions/Collisions.h" #include "lb/SimulationState.h" -#include "net/IOCommunicator.h" #include "unittests/FourCubeLatticeData.h" #include "unittests/OneInOneOutSimConfig.h" #include "unittests/helpers/FolderTestFixture.h" diff --git a/Code/unittests/helpers/HasCommsTestFixture.h b/Code/unittests/helpers/HasCommsTestFixture.h index 1e2b5f0f3..27e34cd84 100644 --- a/Code/unittests/helpers/HasCommsTestFixture.h +++ b/Code/unittests/helpers/HasCommsTestFixture.h @@ -7,7 +7,7 @@ #ifndef HEMELB_UNITTESTS_HELPERS_HASCOMMSTESTFIXTURE_H #define HEMELB_UNITTESTS_HELPERS_HASCOMMSTESTFIXTURE_H #include -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" namespace hemelb { @@ -29,21 +29,21 @@ namespace hemelb //hemelbCommunicator = NULL; } - static void Init(const net::IOCommunicator& inst) + static void Init(comm::Communicator::ConstPtr inst) { - hemelbCommunicator = &inst; + hemelbCommunicator = inst; } protected: - static const net::IOCommunicator& Comms() + static comm::Communicator::ConstPtr Comms() { - return *hemelbCommunicator; + return hemelbCommunicator; } private: - static const net::IOCommunicator* hemelbCommunicator; + static comm::Communicator::ConstPtr hemelbCommunicator; }; - const net::IOCommunicator* HasCommsTestFixture::hemelbCommunicator = NULL; + comm::Communicator::ConstPtr HasCommsTestFixture::hemelbCommunicator = NULL; } } } diff --git a/Code/unittests/helpers/MockNetHelper.h b/Code/unittests/helpers/MockNetHelper.h index ccba5f125..1b6ca4e1f 100644 --- a/Code/unittests/helpers/MockNetHelper.h +++ b/Code/unittests/helpers/MockNetHelper.h @@ -8,59 +8,167 @@ #define HEMELB_UNITTESTS_HELPERS_MOCKNETHELPER_H #include "unittests/net/NetMock.h" +#include + namespace hemelb { namespace unittests { namespace helpers { - class MockMpiCommunicator : public net::MpiCommunicator + using namespace comm; + + class MockMpiCommunicator : public comm::Communicator { - public: - /*** - * Constructor for a dummy communicator - * Can be useful for testing but can't actually be used - * @param rank - * @param size - */ - MockMpiCommunicator(int rank_, int size_) : - MpiCommunicator(), rank(rank_), size(size_) - { + public: + /*** + * Constructor for a dummy communicator + * Can be useful for testing but can't actually be used + * @param rank + * @param size + */ + MockMpiCommunicator(int rank_, int size_) : + rank(rank_), size(size_) + { + + } + + virtual inline int Rank() const + { + return rank; + } + virtual inline int Size() const + { + return size; + } + +#define NOTIMPLEMENTED CPPUNIT_FAIL("Communicator function not implemented") + + virtual void Abort(int errCode) const { NOTIMPLEMENTED; } + + virtual Ptr Duplicate() const { NOTIMPLEMENTED; } + + virtual std::shared_ptr GetGroup() const { NOTIMPLEMENTED; } + virtual Ptr Create(std::shared_ptr grp) const { NOTIMPLEMENTED; } + virtual std::shared_ptr OpenFile(const std::string& filename, int mode, + const MPI_Info info = MPI_INFO_NULL) const { NOTIMPLEMENTED; } + + virtual std::shared_ptr MakeRequestList() const { NOTIMPLEMENTED; } + virtual void Barrier() const { NOTIMPLEMENTED; } + virtual std::shared_ptr Ibarrier() const { NOTIMPLEMENTED; } - } + virtual bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const { NOTIMPLEMENTED; } + template + void AddGatherResult(const std::vector& data) + { + CPPUNIT_ASSERT_EQUAL_MESSAGE("Number of elements must be a multiple of comm size", + 0, int(data.size()) % Size()); + gather_data.emplace_back(data.size() / Size(), sizeof(T), data.data()); + } + + template + void AddGatherVResult(const std::vector& data) + { + gatherv_data.emplace_back(data.size(), sizeof(T), data.data()); + } + private: + int rank, size; + typedef std::tuple gather_info; + mutable std::deque gather_data; + mutable std::deque gatherv_data; + + virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const { NOTIMPLEMENTED; } + virtual std::shared_ptr IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const { NOTIMPLEMENTED; } + virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const { NOTIMPLEMENTED; } + virtual std::shared_ptr IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const { NOTIMPLEMENTED; } + virtual std::shared_ptr IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const { NOTIMPLEMENTED; } + virtual void ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const { NOTIMPLEMENTED; } + + virtual void GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype, + int root) const { + CPPUNIT_ASSERT(gather_data.size()); + int nEl, elSize; + const void* ptr; + std::tie(nEl, elSize, ptr) = gather_data.front(); + CPPUNIT_ASSERT_EQUAL(nEl, sendcount); + + { // assert that send[:] == stored_data[sendcount*rank:sendcount*(rank+1)] + int size_bytes = nEl*elSize; + const char* expected = static_cast(ptr) + Rank()*size_bytes; + const char* actual = static_cast(send); + for (auto i = 0; i < size_bytes; ++i) + CPPUNIT_ASSERT_EQUAL(expected[i], actual[i]); + } + + if (Rank() == root) { + std::memcpy(recv, ptr, nEl*elSize*Size()); + } + gather_data.pop_front(); + } + + virtual void GathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype, + int root) const { + CPPUNIT_ASSERT(gatherv_data.size()); + int nElTotal, elSize; + const void* ptr; + std::tie(nElTotal, elSize, ptr) = gatherv_data.front(); + if (Rank() == root) + CPPUNIT_ASSERT_EQUAL_MESSAGE("This rank's count must match", + recvcounts[Rank()], sendcount); - virtual inline int Rank() const - { - return rank; - } - virtual inline int Size() const - { - return size; - } - private: - int rank, size; + if (root == Rank()) { + // sum(recvcounts) == nElTotal (only significant on root of collective) + int sum_recvcounts = std::accumulate(recvcounts, recvcounts + Size(), 0); + CPPUNIT_ASSERT_EQUAL(nElTotal, sum_recvcounts); + } + // assert that send[:] == stored_data[sendcount*rank:sendcount*(rank+1)] + if (Rank() == root) { + std::memcpy(recvbuf, ptr, nElTotal*elSize); + } + gatherv_data.pop_front(); + } + + virtual void AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } + virtual void AllgathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } + virtual void AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } + virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { NOTIMPLEMENTED; } + virtual void SsendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { NOTIMPLEMENTED; } + virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int src, int tag, MPI_Status* stat) const { NOTIMPLEMENTED; } + virtual std::shared_ptr IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { NOTIMPLEMENTED; } + virtual std::shared_ptr IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { NOTIMPLEMENTED; } + virtual std::shared_ptr IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int tag) const { NOTIMPLEMENTED; } }; class MockNetHelper { - protected: - MockNetHelper() : - communicatorMock(NULL), netMock(NULL) - { - } - void setUp(const proc_t core_count, const proc_t current_core) - { - communicatorMock = new MockMpiCommunicator(current_core, core_count); - netMock = new net::NetMock(*communicatorMock); - } - void tearDown() - { - delete communicatorMock; - delete netMock; - } + protected: + MockNetHelper() : + communicatorMock(NULL), netMock(NULL) + { + } + void setUp(const proc_t core_count, const proc_t current_core) + { + communicatorMock = std::make_shared(current_core, core_count); + netMock = new net::NetMock(communicatorMock); + } + void tearDown() + { + delete netMock; + } - net::MpiCommunicator *communicatorMock; - net::NetMock *netMock; + comm::Communicator::Ptr communicatorMock; + net::NetMock *netMock; }; } diff --git a/Code/unittests/main.cc b/Code/unittests/main.cc index 3c6c3bb04..63bafec7f 100644 --- a/Code/unittests/main.cc +++ b/Code/unittests/main.cc @@ -32,10 +32,10 @@ int main(int argc, char **argv) { // Start MPI and the logger. - hemelb::net::MpiEnvironment mpi(argc, argv); + hemelb::comm::MpiEnvironment mpi(argc, argv); hemelb::log::Logger::Init(); - hemelb::net::MpiCommunicator commWorld = hemelb::net::MpiCommunicator::World(); + hemelb::comm::Communicator::Ptr commWorld = mpi.World(); // Read options std::ostream * reportto = &std::cerr; @@ -59,7 +59,7 @@ int main(int argc, char **argv) hemelb::debug::Debugger::Init(debug, argv[0], commWorld); // Initialise the global IOCommunicator. - hemelb::net::IOCommunicator testCommunicator(commWorld); + hemelb::comm::Communicator::Ptr testCommunicator = commWorld; hemelb::unittests::helpers::HasCommsTestFixture::Init(testCommunicator); std::string testPath = (optind < argc) diff --git a/Code/unittests/multiscale/MockIntercommunicator.h b/Code/unittests/multiscale/MockIntercommunicator.h index 9945e93c5..6f4b12411 100644 --- a/Code/unittests/multiscale/MockIntercommunicator.h +++ b/Code/unittests/multiscale/MockIntercommunicator.h @@ -13,7 +13,6 @@ #include #include "multiscale/Intercommunicator.h" -#include "net/mpi.h" namespace hemelb { @@ -30,7 +29,7 @@ namespace hemelb typedef MPI_Datatype RuntimeType; template static RuntimeType GetType() { - return net::MpiDataTypeTraits::GetMpiDataType(); + return comm::MpiDataTypeTraits::GetMpiDataType(); } }; diff --git a/Code/unittests/net/MpiTests.h b/Code/unittests/net/MpiTests.h index e2dbc0321..49192ceac 100644 --- a/Code/unittests/net/MpiTests.h +++ b/Code/unittests/net/MpiTests.h @@ -8,15 +8,16 @@ #define HEMELB_UNITTESTS_NET_MPITESTS_H #include -#include "net/mpi.h" +#include "comm/MpiCommunicator.h" +#include "comm/MpiEnvironment.h" namespace hemelb { namespace unittests { - namespace net + namespace comm { - using namespace hemelb::net; + using namespace hemelb::comm; /** * Unittests of the MPI abstraction layer * @@ -32,31 +33,12 @@ namespace hemelb void TestMpiComm() { MpiCommunicator commNull; - CPPUNIT_ASSERT(!commNull); + //CPPUNIT_ASSERT(!commNull); - MpiCommunicator commWorld = MpiCommunicator::World(); + Communicator::Ptr commWorld = MpiEnvironment::World(); CPPUNIT_ASSERT(commWorld); - CPPUNIT_ASSERT(commNull== commNull); - CPPUNIT_ASSERT(commWorld == commWorld); - CPPUNIT_ASSERT(commWorld != commNull); - - { - MpiCommunicator commWorld2 = commWorld; - CPPUNIT_ASSERT(commWorld2 == commWorld); - } - - { - MpiCommunicator commWorld2 = MpiCommunicator::World(); - CPPUNIT_ASSERT(commWorld2 == commWorld); - } - { - MpiGroup groupWorld = commWorld.Group(); - MpiCommunicator commWorld2 = commWorld.Create(groupWorld); - // Same ranks, but different context. - CPPUNIT_ASSERT(commWorld2 != commWorld); - } - } + } }; CPPUNIT_TEST_SUITE_REGISTRATION (MpiTests); } diff --git a/Code/unittests/net/NetMock.h b/Code/unittests/net/NetMock.h index ecf5f0d65..e4c29d966 100644 --- a/Code/unittests/net/NetMock.h +++ b/Code/unittests/net/NetMock.h @@ -12,7 +12,6 @@ #include #include "constants.h" -#include "net/mpi.h" #include "net/net.h" #include "unittests/net/RecordingNet.h" @@ -24,13 +23,12 @@ namespace hemelb { using namespace hemelb::net; - class NetMock : public InterfaceDelegationNet,public RecordingNet, - public ViaPointPointGathers + class NetMock : public InterfaceDelegationNet,public RecordingNet { public: - NetMock(net::MpiCommunicator & communicator) : + NetMock(comm::Communicator::ConstPtr communicator) : BaseNet(communicator), StoringNet(communicator), InterfaceDelegationNet(communicator), - RecordingNet(communicator), ViaPointPointGathers(communicator) + RecordingNet(communicator) { } diff --git a/Code/unittests/net/RecordingNet.h b/Code/unittests/net/RecordingNet.h index 1c24f2bac..d3c7921f7 100644 --- a/Code/unittests/net/RecordingNet.h +++ b/Code/unittests/net/RecordingNet.h @@ -12,7 +12,6 @@ #include #include "constants.h" -#include "net/mpi.h" #include "net/net.h" #include "unittests/net/LabelledRequest.h" @@ -41,7 +40,7 @@ namespace hemelb class RecordingNet : public virtual StoringNet { public: - RecordingNet(const MpiCommunicator& comms) : + RecordingNet(comm::Communicator::ConstPtr comms) : BaseNet(comms), StoringNet(comms), requiredReceipts(), requiredSends() { } @@ -58,7 +57,7 @@ namespace hemelb { requiredReceipts[rank].push_back(LabelledRequest(pointer, count, - MpiDataType(), + comm::MpiDataType(), rank, label)); } @@ -74,7 +73,7 @@ namespace hemelb { requiredSends[rank].push_back(LabelledRequest(pointer, count, - MpiDataType(), + comm::MpiDataType(), rank, label)); } diff --git a/Code/unittests/reporting/Mocks.h b/Code/unittests/reporting/Mocks.h index 73cfd89e3..cf13365b0 100644 --- a/Code/unittests/reporting/Mocks.h +++ b/Code/unittests/reporting/Mocks.h @@ -30,7 +30,7 @@ namespace hemelb double fakeTime; }; - class MPICommsMock + /* class MPICommsMock { public: MPICommsMock(const net::IOCommunicator& ignored) : @@ -61,7 +61,7 @@ namespace hemelb } private: unsigned int calls; - }; + };*/ } } } diff --git a/Code/unittests/reporting/ReporterTests.h b/Code/unittests/reporting/ReporterTests.h index 44f13f88f..19d14f46b 100644 --- a/Code/unittests/reporting/ReporterTests.h +++ b/Code/unittests/reporting/ReporterTests.h @@ -24,7 +24,7 @@ namespace hemelb { using namespace hemelb::reporting; - typedef TimersBase TimersMock; + typedef TimersBase TimersMock; typedef lb::IncompressibilityChecker IncompressibilityCheckerMock; class ReporterTests : public helpers::HasCommsTestFixture @@ -131,8 +131,8 @@ namespace hemelb expectation << std::setprecision(3); for (unsigned int row = 0; row < Timers::numberOfTimers; row++) { - expectation << "N" << TimersMock::timerNames[row] << "L" << row * 10.0 << "MI" << row * 15.0 << "ME" - << row * 2.0 << "MA" << row * 5.0 << " " << std::flush; + expectation << "N" << TimersMock::timerNames[row] << "L" << row * 10.0 << "MI" << row * 10.0 << "ME" + << row * 10.0 << "MA" << row * 10.0 << " " << std::flush; } AssertTemplate(expectation.str(), "{{#TIMER}}N{{NAME}}L{{LOCAL}}MI{{MIN}}ME{{MEAN}}MA{{MAX}} {{/TIMER}}"); } diff --git a/Code/unittests/reporting/TimerTests.h b/Code/unittests/reporting/TimerTests.h index 82142f807..8356a4a7d 100644 --- a/Code/unittests/reporting/TimerTests.h +++ b/Code/unittests/reporting/TimerTests.h @@ -83,7 +83,7 @@ namespace hemelb public: void setUp() { - timers = new TimersBase(Comms()); + timers = new TimersBase(Comms()); } void tearDown() @@ -130,14 +130,14 @@ namespace hemelb timers->Reduce(); // trigger the mock for (unsigned int i = 0; i < Timers::numberOfTimers; i++) { - CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 5.0, timers->Maxes()[i], 1e-6); - CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 2.0, timers->Means()[i], 1e-6); - CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 15.0, timers->Mins()[i], 1e-6); + CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 10.0, timers->Maxes()[i], 1e-6); + CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 10.0, timers->Means()[i], 1e-6); + CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 10.0, timers->Mins()[i], 1e-6); } } private: - TimersBase *timers; + TimersBase *timers; }; CPPUNIT_TEST_SUITE_REGISTRATION(TimersTests); From ef5d20b2399f61177fbf2c10e0fe04bd1956e369 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Sun, 23 Oct 2016 20:56:07 +0100 Subject: [PATCH 73/99] move colloids and some geometry away from the net --- Code/colloids/Particle.h | 2 +- Code/colloids/ParticleSet.cc | 70 +++--- Code/colloids/ParticleSet.h | 2 - Code/comm/Async.h | 61 ++++++ Code/geometry/GeometryReader.cc | 64 +++--- .../decomposition/OptimisedDecomposition.cc | 200 +++++++++--------- 6 files changed, 234 insertions(+), 165 deletions(-) create mode 100644 Code/comm/Async.h diff --git a/Code/colloids/Particle.h b/Code/colloids/Particle.h index b58eb72da..1e9468679 100644 --- a/Code/colloids/Particle.h +++ b/Code/colloids/Particle.h @@ -27,7 +27,7 @@ namespace hemelb * all persisted properties, i.e. those that are read in from a config file, * are inherited from the PersistedParticle class (which handles the I/O) */ - class Particle : PersistedParticle + class Particle : public PersistedParticle { public: /** constructor - gets initial values from an xml configuration file */ diff --git a/Code/colloids/ParticleSet.cc b/Code/colloids/ParticleSet.cc index e9fdcd1c9..cc9700e35 100644 --- a/Code/colloids/ParticleSet.cc +++ b/Code/colloids/ParticleSet.cc @@ -12,6 +12,7 @@ #include "io/writers/xdr/XdrMemWriter.h" #include "io/formats/formats.h" #include "io/formats/colloids.h" +#include "comm/Async.h" namespace hemelb { @@ -45,7 +46,7 @@ namespace hemelb std::vector& neighbourProcessors, comm::Communicator::ConstPtr ioComms_, const std::string& outputPath) : - ioComms(ioComms_), localRank(ioComms->Rank()), latDatLBM(latDatLBM), propertyCache(propertyCache), path(outputPath), net(ioComms) + ioComms(ioComms_), localRank(ioComms->Rank()), latDatLBM(latDatLBM), propertyCache(propertyCache), path(outputPath) { /** * Open the file, unless it already exists, for writing only, creating it if it doesn't exist. @@ -285,19 +286,22 @@ namespace hemelb { return; } - - for (scanMapIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) + { - const proc_t& neighbourRank = iterMap->first; - if (neighbourRank != localRank) - { - unsigned int& numberOfParticlesToRecv = iterMap->second.first; - net.RequestSendR(numberOfParticlesToSend, neighbourRank); - net.RequestReceiveR(numberOfParticlesToRecv, neighbourRank); - } + comm::Async commQ(ioComms); + + for (scanMapIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) + { + const proc_t& neighbourRank = iterMap->first; + if (neighbourRank != localRank) + { + unsigned int& numberOfParticlesToRecv = iterMap->second.first; + commQ.Isend(numberOfParticlesToSend, neighbourRank); + commQ.Irecv(numberOfParticlesToRecv, neighbourRank); + } } - net.Dispatch(); - + } + unsigned int numberOfParticles = 0; for (scanMapConstIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) numberOfParticles += iterMap->second.first; @@ -305,18 +309,21 @@ namespace hemelb std::vector::iterator iterSendBegin = particles.begin(); std::vector::iterator iterRecvBegin = particles.begin() + numberOfParticlesToSend; + // Handle the request list ourselves to force the template instaniation to be correct. + auto commQ = ioComms->MakeRequestList(); for (scanMapConstIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) { const proc_t& neighbourRank = iterMap->first; if (neighbourRank != localRank) { const unsigned int& numberOfParticlesToRecv = iterMap->second.first; - net.RequestSend(& ((PersistedParticle&) *iterSendBegin), numberOfParticlesToSend, neighbourRank); - net.RequestReceive(& ((PersistedParticle&) * (iterRecvBegin)), numberOfParticlesToRecv, neighbourRank); + commQ->push_back(ioComms->Isend(&*iterSendBegin, numberOfParticlesToSend, neighbourRank)); + commQ->push_back(ioComms->Irecv(&*iterRecvBegin, numberOfParticlesToRecv, neighbourRank)); + iterRecvBegin += numberOfParticlesToRecv; } } - net.Dispatch(); + commQ->WaitAll(); // remove particles owned by unknown ranks std::vector::iterator newEndOfParticles = @@ -351,20 +358,22 @@ namespace hemelb { return; } - - // exchange counts - for (scanMapIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) + { - const proc_t& neighbourRank = iterMap->first; - if (neighbourRank != localRank) - { - unsigned int& numberOfVelocitiesToSend = iterMap->second.first; - unsigned int& numberOfVelocitiesToRecv = iterMap->second.second; - net.RequestSendR(numberOfVelocitiesToSend, neighbourRank); - net.RequestReceiveR(numberOfVelocitiesToRecv, neighbourRank); - } + comm::Async commQ(ioComms); + // exchange counts + for (scanMapIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) + { + const proc_t& neighbourRank = iterMap->first; + if (neighbourRank != localRank) + { + unsigned int& numberOfVelocitiesToSend = iterMap->second.first; + unsigned int& numberOfVelocitiesToRecv = iterMap->second.second; + commQ.Isend(numberOfVelocitiesToSend, neighbourRank); + commQ.Irecv(numberOfVelocitiesToRecv, neighbourRank); + } + } } - net.Dispatch(); // sum counts unsigned int numberOfIncomingVelocities = 0; @@ -373,6 +382,7 @@ namespace hemelb velocityBuffer.resize(numberOfIncomingVelocities); // exchange velocities + auto commQ = ioComms->MakeRequestList(); std::vector::iterator iterSendBegin = particles.begin(); std::vector > >::iterator iterRecvBegin = velocityBuffer.begin(); for (scanMapConstIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) @@ -382,12 +392,12 @@ namespace hemelb { const unsigned int& numberOfVelocitiesToSend = iterMap->second.first; const unsigned int& numberOfVelocitiesToRecv = iterMap->second.second; - net.RequestSend(& ((Particle&) *iterSendBegin), numberOfVelocitiesToSend, neighbourRank); - net.RequestReceive(& (* (iterRecvBegin)), numberOfVelocitiesToRecv, neighbourRank); + commQ->push_back(ioComms->Isend(&*iterSendBegin, numberOfVelocitiesToSend, neighbourRank)); + commQ->push_back(ioComms->Irecv(&*iterRecvBegin, numberOfVelocitiesToRecv, neighbourRank)); iterRecvBegin += numberOfVelocitiesToRecv; } } - net.Dispatch(); + commQ->WaitAll(); // sum velocities velocityMap.clear(); diff --git a/Code/colloids/ParticleSet.h b/Code/colloids/ParticleSet.h index cb86353a3..196fa1245 100644 --- a/Code/colloids/ParticleSet.h +++ b/Code/colloids/ParticleSet.h @@ -93,8 +93,6 @@ namespace hemelb */ lb::MacroscopicPropertyCache& propertyCache; - /** abstracts communication via MPI */ - net::Net net; /** * Reusable output buffer. */ diff --git a/Code/comm/Async.h b/Code/comm/Async.h new file mode 100644 index 000000000..867c0f4cf --- /dev/null +++ b/Code/comm/Async.h @@ -0,0 +1,61 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_COMM_ASYNC_H +#define HEMELB_COMM_ASYNC_H + +#include "comm/Communicator.h" +#include "comm/Request.h" + +namespace hemelb +{ + namespace comm + { + // Simple class that manages a bunch of asynchronous + // communications. Construct it and add the comms. Their + // completion is waited on when the object is destructed. + class Async + { + public: + typedef std::shared_ptr Ptr; + + inline static Ptr New(Communicator::ConstPtr c) + { + return std::make_shared(c); + } + + inline Async(Communicator::ConstPtr c) : comms(c), q(c->MakeRequestList()) + { + } + inline ~Async() + { + q->WaitAll(); + } + + template + void Isend(Ts... args) + { + q->push_back(comms->Isend(args...)); + } + template + void Issend(Ts... args) + { + q->push_back(comms->Issend(args...)); + } + template + void Irecv(Ts... args) + { + q->push_back(comms->Irecv(args...)); + } + + private: + Communicator::ConstPtr comms; + RequestList::Ptr q; + }; + + } +} + +#endif diff --git a/Code/geometry/GeometryReader.cc b/Code/geometry/GeometryReader.cc index 0b2f01f4f..4ac804ca9 100644 --- a/Code/geometry/GeometryReader.cc +++ b/Code/geometry/GeometryReader.cc @@ -17,11 +17,11 @@ #include "geometry/decomposition/OptimisedDecomposition.h" #include "geometry/GeometryReader.h" #include "lb/lattices/D3Q27.h" -#include "net/net.h" #include "log/Logger.h" #include "util/utilityFunctions.h" #include "constants.h" #include "comm/Group.h" +#include "comm/Async.h" namespace hemelb { @@ -367,40 +367,38 @@ namespace hemelb std::vector compressedBlockData; proc_t readingCore = GetReadingCoreForBlock(blockNumber); - net::Net net = net::Net(computeComms); - - if (readingCore == computeComms->Rank()) - { - timings[hemelb::reporting::Timers::readBlock].Start(); - // Read the data. - compressedBlockData.resize(bytesPerCompressedBlock[blockNumber]); - file->ReadAt(offsetSoFar, compressedBlockData); - - // Spread it. - for (std::vector::const_iterator receiver = procsWantingThisBlock.begin(); receiver - != procsWantingThisBlock.end(); receiver++) - { - if (*receiver != computeComms->Rank()) - { - - net.RequestSendV(compressedBlockData, *receiver); - } - } - timings[hemelb::reporting::Timers::readBlock].Stop(); - } - else if (neededOnThisRank) { - compressedBlockData.resize(bytesPerCompressedBlock[blockNumber]); - - net.RequestReceiveV(compressedBlockData, readingCore); - - } - else - { - return; + comm::Async requestQ(computeComms); + + if (readingCore == computeComms->Rank()) + { + timings[hemelb::reporting::Timers::readBlock].Start(); + // Read the data. + compressedBlockData.resize(bytesPerCompressedBlock[blockNumber]); + file->ReadAt(offsetSoFar, compressedBlockData); + + // Spread it. + for (std::vector::const_iterator receiver = procsWantingThisBlock.begin(); receiver + != procsWantingThisBlock.end(); receiver++) + { + if (*receiver != computeComms->Rank()) + { + requestQ.Isend(compressedBlockData, *receiver); + } + } + timings[hemelb::reporting::Timers::readBlock].Stop(); + } + else if (neededOnThisRank) + { + compressedBlockData.resize(bytesPerCompressedBlock[blockNumber]); + requestQ.Irecv(compressedBlockData, readingCore); + } + else + { + return; + } + timings[hemelb::reporting::Timers::readNet].Start(); } - timings[hemelb::reporting::Timers::readNet].Start(); - net.Dispatch(); timings[hemelb::reporting::Timers::readNet].Stop(); timings[hemelb::reporting::Timers::readParse].Start(); if (neededOnThisRank) diff --git a/Code/geometry/decomposition/OptimisedDecomposition.cc b/Code/geometry/decomposition/OptimisedDecomposition.cc index 72d71f5fa..f29a022c4 100644 --- a/Code/geometry/decomposition/OptimisedDecomposition.cc +++ b/Code/geometry/decomposition/OptimisedDecomposition.cc @@ -8,8 +8,8 @@ #include "geometry/decomposition/DecompositionWeights.h" #include "lb/lattices/D3Q27.h" #include "log/Logger.h" -#include "net/net.h" #include "comm/MpiCommunicator.h" +#include "comm/Async.h" #include "Exception.h" namespace hemelb @@ -559,8 +559,6 @@ namespace hemelb { timers[hemelb::reporting::Timers::moveForcingNumbers].Start(); - net::Net netForMoveSending(comms); - // We also need to force some data upon blocks, i.e. when they're receiving data from a new // block they didn't previously want to know about. std::map > blockForcedUponX; @@ -586,28 +584,30 @@ namespace hemelb std::vector blocksForcedOnMe = comms->AllToAll(numberOfBlocksIForceUponX); timers[hemelb::reporting::Timers::moveForcingNumbers].Stop(); + // Now get all the blocks being forced upon me. timers[hemelb::reporting::Timers::moveForcingData].Start(); - // Now get all the blocks being forced upon me. - std::map > blocksForcedOnMeByEachProc; - for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) + std::map > blocksForcedOnMeByEachProc; { - if (blocksForcedOnMe[otherProc] > 0) - { - blocksForcedOnMeByEachProc[otherProc] = - std::vector(blocksForcedOnMe[otherProc]); - netForMoveSending.RequestReceiveV(blocksForcedOnMeByEachProc[otherProc], otherProc); - } - if (numberOfBlocksIForceUponX[otherProc] > 0) - { - netForMoveSending.RequestSendV(blockForcedUponX[otherProc], otherProc); - } - log::Logger::Log("I'm forcing %i blocks on proc %i.", - numberOfBlocksIForceUponX[otherProc], - otherProc); - } - - log::Logger::Log("Moving forcing block ids"); - netForMoveSending.Dispatch(); + comm::Async requestQ(comms); + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) + { + if (blocksForcedOnMe[otherProc] > 0) + { + blocksForcedOnMeByEachProc[otherProc] = + std::vector(blocksForcedOnMe[otherProc]); + requestQ.Irecv(blocksForcedOnMeByEachProc[otherProc], otherProc); + } + if (numberOfBlocksIForceUponX[otherProc] > 0) + { + requestQ.Isend(blockForcedUponX[otherProc], otherProc); + } + log::Logger::Log("I'm forcing %i blocks on proc %i.", + numberOfBlocksIForceUponX[otherProc], + otherProc); + } + log::Logger::Log("Moving forcing block ids"); + } + // Now go through every block forced upon me and add it to the list of ones I want. for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { @@ -698,18 +698,19 @@ namespace hemelb // Awesome. Now we need to get a list of all the blocks wanted from each core by each other // core. - net::Net netForMoveSending(comms); - for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) - { - blockIdsXRequiresFromMe[otherProc] = - std::vector(numberOfBlocksXRequiresFromMe[otherProc]); - log::Logger::Log("Proc %i requires %i blocks from me", - otherProc, - blockIdsXRequiresFromMe[otherProc].size()); - netForMoveSending.RequestReceiveV(blockIdsXRequiresFromMe[otherProc], otherProc); - netForMoveSending.RequestSendV(blockIdsIRequireFromX[otherProc], otherProc); + { + comm::Async requestQ(comms); + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) + { + blockIdsXRequiresFromMe[otherProc] = + std::vector(numberOfBlocksXRequiresFromMe[otherProc]); + log::Logger::Log("Proc %i requires %i blocks from me", + otherProc, + blockIdsXRequiresFromMe[otherProc].size()); + requestQ.Irecv(blockIdsXRequiresFromMe[otherProc], otherProc); + requestQ.Isend(blockIdsIRequireFromX[otherProc], otherProc); + } } - netForMoveSending.Dispatch(); timers[hemelb::reporting::Timers::blockRequirements].Stop(); } @@ -765,29 +766,30 @@ namespace hemelb movesForEachLocalBlock[blockId]++; } - net::Net netForMoveSending(comms); - for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) - { - for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); - it != blockIdsIRequireFromX[otherProc].end(); ++it) - { - netForMoveSending.RequestReceiveR(movesForEachBlockWeCareAbout[*it], otherProc); - log::Logger::Log("I want the move count for block %i from proc %i", - *it, - otherProc); - } - for (std::vector::iterator it = blockIdsXRequiresFromMe[otherProc].begin(); - it != blockIdsXRequiresFromMe[otherProc].end(); ++it) - { - netForMoveSending.RequestSendR(movesForEachLocalBlock[*it], otherProc); - log::Logger::Log("I'm sending move count for block %i to proc %i", - *it, - otherProc); - } - } - - log::Logger::Log("Sending move counts"); - netForMoveSending.Dispatch(); + { + comm::Async requestQ(comms); + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) + { + for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); + it != blockIdsIRequireFromX[otherProc].end(); ++it) + { + requestQ.Irecv(movesForEachBlockWeCareAbout[*it], otherProc); + log::Logger::Log("I want the move count for block %i from proc %i", + *it, + otherProc); + } + for (std::vector::iterator it = blockIdsXRequiresFromMe[otherProc].begin(); + it != blockIdsXRequiresFromMe[otherProc].end(); ++it) + { + requestQ.Isend(movesForEachLocalBlock[*it], otherProc); + log::Logger::Log("I'm sending move count for block %i to proc %i", + *it, + otherProc); + } + } + + log::Logger::Log("Sending move counts"); + } timers[hemelb::reporting::Timers::moveCountsSending].Stop(); } @@ -810,48 +812,48 @@ namespace hemelb movesList.resize(totalMovesToReceive * 3); idx_t localMoveId = 0; - net::Net netForMoveSending(comms); - - for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) - { - allMoves[otherProc] = 0; - for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); - it != blockIdsIRequireFromX[otherProc].end(); ++it) - { - if (movesForEachBlockWeCareAbout[*it] > 0) - { - netForMoveSending.RequestReceive(&movesList[localMoveId * 3], - 3 * movesForEachBlockWeCareAbout[*it], - otherProc); - localMoveId += movesForEachBlockWeCareAbout[*it]; - allMoves[otherProc] += movesForEachBlockWeCareAbout[*it]; - log::Logger::Log("Expect %i moves from from proc %i about block %i", - movesForEachBlockWeCareAbout[*it], - otherProc, - *it); - } - } - - for (std::vector::iterator it = blockIdsXRequiresFromMe[otherProc].begin(); - it != blockIdsXRequiresFromMe[otherProc].end(); ++it) - { - if (moveDataForEachBlock[*it].size() > 0) - { - netForMoveSending.RequestSendV(moveDataForEachBlock[*it], otherProc); - log::Logger::Log("Sending %i moves from to proc %i about block %i", - moveDataForEachBlock[*it].size() / 3, - otherProc, - *it); - } - } - - log::Logger::Log("%i moves from proc %i", - allMoves[otherProc], - otherProc); - } - - log::Logger::Log("Sending move data"); - netForMoveSending.Dispatch(); + { + comm::Async requestQ(comms); + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) + { + allMoves[otherProc] = 0; + for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); + it != blockIdsIRequireFromX[otherProc].end(); ++it) + { + if (movesForEachBlockWeCareAbout[*it] > 0) + { + requestQ.Irecv(&movesList[localMoveId * 3], + 3 * movesForEachBlockWeCareAbout[*it], + otherProc, 0); + localMoveId += movesForEachBlockWeCareAbout[*it]; + allMoves[otherProc] += movesForEachBlockWeCareAbout[*it]; + log::Logger::Log("Expect %i moves from from proc %i about block %i", + movesForEachBlockWeCareAbout[*it], + otherProc, + *it); + } + } + + for (std::vector::iterator it = blockIdsXRequiresFromMe[otherProc].begin(); + it != blockIdsXRequiresFromMe[otherProc].end(); ++it) + { + if (moveDataForEachBlock[*it].size() > 0) + { + requestQ.Isend(moveDataForEachBlock[*it], otherProc); + log::Logger::Log("Sending %i moves from to proc %i about block %i", + moveDataForEachBlock[*it].size() / 3, + otherProc, + *it); + } + } + + log::Logger::Log("%i moves from proc %i", + allMoves[otherProc], + otherProc); + } + + log::Logger::Log("Sending move data"); + } timers[hemelb::reporting::Timers::moveDataSending].Stop(); } From d4fda7be7298d27e76b50ef48200ed617db6fbd9 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 24 Oct 2016 17:15:34 +0100 Subject: [PATCH 74/99] move more away from net --- Code/geometry/LatticeData.cc | 9 +++--- Code/lb/iolets/InOutLetMultiscale.cc | 41 ++++++++++++++-------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Code/geometry/LatticeData.cc b/Code/geometry/LatticeData.cc index dd14bef47..5460b994c 100644 --- a/Code/geometry/LatticeData.cc +++ b/Code/geometry/LatticeData.cc @@ -14,6 +14,7 @@ #include "geometry/LatticeData.h" #include "geometry/neighbouring/NeighbouringLatticeData.h" #include "util/utilityFunctions.h" +#include "comm/Async.h" namespace hemelb { @@ -441,7 +442,7 @@ namespace hemelb // propagate to different partitions is avoided (only their values // will be communicated). It's here! // Allocate the request variable. - net::Net tempNet(comms); + comm::Async commQ(comms); for (size_t neighbourId = 0; neighbourId < neighbouringProcs.size(); neighbourId++) { NeighbouringProcessor* neigh_proc_p = &neighbouringProcs[neighbourId]; @@ -451,16 +452,14 @@ namespace hemelb // other processor. if (neigh_proc_p->Rank > localRank) { - tempNet.RequestSendV(sharedFLocationForEachProc[neigh_proc_p->Rank], neigh_proc_p->Rank); + commQ.Isend(sharedFLocationForEachProc[neigh_proc_p->Rank], neigh_proc_p->Rank); } else { sharedFLocationForEachProc[neigh_proc_p->Rank].resize(neigh_proc_p->SharedDistributionCount * 4); - tempNet.RequestReceiveV(sharedFLocationForEachProc[neigh_proc_p->Rank], neigh_proc_p->Rank); + commQ.Irecv(sharedFLocationForEachProc[neigh_proc_p->Rank], neigh_proc_p->Rank); } } - - tempNet.Dispatch(); } void LatticeData::InitialiseReceiveLookup(std::vector >& sharedFLocationForEachProc) diff --git a/Code/lb/iolets/InOutLetMultiscale.cc b/Code/lb/iolets/InOutLetMultiscale.cc index 18dca2d1b..ea9f153b2 100644 --- a/Code/lb/iolets/InOutLetMultiscale.cc +++ b/Code/lb/iolets/InOutLetMultiscale.cc @@ -8,6 +8,7 @@ #include "configuration/SimConfig.h" #include "lb/iolets/BoundaryComms.h" #include "lb/iolets/BoundaryValues.h" +#include "comm/Async.h" namespace hemelb { @@ -122,27 +123,27 @@ namespace hemelb pressure_array[1] = minPressure.GetPayload(); pressure_array[2] = maxPressure.GetPayload(); - net::Net commsNet(bcComms.GetComm()); - - const std::vector& procList = comms->GetListOfProcs(); //TODO: CHECK + IMPROVE! - - // If this proc is to do IO, send the pressure array list to all cores that require it. - if (isIoProc && procList[0] != bcComms.GetBCProcRank()) - { - for (std::vector::const_iterator it = procList.begin(); it != procList.end(); it++) - { - commsNet.RequestSend(pressure_array, 3, *it); - } + { + comm::Async commsNet(bcComms.GetComm()); + const std::vector& procList = comms->GetListOfProcs(); //TODO: CHECK + IMPROVE! + + // If this proc is to do IO, send the pressure array list to all cores that require it. + if (isIoProc && procList[0] != bcComms.GetBCProcRank()) + { + for (std::vector::const_iterator it = procList.begin(); it != procList.end(); it++) + { + commsNet.Isend(pressure_array, 3, *it, 0); + } + } + // Otherwise, receive the pressure array list from the core. + else if (procList[0] != bcComms.GetBCProcRank()) + { + commsNet.Irecv(pressure_array, 3, bcComms.GetBCProcRank(), 0); + } + + // Perform the send / receive when the Async destructs } - // Otherwise, receive the pressure array list from the core. - else if (procList[0] != bcComms.GetBCProcRank()) - { - commsNet.RequestReceive(pressure_array, 3, bcComms.GetBCProcRank()); - } - - // Perform the send / receive. - commsNet.Dispatch(); - + if (!isIoProc) { pressure.SetPayload(static_cast (pressure_array[0])); From 03f78a4b790864794509638b78a1dbea18e782f3 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 27 Oct 2016 10:28:07 +0100 Subject: [PATCH 75/99] remove the net from most of the rest --- Code/SimulationMaster.cc | 10 ++-- Code/SimulationMaster.h | 2 +- Code/comm/Async.h | 9 ++++ Code/comm/MpiRequest.cc | 4 ++ Code/comm/MpiRequest.h | 2 +- Code/comm/Request.h | 1 + Code/geometry/LatticeData.cc | 17 +++---- Code/geometry/LatticeData.h | 4 +- .../neighbouring/NeighbouringDataManager.cc | 48 ++++++++++--------- .../neighbouring/NeighbouringDataManager.h | 8 ++-- Code/lb/IncompressibilityChecker.cc | 12 +++-- Code/lb/IncompressibilityChecker.h | 6 +-- Code/lb/StabilityTester.h | 2 +- Code/lb/StabilityTester.hpp | 5 +- Code/lb/lb.h | 5 +- Code/lb/lb.hpp | 6 +-- 16 files changed, 81 insertions(+), 60 deletions(-) diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index 8039562a5..6aaeb72cc 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -33,7 +33,7 @@ * object. */ SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, hemelb::comm::Communicator::ConstPtr ioComm) : - ioComms(ioComm), timings(ioComm), build_info(), communicationNet(ioComm) + ioComms(ioComm), timings(ioComm), build_info(), communicationNet(ioComm), asyncCommQ(hemelb::comm::Async::New(ioComm)) { timings[hemelb::reporting::Timers::total].Start(); @@ -154,10 +154,10 @@ void SimulationMaster::Initialise() neighbouringDataManager = new hemelb::geometry::neighbouring::NeighbouringDataManager(*latticeData, latticeData->GetNeighbouringData(), - communicationNet); + asyncCommQ); hemelb::log::Logger::Log("Initialising LBM."); latticeBoltzmannModel = new hemelb::lb::LBM(simConfig, - &communicationNet, + asyncCommQ, latticeData, simulationState, timings, @@ -203,7 +203,7 @@ void SimulationMaster::Initialise() } stabilityTester = new hemelb::lb::StabilityTester(latticeData, - &communicationNet, + ioComms, simulationState, timings, monitoringConfig->doConvergenceCheck, @@ -213,7 +213,7 @@ void SimulationMaster::Initialise() if (monitoringConfig->doIncompressibilityCheck) { incompressibilityChecker = new hemelb::lb::IncompressibilityChecker(latticeData, - &communicationNet, + ioComms, simulationState, latticeBoltzmannModel->GetPropertyCache(), timings); diff --git a/Code/SimulationMaster.h b/Code/SimulationMaster.h index a35dda748..2179f79b8 100644 --- a/Code/SimulationMaster.h +++ b/Code/SimulationMaster.h @@ -95,7 +95,7 @@ class SimulationMaster hemelb::colloids::ColloidController* colloidController; hemelb::net::Net communicationNet; - + hemelb::comm::Async::Ptr asyncCommQ; const hemelb::util::UnitConverter* unitConverter; hemelb::extraction::IterableDataSource* propertyDataSource; diff --git a/Code/comm/Async.h b/Code/comm/Async.h index 867c0f4cf..1082ebe6b 100644 --- a/Code/comm/Async.h +++ b/Code/comm/Async.h @@ -34,6 +34,15 @@ namespace hemelb q->WaitAll(); } + inline void Wait() { + q->WaitAll(); + q->clear(); + } + + inline Communicator::ConstPtr GetComm() const { + return comms; + } + template void Isend(Ts... args) { diff --git a/Code/comm/MpiRequest.cc b/Code/comm/MpiRequest.cc index 295a1afd7..c27c1cb1d 100644 --- a/Code/comm/MpiRequest.cc +++ b/Code/comm/MpiRequest.cc @@ -48,6 +48,10 @@ namespace hemelb { reqs.resize(i); } + void MpiRequestList::clear() + { + reqs.clear(); + } void MpiRequestList::push_back(Request::Ptr r) { diff --git a/Code/comm/MpiRequest.h b/Code/comm/MpiRequest.h index e0314a87d..8432e1c80 100644 --- a/Code/comm/MpiRequest.h +++ b/Code/comm/MpiRequest.h @@ -57,7 +57,7 @@ namespace hemelb virtual void resize(size_t i); virtual void push_back(Request::Ptr); virtual void set(size_t i, Request::Ptr); - + virtual void clear(); virtual void WaitAll(); virtual bool TestAll(); private: diff --git a/Code/comm/Request.h b/Code/comm/Request.h index a50152af5..ceb80359a 100644 --- a/Code/comm/Request.h +++ b/Code/comm/Request.h @@ -44,6 +44,7 @@ namespace hemelb virtual size_t size() const = 0; virtual void resize(size_t i) = 0; virtual void push_back(Request::Ptr) = 0; + virtual void clear() = 0; virtual void set(size_t i, Request::Ptr) = 0; virtual void WaitAll() = 0; diff --git a/Code/geometry/LatticeData.cc b/Code/geometry/LatticeData.cc index 5460b994c..28325e8ba 100644 --- a/Code/geometry/LatticeData.cc +++ b/Code/geometry/LatticeData.cc @@ -652,20 +652,21 @@ namespace hemelb blockCoords.x = blockIJData / blockCounts.y; } - void LatticeData::SendAndReceive(hemelb::net::Net* net) + void LatticeData::SendAndReceive(comm::Async::Ptr commQ) { for (std::vector::const_iterator it = neighbouringProcs.begin(); it != neighbouringProcs.end(); ++it) { // Request the receive into the appropriate bit of FOld. - net->RequestReceive(GetFOld( (*it).FirstSharedDistribution), - (int) ( ( (*it).SharedDistributionCount)), - (*it).Rank); + commQ->Irecv(GetFOld( (*it).FirstSharedDistribution), + (int) ( ( (*it).SharedDistributionCount)), + (*it).Rank, + 1); // Request the send from the right bit of FNew. - net->RequestSend(GetFNew( (*it).FirstSharedDistribution), - (int) ( ( (*it).SharedDistributionCount)), - (*it).Rank); - + commQ->Isend(GetFNew( (*it).FirstSharedDistribution), + (int) ( ( (*it).SharedDistributionCount)), + (*it).Rank, + 1); } } diff --git a/Code/geometry/LatticeData.h b/Code/geometry/LatticeData.h index 9ac7879df..11eba4515 100644 --- a/Code/geometry/LatticeData.h +++ b/Code/geometry/LatticeData.h @@ -10,7 +10,7 @@ #include #include -#include "net/net.h" +#include "comm/Async.h" #include "constants.h" #include "configuration/SimConfig.h" #include "geometry/Block.h" @@ -51,7 +51,7 @@ namespace hemelb oldDistributions.swap(newDistributions); } - void SendAndReceive(net::Net* net); + void SendAndReceive(comm::Async::Ptr commQ); void CopyReceived(); /** diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.cc b/Code/geometry/neighbouring/NeighbouringDataManager.cc index ac20359db..1903d12ac 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.cc +++ b/Code/geometry/neighbouring/NeighbouringDataManager.cc @@ -20,9 +20,9 @@ namespace hemelb NeighbouringDataManager::NeighbouringDataManager( const LatticeData & localLatticeData, NeighbouringLatticeData & neighbouringLatticeData, - net::InterfaceDelegationNet & net) : + comm::Async::Ptr cq) : localLatticeData(localLatticeData), neighbouringLatticeData(neighbouringLatticeData), - net(net), needsHaveBeenShared(false) + commQ(cq), needsHaveBeenShared(false) { } void NeighbouringDataManager::RegisterNeededSite(site_t globalId, @@ -43,7 +43,10 @@ namespace hemelb { return localLatticeData.ProcProvidingSiteByGlobalNoncontiguousId(site); } - + + // TODO: figure out if this needs to force the commQ to wait + // (current behaviour) or if it can create a new queue and wait + // on that only. void NeighbouringDataManager::TransferNonFieldDependentInformation() { // Ordering is important here, to ensure the requests are registered in the same order @@ -55,15 +58,14 @@ namespace hemelb { proc_t source = ProcForSite(*localNeed); NeighbouringSite site = neighbouringLatticeData.GetSite(*localNeed); - - net.RequestReceiveR(site.GetSiteData().GetWallIntersectionData(), source); - net.RequestReceiveR(site.GetSiteData().GetIoletIntersectionData(), source); - net.RequestReceiveR(site.GetSiteData().GetIoletId(), source); - net.RequestReceiveR(site.GetSiteData().GetSiteType(), source); - net.RequestReceive(site.GetWallDistances(), - localLatticeData.GetLatticeInfo().GetNumVectors() - 1, - source); - net.RequestReceiveR(site.GetWallNormal(), source); + commQ->Irecv(site.GetSiteData().GetWallIntersectionData(), source); + commQ->Irecv(site.GetSiteData().GetIoletIntersectionData(), source); + commQ->Irecv(site.GetSiteData().GetIoletId(), source); + commQ->Irecv(site.GetSiteData().GetSiteType(), source); + commQ->Irecv(site.GetWallDistances(), + localLatticeData.GetLatticeInfo().GetNumVectors() - 1, + source); + commQ->Irecv(site.GetWallNormal(), source); } for (IdsMap::const_iterator iter = needsEachProcHasFromMe.begin(); @@ -82,23 +84,23 @@ namespace hemelb const Site site = localLatticeData.GetSite(localContiguousId); const SiteData& sd = site.GetSiteData(); - net.RequestSendR(sd.GetWallIntersectionData(), other); - net.RequestSendR(sd.GetIoletIntersectionData(), other); - net.RequestSendR(sd.GetIoletId(), other); - net.RequestSendR(sd.GetSiteType(), other); - net.RequestSend(site.GetWallDistances(), + commQ->Isend(sd.GetWallIntersectionData(), other); + commQ->Isend(sd.GetIoletIntersectionData(), other); + commQ->Isend(sd.GetIoletId(), other); + commQ->Isend(sd.GetSiteType(), other); + commQ->Isend(site.GetWallDistances(), localLatticeData.GetLatticeInfo().GetNumVectors() - 1, other); - net.RequestSendR(site.GetWallNormal(), other); + commQ->Isend(site.GetWallNormal(), other); } } - net.Dispatch(); + commQ->Wait(); } void NeighbouringDataManager::TransferFieldDependentInformation() { RequestComms(); - net.Dispatch(); + commQ->Wait(); } void NeighbouringDataManager::RequestComms() @@ -118,7 +120,7 @@ namespace hemelb { proc_t source = ProcForSite(*localNeed); NeighbouringSite site = neighbouringLatticeData.GetSite(*localNeed); - net.RequestReceive(site.GetFOld(localLatticeData.GetLatticeInfo().GetNumVectors()), + commQ->Irecv(site.GetFOld(localLatticeData.GetLatticeInfo().GetNumVectors()), localLatticeData.GetLatticeInfo().GetNumVectors(), source); @@ -138,7 +140,7 @@ namespace hemelb localLatticeData.GetLocalContiguousIdFromGlobalNoncontiguousId(*needOnProcFromMe); const Site site = localLatticeData.GetSite(localContiguousId); // have to cast away the const, because no respect for const-ness for sends in MPI - net.RequestSend(site.GetFOld(Q), + commQ->Isend(site.GetFOld(Q), Q, other); } @@ -166,8 +168,8 @@ namespace hemelb // This will store the numbers of sites other ranks need from this rank CountMap countOfNeedsOnEachProcFromMe; + auto comms = commQ->GetComm(); // This is collective - comm::Communicator::ConstPtr comms = net.GetCommunicator(); comm::MapAllToAll(comms, countOfNeedsIHaveFromEachProc, countOfNeedsOnEachProcFromMe, 1234); //comm::Request::ReqVec requestQueue; diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.h b/Code/geometry/neighbouring/NeighbouringDataManager.h index 49f477e49..b0d19b1b3 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.h +++ b/Code/geometry/neighbouring/NeighbouringDataManager.h @@ -29,12 +29,15 @@ namespace hemelb public: NeighbouringDataManager(const LatticeData & localLatticeData, NeighbouringLatticeData & neighbouringLatticeData, - net::InterfaceDelegationNet & net); + comm::Async::Ptr cq); // Initially, the required site information will not be used -- we just transfer everything. // This considerably simplifies matters. // Nevertheless, we provide the interface here in its final form void RegisterNeededSite(site_t globalId, RequiredSiteInformation requirements = RequiredSiteInformation(true)); + + // This is collective across the communicator used by the + // communication queue given to the constructor void ShareNeeds(); std::vector &GetNeedsForProc(proc_t proc) { @@ -53,8 +56,7 @@ namespace hemelb private: const LatticeData & localLatticeData; NeighbouringLatticeData & neighbouringLatticeData; - net::InterfaceDelegationNet & net; - + comm::Async::Ptr commQ; typedef std::vector IdVec; typedef std::map CountMap; typedef std::map IdsMap; diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index 63879c318..ee5eac9c8 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -51,11 +51,13 @@ namespace hemelb } } - IncompressibilityChecker::IncompressibilityChecker( - const geometry::LatticeData * latticeData, net::Net* net, SimulationState* simState, - lb::MacroscopicPropertyCache& propertyCache, reporting::Timers& timings, - distribn_t maximumRelativeDensityDifferenceAllowed) : - net::CollectiveAction(net->GetCommunicator(), timings[reporting::Timers::mpiWait]), + IncompressibilityChecker::IncompressibilityChecker(const geometry::LatticeData * latticeData, + comm::Communicator::ConstPtr comms, + SimulationState* simState, + lb::MacroscopicPropertyCache& propertyCache, + reporting::Timers& timings, + distribn_t maximumRelativeDensityDifferenceAllowed) : + net::CollectiveAction(comms, timings[reporting::Timers::mpiWait]), mLatDat(latticeData), propertyCache(propertyCache), mSimState(simState), maximumRelativeDensityDifferenceAllowed(maximumRelativeDensityDifferenceAllowed), workTimer(timings[reporting::Timers::monitoring]) diff --git a/Code/lb/IncompressibilityChecker.h b/Code/lb/IncompressibilityChecker.h index ba5860f1f..17f15765b 100644 --- a/Code/lb/IncompressibilityChecker.h +++ b/Code/lb/IncompressibilityChecker.h @@ -6,7 +6,7 @@ #ifndef HEMELB_LB_INCOMPRESSIBILITYCHECKER_H #define HEMELB_LB_INCOMPRESSIBILITYCHECKER_H - +#include "comm/Communicator.h" #include "net/CollectiveAction.h" #include "geometry/LatticeData.h" #include "lb/MacroscopicPropertyCache.h" @@ -37,11 +37,11 @@ namespace hemelb * Constructor * * @param latticeData geometry object - * @param net network interface object + * @param comms communicator object * @param simState simulation state * @param maximumRelativeDensityDifferenceAllowed maximum density difference allowed in the domain (relative to reference density, default 5%) */ - IncompressibilityChecker(const geometry::LatticeData * latticeData, net::Net* net, + IncompressibilityChecker(const geometry::LatticeData * latticeData, comm::Communicator::ConstPtr comms, SimulationState* simState, lb::MacroscopicPropertyCache& propertyCache, reporting::Timers& timings, diff --git a/Code/lb/StabilityTester.h b/Code/lb/StabilityTester.h index 8a5670a52..73bb65b70 100644 --- a/Code/lb/StabilityTester.h +++ b/Code/lb/StabilityTester.h @@ -21,7 +21,7 @@ namespace hemelb class StabilityTester : public net::CollectiveAction { public: - StabilityTester(const geometry::LatticeData * iLatDat, net::Net* net, + StabilityTester(const geometry::LatticeData * iLatDat, comm::Communicator::ConstPtr comms, SimulationState* simState, reporting::Timers& timings, bool checkForConvergence, double relativeTolerance); diff --git a/Code/lb/StabilityTester.hpp b/Code/lb/StabilityTester.hpp index 445aae279..987072238 100644 --- a/Code/lb/StabilityTester.hpp +++ b/Code/lb/StabilityTester.hpp @@ -19,11 +19,12 @@ namespace hemelb */ template StabilityTester::StabilityTester(const geometry::LatticeData * iLatDat, - net::Net* net, SimulationState* simState, + comm::Communicator::ConstPtr comms, + SimulationState* simState, reporting::Timers& timings, bool checkForConvergence, double relativeTolerance) : - CollectiveAction(net->GetCommunicator(), timings[reporting::Timers::monitoring]), + CollectiveAction(comms, timings[reporting::Timers::monitoring]), mLatDat(iLatDat), mSimState(simState), checkForConvergence(checkForConvergence), relativeTolerance(relativeTolerance), workTimer(timings[reporting::Timers::monitoring]) { diff --git a/Code/lb/lb.h b/Code/lb/lb.h index 68bceb37f..aaa28f719 100644 --- a/Code/lb/lb.h +++ b/Code/lb/lb.h @@ -7,7 +7,6 @@ #ifndef HEMELB_LB_LB_H #define HEMELB_LB_LB_H -#include "net/net.h" #include "net/IteratedAction.h" #include "comm/Communicator.h" #include "lb/SimulationState.h" @@ -56,7 +55,7 @@ namespace hemelb * the partially initialized LBM in order to initialize the arguments to the second construction phase. */ LBM(hemelb::configuration::SimConfig *iSimulationConfig, - net::Net* net, + comm::Async::Ptr commQ, geometry::LatticeData* latDat, SimulationState* simState, reporting::Timers &atimings, @@ -133,7 +132,7 @@ namespace hemelb unsigned int outletCount; configuration::SimConfig *mSimConfig; - net::Net* mNet; + comm::Async::Ptr mCommQ; geometry::LatticeData* mLatDat; SimulationState* mState; iolets::BoundaryValues *mInletValues, *mOutletValues; diff --git a/Code/lb/lb.hpp b/Code/lb/lb.hpp index f828f630b..7e79baf2f 100644 --- a/Code/lb/lb.hpp +++ b/Code/lb/lb.hpp @@ -29,12 +29,12 @@ namespace hemelb template LBM::LBM(configuration::SimConfig *iSimulationConfig, - net::Net* net, + comm::Async::Ptr commQ, geometry::LatticeData* latDat, SimulationState* simState, reporting::Timers &atimings, geometry::neighbouring::NeighbouringDataManager *neighbouringDataManager) : - mSimConfig(iSimulationConfig), mNet(net), mLatDat(latDat), mState(simState), + mSimConfig(iSimulationConfig), mCommQ(commQ), mLatDat(latDat), mState(simState), mParams(iSimulationConfig->GetTimeStepLength(), iSimulationConfig->GetVoxelSize()), timings(atimings), propertyCache(*simState, *latDat), neighbouringDataManager(neighbouringDataManager) { @@ -175,7 +175,7 @@ namespace hemelb // (via the Net object). // NOTE that this doesn't actually *perform* the sends and receives, it asks the Net // to include them in the ISends and IRecvs that happen later. - mLatDat->SendAndReceive(mNet); + mLatDat->SendAndReceive(mCommQ); timings[hemelb::reporting::Timers::lb].Stop(); } From 91e460cfef37c46b37f73fa0bae59bba5bd08b08 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 27 Oct 2016 10:57:30 +0100 Subject: [PATCH 76/99] remove the steering! --- CMakeLists.txt | 6 - Code/CMakeLists.txt | 5 - Code/SimulationMaster.cc | 32 +-- Code/SimulationMaster.h | 5 - Code/configuration/CommandLine.cc | 8 +- Code/configuration/CommandLine.h | 9 - Code/geometry/GeometryReader.cc | 152 ++++------- Code/geometry/GeometryReader.h | 9 +- Code/reporting/BuildInfo.h.in | 2 - Code/reporting/Timers.h | 3 +- Code/resources/report.txt.ctp | 1 - Code/resources/report.xml.ctp | 1 - Code/steering/CMakeLists.txt | 45 ---- Code/steering/ClientConnection.h | 45 ---- Code/steering/Network.h | 54 ---- Code/steering/SteeringComponent.h | 94 ------- Code/steering/basic/ClientConnection.cc | 167 ------------ Code/steering/basic/HttpPost.cc | 277 -------------------- Code/steering/basic/HttpPost.h | 56 ---- Code/steering/basic/Network.cc | 257 ------------------ Code/steering/basic/SimulationParameters.cc | 41 --- Code/steering/basic/SimulationParameters.h | 40 --- Code/steering/basic/SteeringComponentB.cc | 97 ------- Code/steering/common/SteeringComponentC.cc | 29 -- Code/steering/none/ClientConnection.cc | 47 ---- Code/steering/none/Network.cc | 86 ------ Code/steering/none/SteeringComponentN.cc | 53 ---- deploy/compile_options.yml | 4 - deploy/fab.py | 48 +--- deploy/templates/multiscale_hemelb | 2 +- deploy/templates/regression | 2 +- deploy/test/fixtures/templates/hemelb | 2 +- 32 files changed, 65 insertions(+), 1614 deletions(-) delete mode 100644 Code/steering/CMakeLists.txt delete mode 100644 Code/steering/ClientConnection.h delete mode 100644 Code/steering/Network.h delete mode 100644 Code/steering/SteeringComponent.h delete mode 100644 Code/steering/basic/ClientConnection.cc delete mode 100644 Code/steering/basic/HttpPost.cc delete mode 100644 Code/steering/basic/HttpPost.h delete mode 100644 Code/steering/basic/Network.cc delete mode 100644 Code/steering/basic/SimulationParameters.cc delete mode 100644 Code/steering/basic/SimulationParameters.h delete mode 100644 Code/steering/basic/SteeringComponentB.cc delete mode 100644 Code/steering/common/SteeringComponentC.cc delete mode 100644 Code/steering/none/ClientConnection.cc delete mode 100644 Code/steering/none/Network.cc delete mode 100644 Code/steering/none/SteeringComponentN.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 7f6d11949..fc14273b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,8 +20,6 @@ set(HEMELB_READING_GROUP_SIZE 5 CACHE INTEGER "Number of cores to use to read geometry file.") set(HEMELB_LOG_LEVEL Info CACHE STRING "Log level, choose 'Critical', 'Error', 'Warning', 'Info', 'Debug' or 'Trace'" ) -set(HEMELB_STEERING_LIB basic - CACHE STRING "Steering library, choose 'basic' or 'none'" ) option(HEMELB_USE_MULTIMACHINE "Use multi-level parallelism support" OFF) option(HEMELB_BUILD_UNITTESTS "Build the unit-tests" ON) option(HEMELB_USE_ALL_WARNINGS_GNU "Show all compiler warnings on development builds (gnu-style-compilers)" ON) @@ -33,10 +31,8 @@ set(HEMELB_KERNEL "LBGK" CACHE STRING "Select the kernel to use (LBGK,EntropicAnsumali,EntropicChik,MRT,NNCY,NNC,NNTPL)") set(HEMELB_WALL_BOUNDARY "SIMPLEBOUNCEBACK" CACHE STRING "Select the boundary conditions to be used at the walls (FINTERPOLATION,GZS,SIMPLEBOUNCEBACK,JUNKYANG)") -set(HEMELB_STEERING_HOST "CCS" CACHE STRING "Use a default host suffix for steering? (CCS, NGS2Leeds, NGS2Manchester, LONI, NCSA or blank)") option(HEMELB_DEPENDENCIES_SET_RPATH "Set runtime RPATH" ON) set(HEMELB_SUBPROJECT_MAKE_JOBS 1 CACHE INTEGER "Number of jobs to use for subproject build steps") -option(HEMELB_WAIT_ON_CONNECT "Wait for steering client" OFF) option(HEMELB_IMAGES_TO_NULL "Write images to null" OFF) option(HEMELB_USE_SSE3 "Use SSE3 intrinsics" OFF) set(HEMELB_COMPUTE_ARCHITECTURE "AMDBULLDOZER" @@ -60,7 +56,6 @@ ExternalProject_Add( -DHEMELB_USE_DEBUGGER=${HEMELB_USE_DEBUGGER} -DHEMELB_VALIDATE_GEOMETRY=${HEMELB_VALIDATE_GEOMETRY} -DHEMELB_LOG_LEVEL=${HEMELB_LOG_LEVEL} - -DHEMELB_STEERING_LIB=${HEMELB_STEERING_LIB} -DHEMELB_USE_MULTIMACHINE=${HEMELB_USE_MULTIMACHINE} -DHEMELB_BUILD_UNITTESTS=${HEMELB_BUILD_UNITTESTS} -DHEMELB_OPTIMISATION=${HEMELB_OPTIMISATION} @@ -72,7 +67,6 @@ ExternalProject_Add( -DMPI_C_NO_INTERROGATE=${MPI_C_NO_INTERROGATE} -DMPI_CXX_NO_INTERROGATE=${MPI_CXX_NO_INTERROGATE} -DHEMELB_USE_ALL_WARNINGS_GNU=${HEMELB_USE_ALL_WARNINGS_GNU} - -DHEMELB_STEERING_HOST=${HEMELB_STEERING_HOST} -DCTEMPLATE_USE_STATIC=${CTEMPLATE_USE_STATIC} -DCPPUNIT_USE_STATIC=${CPPUNIT_USE_STATIC} -DHEMELB_DEPENDENCIES_SET_RPATH=${HEMELB_DEPENDENCIES_SET_RPATH} diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 85870a32e..ada5fb5ae 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -19,7 +19,6 @@ option(HEMELB_BUILD_TESTS_UNIT "Build the unit-tests (HEMELB_BUILD_TESTS_ALL tak option(HEMELB_BUILD_TESTS_FUNCTIONAL "Build the functional tests (HEMELB_BUILD_TESTS_ALL takes precedence)" ON) option(HEMELB_USE_ALL_WARNINGS_GNU "Show all compiler warnings on development builds (gnu-style-compilers)" ON) option(HEMELB_DEPENDENCIES_SET_RPATH "Set runtime RPATH" ON) -option(HEMELB_WAIT_ON_CONNECT "Wait for steering client" OFF) option(HEMELB_BUILD_MULTISCALE "Build HemeLB Multiscale functionality" OFF) option(HEMELB_IMAGES_TO_NULL "Write images to null" OFF) option(HEMELB_USE_SSE3 "Use SSE3 intrinsics" OFF) @@ -31,8 +30,6 @@ set(HEMELB_READING_GROUP_SIZE 5 CACHE INTEGER "Number of cores to use to read geometry file.") set(HEMELB_LOG_LEVEL Info CACHE STRING "Log level, choose 'Critical', 'Error', 'Warning', 'Info', 'Debug' or 'Trace'" ) -set(HEMELB_STEERING_LIB basic - CACHE STRING "Steering library, choose 'basic' or 'none'" ) set(HEMELB_DEPENDENCIES_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../dependencies" CACHE FILEPATH "Path to find dependency find modules") set(HEMELB_DEPENDENCIES_INSTALL_PATH ${HEMELB_DEPENDENCIES_PATH} @@ -182,7 +179,6 @@ set(package_subdirs comm extraction reporting - steering lb geometry net @@ -229,7 +225,6 @@ if (HEMELB_BUILD_MULTISCALE) configuration extraction reporting - steering lb geometry net diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index 6aaeb72cc..817dc5345 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -41,14 +41,12 @@ SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, colloidController = NULL; latticeBoltzmannModel = NULL; - steeringCpt = NULL; propertyDataSource = NULL; propertyExtractor = NULL; simulationState = NULL; stepManager = NULL; netConcern = NULL; neighbouringDataManager = NULL; - steeringSessionId = options.GetSteeringSessionId(); fileManager = new hemelb::io::PathManager(options, IsCurrentProcTheIOProc(), GetProcessorCount()); simConfig = hemelb::configuration::SimConfig::New(fileManager->GetInputFile()); @@ -85,8 +83,6 @@ SimulationMaster::~SimulationMaster() delete latticeBoltzmannModel; delete inletValues; delete outletValues; - delete network; - delete steeringCpt; delete propertyExtractor; delete propertyDataSource; delete stabilityTester; @@ -123,7 +119,7 @@ int SimulationMaster::GetProcessorCount() } /** - * Initialises various elements of the simulation if necessary - steering, + * Initialises various elements of the simulation if necessary - * domain decomposition, and LBM. */ void SimulationMaster::Initialise() @@ -140,8 +136,7 @@ void SimulationMaster::Initialise() // Use a reader to read in the file. hemelb::log::Logger::Log("Loading file and decomposing geometry."); - hemelb::geometry::GeometryReader reader(hemelb::steering::SteeringComponent::RequiresSeparateSteeringCore(), - latticeType::GetLatticeInfo(), + hemelb::geometry::GeometryReader reader(latticeType::GetLatticeInfo(), timings, ioComms); hemelb::geometry::Geometry readGeometryData = reader.LoadAndDecompose(simConfig->GetDataFilePath()); @@ -192,15 +187,6 @@ void SimulationMaster::Initialise() } timings[hemelb::reporting::Timers::colloidInitialisation].Stop(); - // Initialise and begin the steering. - if (ioComms->OnIORank()) - { - network = new hemelb::steering::Network(steeringSessionId, timings); - } - else - { - network = NULL; - } stabilityTester = new hemelb::lb::StabilityTester(latticeData, ioComms, @@ -241,13 +227,6 @@ void SimulationMaster::Initialise() neighbouringDataManager->ShareNeeds(); neighbouringDataManager->TransferNonFieldDependentInformation(); - steeringCpt = new hemelb::steering::SteeringComponent(network, - &communicationNet, - simulationState, - simConfig, - unitConverter, - timings); - propertyDataSource = new hemelb::extraction::LbDataSourceIterator(latticeBoltzmannModel->GetPropertyCache(), *latticeData, @@ -282,7 +261,6 @@ void SimulationMaster::Initialise() stepManager->RegisterIteratedActorSteps(*inletValues, 1); stepManager->RegisterIteratedActorSteps(*outletValues, 1); - stepManager->RegisterIteratedActorSteps(*steeringCpt, 1); stepManager->RegisterIteratedActorSteps(*stabilityTester, 1); stepManager->RegisterCommsSteps(*stabilityTester, 1); if (entropyTester != NULL) @@ -301,10 +279,6 @@ void SimulationMaster::Initialise() stepManager->RegisterIteratedActorSteps(*propertyExtractor, 1); } - if (ioComms->OnIORank()) - { - stepManager->RegisterIteratedActorSteps(*network, 1); - } stepManager->RegisterCommsForAllPhases(*netConcern); } @@ -392,8 +366,6 @@ void SimulationMaster::DoTimeStep() entropyTester->MustFinishThisTimeStep(); if (incompressibilityChecker) incompressibilityChecker->MustFinishThisTimeStep(); - if (steeringCpt) - steeringCpt->MustFinishThisTimeStep(); } if (simulationState->GetTimeStep() % 100 == 0) diff --git a/Code/SimulationMaster.h b/Code/SimulationMaster.h index 2179f79b8..e5c3af678 100644 --- a/Code/SimulationMaster.h +++ b/Code/SimulationMaster.h @@ -11,7 +11,6 @@ #include "lb/lb.hpp" #include "lb/StabilityTester.h" #include "net/net.h" -#include "steering/SteeringComponent.h" #include "lb/EntropyTester.h" #include "lb/iolets/BoundaryValues.h" #include "util/UnitConverter.h" @@ -81,9 +80,6 @@ class SimulationMaster hemelb::reporting::BuildInfo build_info; typedef std::multimap MapType; - hemelb::steering::Network* network; - hemelb::steering::SteeringComponent* steeringCpt; - hemelb::lb::SimulationState* simulationState; /** Struct containing the configuration of various checkers/testers */ @@ -104,7 +100,6 @@ class SimulationMaster hemelb::net::phased::StepManager* stepManager; hemelb::net::phased::NetConcern* netConcern; - int steeringSessionId; static const hemelb::LatticeTimeStep FORCE_FLUSH_PERIOD=1000; }; diff --git a/Code/configuration/CommandLine.cc b/Code/configuration/CommandLine.cc index 9e8464c1b..81bbe4e72 100644 --- a/Code/configuration/CommandLine.cc +++ b/Code/configuration/CommandLine.cc @@ -13,7 +13,7 @@ namespace hemelb { CommandLine::CommandLine(int aargc, const char * const * const aargv) : - inputFile("input.xml"), outputDir(""), steeringSessionId(1), debugMode(false), argc(aargc), + inputFile("input.xml"), outputDir(""), debugMode(false), argc(aargc), argv(aargv) { @@ -37,11 +37,6 @@ namespace hemelb { outputDir = std::string(paramValue); } - else if (std::strcmp(paramName, "-ss") == 0) - { - char *dummy; - steeringSessionId = (unsigned int) (strtoul(paramValue, &dummy, 10)); - } else if (std::strcmp(paramName, "-debug") == 0) { debugMode = std::strcmp(paramName, "0") == 0 ? false : true; @@ -60,7 +55,6 @@ namespace hemelb ans.append("-in \t Path to the configuration xml file (default is config.xml)\n"); ans.append("-out \t Path to the output folder (default is based on input file, e.g. config_xml_results)\n"); ans.append("-i \t Number of images to create (default is 10)\n"); - ans.append("-ss \t Steering session identifier (default is 1)\n"); return ans; } } diff --git a/Code/configuration/CommandLine.h b/Code/configuration/CommandLine.h index fa756eae0..d46d104c8 100644 --- a/Code/configuration/CommandLine.h +++ b/Code/configuration/CommandLine.h @@ -21,7 +21,6 @@ namespace hemelb * Arguments should be: * - -in input xml configuration file (default input.xml) * - -out output folder (empty default, but the hemelb::io::PathManager will guess a value from the input file if not given.) - * - -ss steering session i.d. (default 1) */ class CommandLine { @@ -54,13 +53,6 @@ namespace hemelb { return (inputFile); } - /** - * @return A unique integer representing the steering session to which to attach. - */ - int GetSteeringSessionId() const - { - return (steeringSessionId); - } /** * @return Whether the user requested a debug mode. @@ -95,7 +87,6 @@ namespace hemelb private: std::string inputFile; //! local or full path to input file std::string outputDir; //! local or full path to input file - int steeringSessionId; //! unique identifier for steering session bool debugMode; //! Use debugger int argc; //! count of command line arguments, including program name const char * const * const argv; //! command line arguments diff --git a/Code/geometry/GeometryReader.cc b/Code/geometry/GeometryReader.cc index 4ac804ca9..bd8f67118 100644 --- a/Code/geometry/GeometryReader.cc +++ b/Code/geometry/GeometryReader.cc @@ -28,34 +28,10 @@ namespace hemelb namespace geometry { - GeometryReader::GeometryReader(const bool reserveSteeringCore, - const lb::lattices::LatticeInfo& latticeInfo, + GeometryReader::GeometryReader(const lb::lattices::LatticeInfo& latticeInfo, reporting::Timers &atimings, comm::Communicator::ConstPtr ioComm) : latticeInfo(latticeInfo), hemeLbComms(ioComm), timings(atimings) { - // This rank should participate in the domain decomposition if - // - there's no steering core (then all ranks are involved) - // - we're not on core 0 (the only core that might ever not participate) - // - there's only one processor (so core 0 has to participate) - - // Create our own group, without the root node if we're not running with it. - if (reserveSteeringCore && ioComm->Size() > 1) - { - participateInTopology = !ioComm->OnIORank(); - - std::vector lExclusions(1); - lExclusions[0] = 0; - comm::Group::Ptr computeGroup = hemeLbComms->GetGroup()->Exclude(lExclusions); - // Create a communicator just for the domain decomposition. - computeComms = ioComm->Create(computeGroup); - // Note that on the steering core, this is a null communicator. - } - else - { - participateInTopology = true; - computeComms = ioComm; - } - } GeometryReader::~GeometryReader() @@ -99,54 +75,43 @@ namespace hemelb log::Logger::Log("Reading file header"); ReadHeader(geometry.GetBlockCount()); - // Close the file - only the ranks participating in the topology need to read it again. + // Close the file file->Close(); timings[hemelb::reporting::Timers::initialDecomposition].Start(); log::Logger::Log("Beginning initial decomposition"); principalProcForEachBlock.resize(geometry.GetBlockCount()); - if (!participateInTopology) - { - // If we are the steering core, mark them all as unknown. - for (site_t block = 0; block < geometry.GetBlockCount(); ++block) - { - principalProcForEachBlock[block] = -1; - } - } - else + // Get an initial base-level decomposition of the domain macro-blocks over processors. + // This will later be improved upon by ParMetis. + decomposition::BasicDecomposition basicDecomposer(geometry, + latticeInfo, + hemeLbComms, + fluidSitesOnEachBlock); + basicDecomposer.Decompose(principalProcForEachBlock); + + if (ShouldValidate()) { - // Get an initial base-level decomposition of the domain macro-blocks over processors. - // This will later be improved upon by ParMetis. - decomposition::BasicDecomposition basicDecomposer(geometry, - latticeInfo, - computeComms, - fluidSitesOnEachBlock); - basicDecomposer.Decompose(principalProcForEachBlock); - - if (ShouldValidate()) - { - basicDecomposer.Validate(principalProcForEachBlock); - } + basicDecomposer.Validate(principalProcForEachBlock); } + timings[hemelb::reporting::Timers::initialDecomposition].Stop(); // Perform the initial read-in. log::Logger::Log("Reading in my blocks"); - if (participateInTopology) + // TODO: can we remove the close/reopen? As far as I can tell, + // it's only the set view we are changing + + // Reopen in the file. Read in blocks local to this node. + file = hemeLbComms->OpenFile(dataFilePath, MPI_MODE_RDONLY, fileInfo); + + ReadInBlocksWithHalo(geometry, principalProcForEachBlock, hemeLbComms->Rank()); + + if (ShouldValidate()) { - // Reopen in the file just between the nodes in the topology decomposition. Read in blocks - // local to this node. - file = computeComms->OpenFile(dataFilePath, MPI_MODE_RDONLY, fileInfo); - - ReadInBlocksWithHalo(geometry, principalProcForEachBlock, computeComms->Rank()); - - if (ShouldValidate()) - { - ValidateGeometry(geometry); - } + ValidateGeometry(geometry); } - + timings[hemelb::reporting::Timers::fileRead].Stop(); hemelb::log::Logger::Log("Begin optimising the domain decomposition."); @@ -154,19 +119,16 @@ namespace hemelb // Having done an initial decomposition of the geometry, and read in the data, we optimise the // domain decomposition. - if (participateInTopology) + log::Logger::Log("Beginning domain decomposition optimisation"); + OptimiseDomainDecomposition(geometry, principalProcForEachBlock); + log::Logger::Log("Ending domain decomposition optimisation"); + + if (ShouldValidate()) { - log::Logger::Log("Beginning domain decomposition optimisation"); - OptimiseDomainDecomposition(geometry, principalProcForEachBlock); - log::Logger::Log("Ending domain decomposition optimisation"); - - if (ShouldValidate()) - { - ValidateGeometry(geometry); - } - file->Close(); + ValidateGeometry(geometry); } - + file->Close(); + // Finish up - close the file, set the timings, deallocate memory. HEMELB_MPI_CALL(MPI_Info_free, (&fileInfo)); @@ -325,8 +287,8 @@ namespace hemelb Needs needs(geometry.GetBlockCount(), readBlock, - util::NumericalFunctions::min(READING_GROUP_SIZE, computeComms->Size()), - computeComms, + util::NumericalFunctions::min(READING_GROUP_SIZE, hemeLbComms->Size()), + hemeLbComms, ShouldValidate()); timings[hemelb::reporting::Timers::readBlocksPrelim].Stop(); @@ -368,9 +330,9 @@ namespace hemelb proc_t readingCore = GetReadingCoreForBlock(blockNumber); { - comm::Async requestQ(computeComms); + comm::Async requestQ(hemeLbComms); - if (readingCore == computeComms->Rank()) + if (readingCore == hemeLbComms->Rank()) { timings[hemelb::reporting::Timers::readBlock].Start(); // Read the data. @@ -381,7 +343,7 @@ namespace hemelb for (std::vector::const_iterator receiver = procsWantingThisBlock.begin(); receiver != procsWantingThisBlock.end(); receiver++) { - if (*receiver != computeComms->Rank()) + if (*receiver != hemeLbComms->Rank()) { requestQ.Isend(compressedBlockData, *receiver); } @@ -589,7 +551,7 @@ namespace hemelb proc_t GeometryReader::GetReadingCoreForBlock(site_t blockNumber) { return proc_t(blockNumber % util::NumericalFunctions::min(READING_GROUP_SIZE, - computeComms->Size())); + hemeLbComms->Size())); } /** @@ -648,13 +610,13 @@ namespace hemelb // Reduce using a minimum to find the actual processor for each site (ignoring the // invalid entries). - std::vector procForSiteRecv = computeComms->AllReduce(myProcForSite, MPI_MIN); - std::vector siteDataRecv = computeComms->AllReduce(dummySiteData, MPI_MIN); + std::vector procForSiteRecv = hemeLbComms->AllReduce(myProcForSite, MPI_MIN); + std::vector siteDataRecv = hemeLbComms->AllReduce(dummySiteData, MPI_MIN); for (site_t site = 0; site < geometry.GetSitesPerBlock(); ++site) { - if (procForSiteRecv[site] == ConvertTopologyRankToGlobalRank(computeComms->Rank()) - && (myProcForSite[site] != ConvertTopologyRankToGlobalRank(computeComms->Rank()))) + if (procForSiteRecv[site] == hemeLbComms->Rank() + && (myProcForSite[site] != hemeLbComms->Rank())) { log::Logger::Log("Other cores think this core has site %li on block %li but it disagrees.", site, @@ -756,7 +718,7 @@ namespace hemelb const std::vector& procForEachBlock) { decomposition::OptimisedDecomposition optimiser(timings, - computeComms, + hemeLbComms, geometry, latticeInfo, procForEachBlock, @@ -803,7 +765,7 @@ namespace hemelb // going to be moved to the current proc. idx_t moveIndex = 0; - for (proc_t fromProc = 0; fromProc < computeComms->Size(); ++fromProc) + for (proc_t fromProc = 0; fromProc < hemeLbComms->Size(); ++fromProc) { for (idx_t moveNumber = 0; moveNumber < movesPerProc[fromProc]; ++moveNumber) { @@ -811,15 +773,15 @@ namespace hemelb idx_t toProc = movesList[3 * moveIndex + 2]; ++moveIndex; - if (toProc == (idx_t) computeComms->Rank()) + if (toProc == (idx_t) hemeLbComms->Rank()) { - newProcForEachBlock[block] = computeComms->Rank(); + newProcForEachBlock[block] = hemeLbComms->Rank(); } } } // Reread the blocks into the GlobalLatticeData now. - ReadInBlocksWithHalo(geometry, newProcForEachBlock, computeComms->Rank()); + ReadInBlocksWithHalo(geometry, newProcForEachBlock, hemeLbComms->Rank()); } void GeometryReader::ImplementMoves(Geometry& geometry, @@ -845,8 +807,7 @@ namespace hemelb if (geometry.Blocks[block].Sites[siteIndex].targetProcessor != SITE_OR_BLOCK_SOLID) { // ... set its rank to be the rank it had before optimisation. - geometry.Blocks[block].Sites[siteIndex].targetProcessor - = ConvertTopologyRankToGlobalRank(originalProc); + geometry.Blocks[block].Sites[siteIndex].targetProcessor = originalProc; } } } @@ -856,7 +817,7 @@ namespace hemelb idx_t moveIndex = 0; // For each source proc, go through as many moves as it had. - for (proc_t fromProc = 0; fromProc < computeComms->Size(); ++fromProc) + for (proc_t fromProc = 0; fromProc < hemeLbComms->Size(); ++fromProc) { for (idx_t moveNumber = 0; moveNumber < movesFromEachProc[fromProc]; ++moveNumber) { @@ -872,8 +833,7 @@ namespace hemelb // lFromProc. if (ShouldValidate()) { - if (geometry.Blocks[block].Sites[site].targetProcessor - != ConvertTopologyRankToGlobalRank((proc_t) fromProc)) + if (geometry.Blocks[block].Sites[site].targetProcessor != proc_t(fromProc)) { log::Logger::Log("Block %ld, site %ld from move %u was originally on proc %i, not proc %u.", block, @@ -885,8 +845,7 @@ namespace hemelb } // Implement the move. - geometry.Blocks[block].Sites[site].targetProcessor - = ConvertTopologyRankToGlobalRank((proc_t) toProc); + geometry.Blocks[block].Sites[site].targetProcessor = proc_t(toProc); } ++moveIndex; @@ -894,15 +853,6 @@ namespace hemelb } } - proc_t GeometryReader::ConvertTopologyRankToGlobalRank(proc_t topologyRankIn) const - { - // If the global rank is not equal to the topology rank, we are not using rank 0 for - // LBM. - return (hemeLbComms->Rank() == computeComms->Rank()) - ? topologyRankIn - : (topologyRankIn + 1); - } - bool GeometryReader::ShouldValidate() const { #ifdef HEMELB_VALIDATE_GEOMETRY diff --git a/Code/geometry/GeometryReader.h b/Code/geometry/GeometryReader.h index 0c91f9a40..440645b77 100644 --- a/Code/geometry/GeometryReader.h +++ b/Code/geometry/GeometryReader.h @@ -32,7 +32,7 @@ namespace hemelb public: typedef util::Vector3D BlockLocation; - GeometryReader(const bool reserveSteeringCore, const lb::lattices::LatticeInfo&, + GeometryReader(const lb::lattices::LatticeInfo&, reporting::Timers &timings, comm::Communicator::ConstPtr ioComm); ~GeometryReader(); @@ -147,8 +147,6 @@ namespace hemelb const std::vector& movesFromEachProc, const std::vector& movesList) const; - proc_t ConvertTopologyRankToGlobalRank(proc_t topologyRank) const; - /** * True if we should validate the geometry. * @return @@ -167,10 +165,7 @@ namespace hemelb //! Information about the file, to give cues and hints to MPI. comm::Communicator::ConstPtr hemeLbComms; //! HemeLB's main communicator - comm::Communicator::ConstPtr computeComms; //! Communication info for all ranks that will need a slice of the geometry (i.e. all non-steering cores) - //! True iff this rank is participating in the domain decomposition. - bool participateInTopology; - + //! The number of fluid sites on each block in the geometry std::vector fluidSitesOnEachBlock; //! The number of bytes each block takes up while still compressed. diff --git a/Code/reporting/BuildInfo.h.in b/Code/reporting/BuildInfo.h.in index d8b5fe34f..7c433d120 100644 --- a/Code/reporting/BuildInfo.h.in +++ b/Code/reporting/BuildInfo.h.in @@ -12,7 +12,6 @@ namespace hemelb namespace reporting { static const std::string mercurial_revision_number="@HEMELB_REVISION_NUMBER@"; - static const std::string steering_lib="@HEMELB_STEERING_LIB@"; static const std::string build_type="@CMAKE_BUILD_TYPE@"; static const std::string optimisation="@HEMELB_OPTIMISATION@"; static const std::string use_sse3="@HEMELB_USE_SSE3@"; @@ -33,7 +32,6 @@ namespace hemelb void Report(ctemplate::TemplateDictionary& dictionary){ ctemplate::TemplateDictionary *build = dictionary.AddSectionDictionary("BUILD"); build->SetValue("REVISION", mercurial_revision_number); - build->SetValue("STEERING", steering_lib); build->SetValue("TYPE", build_type); build->SetValue("OPTIMISATION", optimisation); build->SetValue("USE_SSE3", use_sse3); diff --git a/Code/reporting/Timers.h b/Code/reporting/Timers.h index cf641d6ea..ed3dbcc41 100644 --- a/Code/reporting/Timers.h +++ b/Code/reporting/Timers.h @@ -102,7 +102,6 @@ namespace hemelb readBlock, readBlocksPrelim, readBlocksAll, - steeringWait, //!< Time spent waiting for a steering client in wait on connect mode. moveForcingNumbers, moveForcingData, blockRequirements, @@ -217,7 +216,7 @@ namespace hemelb { "Total", "Seed Decomposition", "Domain Decomposition", "File Read", "Re Read", "Unzip", "Moves", "Parmetis", "Lattice Data initialisation", "Lattice Boltzmann", "LB calc only", "Monitoring", "MPI Send", "MPI Wait", "Simulation total", "Reading communications", "Parsing", "Read IO", "Read Blocks prelim", - "Read blocks all", "Steering Client Wait", "Move Forcing Counts", "Move Forcing Data", "Block Requirements", + "Read blocks all", "Move Forcing Counts", "Move Forcing Data", "Block Requirements", "Move Counts Sending", "Move Data Sending", "Populating moves list for decomposition optimisation", "Initial geometry reading", "Colloid initialisation", "Colloid position communication", "Colloid velocity communication", "Colloid force calculations", "Colloid calculations for updating", diff --git a/Code/resources/report.txt.ctp b/Code/resources/report.txt.ctp index 8b74dc4e5..a5c53bc7f 100644 --- a/Code/resources/report.txt.ctp +++ b/Code/resources/report.txt.ctp @@ -27,7 +27,6 @@ Name Local Min Mean Max {{#BUILD}} Revision number:{{REVISION}} -Steering mode: {{STEERING}} Build type: {{TYPE}} Optimisation level: {{OPTIMISATION}} Use SSE3: {{USE_SSE3}} diff --git a/Code/resources/report.xml.ctp b/Code/resources/report.xml.ctp index 6e8ee0e92..635ecf1ac 100644 --- a/Code/resources/report.xml.ctp +++ b/Code/resources/report.xml.ctp @@ -10,7 +10,6 @@ {{#BUILD}} {{REVISION}} - {{STEERING}} {{TYPE}} {{OPTIMISATION}} {{USE_SSE3}} diff --git a/Code/steering/CMakeLists.txt b/Code/steering/CMakeLists.txt deleted file mode 100644 index cb61a38c1..000000000 --- a/Code/steering/CMakeLists.txt +++ /dev/null @@ -1,45 +0,0 @@ - -# This file is part of HemeLB and is Copyright (C) -# the HemeLB team and/or their institutions, as detailed in the -# file AUTHORS. This software is provided under the terms of the -# license in the file LICENSE. -set(HEMELB_STEERING_HOST "CCS" CACHE STRING "Use a default host suffix for steering? (CCS, NGS2Leeds, NGS2Manchester, LONI, NCSA or blank)") -if(HEMELB_STEERING_HOST) - add_definitions("-D${HEMELB_STEERING_HOST}") -endif() - -if(HEMELB_STEERING_LIB MATCHES [Bb]asic) - find_package (Threads) - set(steerers - basic/ClientConnection.cc - basic/HttpPost.cc - basic/Network.cc - basic/SimulationParameters.cc - basic/SteeringComponentB.cc - ) - add_definitions(-DHEMELB_STEERING_LIB=basic) -elseif(HEMELB_STEERING_LIB MATCHES [Nn]one) - set(steerers - none/ClientConnection.cc - none/Network.cc - none/SteeringComponentN.cc - ) - add_definitions(-DHEMELB_STEERING_LIB=none) -else() - message("Unrecognised steering mode, using basic") - set(steerers - basic/ClientConnection.cc - basic/HttpPost.cc - basic/Network.cc - basic/SimulationParameters.cc - basic/SteeringComponentB.cc - ) - add_definitions(-DHEMELB_STEERING_LIB=basic) -endif() - -add_library(hemelb_steering - common/SteeringComponentC.cc - ${steerers} -) - -target_link_libraries(hemelb_steering ${CMAKE_THREAD_LIBS_INIT}) diff --git a/Code/steering/ClientConnection.h b/Code/steering/ClientConnection.h deleted file mode 100644 index 308d2f7d1..000000000 --- a/Code/steering/ClientConnection.h +++ /dev/null @@ -1,45 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_CLIENTCONNECTION_H -#define HEMELB_STEERING_CLIENTCONNECTION_H - -#include -#include -#include "reporting/Timers.h" - -namespace hemelb -{ - namespace steering - { - class ClientConnection - { - public: - ClientConnection(int iSteeringSessionId, reporting::Timers & timings); - ~ClientConnection(); - - int GetWorkingSocket(); - - void ReportBroken(int iSocketNum); - - private: - static const in_port_t MYPORT = 65250; - static const unsigned int CONNECTION_BACKLOG = 10; - - int mCurrentSocket; - int mListeningSocket; - bool mIsBroken; - // Use a semaphore to make sure that we don't create two new connections - // when a broken one is reported simultaneously by two separate threads - // (for example). - std::mutex mIsBusy; - reporting::Timers & timers; - - }; - } -} - -#endif /* HEMELB_STEERING_CLIENTCONNECTION_H */ diff --git a/Code/steering/Network.h b/Code/steering/Network.h deleted file mode 100644 index 2944e1a5e..000000000 --- a/Code/steering/Network.h +++ /dev/null @@ -1,54 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_NETWORK_H -#define HEMELB_STEERING_NETWORK_H - -#include - -#include "steering/ClientConnection.h" -#include "net/IteratedAction.h" - -namespace hemelb -{ - namespace steering - { - - class Network : public net::IteratedAction - { - public: - Network(int iSteeringSessionId, reporting::Timers & timings); - - // Receive a bytestream of known length from a socket into a buffer. - bool recv_all(char *buf, const int length); - - // Send all bytes from a buffer of known length over a socket. - bool send_all(const char *buf, const int length); - - /** - * Use the time between MPI send and MPI receives being completed to perform the - * Network sends. - */ - void PreReceive(); - - bool IsConnected(); - - private: - void Break(int socket); - - long sendInternal(const char* data, long length, int socket); - - ClientConnection clientConnection; - - // Buffers to keep the data from partial sends and receives. - std::string sendBuf; - std::string recvBuf; - }; - - } -} - -#endif // HEMELB_STEERING_NETWORK_H diff --git a/Code/steering/SteeringComponent.h b/Code/steering/SteeringComponent.h deleted file mode 100644 index 6d6a95c92..000000000 --- a/Code/steering/SteeringComponent.h +++ /dev/null @@ -1,94 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_STEERINGCOMPONENT_H -#define HEMELB_STEERING_STEERINGCOMPONENT_H - -#include "net/net.h" -#include "net/CollectiveAction.h" -#include "lb/SimulationState.h" -#include "configuration/SimConfig.h" -#include "steering/Network.h" - -namespace hemelb -{ - namespace steering - { - // TODO: remove most of these - enum parameter - { - SceneCentreX = 0, - SceneCentreY = 1, - SceneCentreZ = 2, - Longitude = 3, - Latitude = 4, - Zoom = 5, - Brightness = 6, - PhysicalVelocityThresholdMax = 7, - PhysicalStressThrehsholdMaximum = 8, - PhysicalPressureThresholdMinimum = 9, - PhysicalPressureThresholdMaximum = 10, - GlyphLength = 11, - PixelsX = 12, - PixelsY = 13, - NewMouseX = 14, - NewMouseY = 15, - SetIsTerminal = 16, - Mode = 17, - StreaklinePerSimulation = 18, - StreaklineLength = 19, - MaxFramerate=20, - SetDoRendering = 21 - }; - - /** - * SteeringComponent - class for passing steering data to all nodes. - * - * We pass this data at regular intervals. No initial action is required by all nodes, and - * we only need to pass from the top-most node (which handles network communication) downwards, - * on one iteration between each pair of consecutive depths. - */ - class SteeringComponent : public net::CollectiveAction - { - public: - SteeringComponent(Network* iNetwork, - net::Net * iNet, - lb::SimulationState * iSimState, - configuration::SimConfig* iSimConfig, - const util::UnitConverter* iUnits, - reporting::Timers& timings); - - static bool RequiresSeparateSteeringCore(); - - /* - * This function initialises all of the steering parameters, on all nodes. - * Although this appears to be part of the removed post-unstable reset functionality, it is also used during initialisation - * so is retained #244 - */ - void ClearValues(); - - void PreSend(); - void Send(); - void PostReceive(); - - private: - void AssignValues(); - - const static int STEERABLE_PARAMETERS = 21; - const static int RootRank = 0; - - bool isConnected; - - Network* mNetwork; - lb::SimulationState* mSimState; - std::vector privateSteeringParams; - const util::UnitConverter* mUnits; - configuration::SimConfig* simConfig; - }; - } -} - -#endif /* HEMELB_STEERING_STEERINGCOMPONENT_H */ diff --git a/Code/steering/basic/ClientConnection.cc b/Code/steering/basic/ClientConnection.cc deleted file mode 100644 index d11415039..000000000 --- a/Code/steering/basic/ClientConnection.cc +++ /dev/null @@ -1,167 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#include -#include -#include -#include - -#include "debug/Debugger.h" -#include "log/Logger.h" -#include "steering/ClientConnection.h" -#include "steering/basic/HttpPost.h" - -namespace hemelb -{ - namespace steering - { - ClientConnection::ClientConnection(int iSteeringSessionId, reporting::Timers & timings) - :mIsBusy(), timers(timings) - { - // Write the name of this machine to a file. - - { - char thisMachineName[255]; - gethostname(thisMachineName, 255); - std::FILE *f = std::fopen("env_details.asc", "w"); - std::fprintf(f, "%s\n", thisMachineName); - std::fclose(f); - } - - mCurrentSocket = -1; - mIsBroken = false; - - // Create the socket. - mListeningSocket = socket(AF_INET, SOCK_STREAM, 0); - if (mListeningSocket == -1) - { - perror("socket"); - exit(1); - } - - // Make the socket reusable. - int yes = 1; - if (setsockopt(mListeningSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1) - { - perror("setsockopt"); - exit(1); - } - - // Bind to the socket. - { - struct sockaddr_in my_address; - - my_address.sin_family = AF_INET; - my_address.sin_port = htons((in_port_t) MYPORT); - my_address.sin_addr.s_addr = INADDR_ANY; - memset(my_address.sin_zero, '\0', sizeof my_address.sin_zero); - - if (bind(mListeningSocket, (struct sockaddr *) &my_address, sizeof my_address) == -1) - { - perror("bind"); - exit(1); - } - } - - // Mark the socket as accepting incoming connections. - if (listen(mListeningSocket, CONNECTION_BACKLOG) == -1) - { - perror("listen"); - exit(1); - } - } - - ClientConnection::~ClientConnection() - { - close(mCurrentSocket); - close(mListeningSocket); - } - - int ClientConnection::GetWorkingSocket() - { - // Lock the mutex and release it when this goes out of scope - std::lock_guard lock(mIsBusy); - - // If we haven't yet had a socket, or the current one is broken, open a new one. - if (mCurrentSocket < 0 || mIsBroken) - { - // Accept an incoming connection from the client. - struct sockaddr_in clientAddress; - socklen_t socketSize = sizeof (clientAddress); - - int lOldSocket = mCurrentSocket; - - // Make the socket non-blocking, just while we try to accept on it, then set - // it back to what it was before. - { - int flags = fcntl(mListeningSocket, F_GETFL, 0); - if (flags == -1) - { - flags = 0; - } -#ifndef HEMELB_WAIT_ON_CONNECT - if (fcntl(mListeningSocket, F_SETFL, flags | O_NONBLOCK) < 0) - { - perror("flags"); - } -#else - log::Logger::Log("Waiting for steering client connection"); - timers[reporting::Timers::steeringWait].Start(); - -#endif - // Try to accept a socket (from the non-blocking socket) - mCurrentSocket - = accept(mListeningSocket, (struct sockaddr *) &clientAddress, &socketSize); -#ifdef HEMELB_WAIT_ON_CONNECT - timers[reporting::Timers::steeringWait].Stop(); - log::Logger::Log("Continuing after receiving steering connection."); -#endif - // We've got a socket - make that socket non-blocking too. - if (mCurrentSocket > 0) - { - log::Logger::Log("Steering client connected"); - flags = fcntl(mCurrentSocket, F_GETFL, 0); - if (flags == -1) - { - flags = 0; - } - if (fcntl(mCurrentSocket, F_SETFL, flags | O_NONBLOCK) < 0) - { - perror("flags"); - } - } - } - - // If we had a socket before, close it. - if (lOldSocket > 0) - { - close(lOldSocket); - } - - // We've only just created the socket so it shouldn't be broken. - mIsBroken = false; - } - - int lRet = mCurrentSocket; - - return lRet; - } - - void ClientConnection::ReportBroken(int iSocketNum) - { - // Lock the mutex and release it when this goes out of scope - std::lock_guard lock(mIsBusy); - if (mCurrentSocket == iSocketNum) - { - mIsBroken = true; - } - } - - } -} diff --git a/Code/steering/basic/HttpPost.cc b/Code/steering/basic/HttpPost.cc deleted file mode 100644 index 6c3145dec..000000000 --- a/Code/steering/basic/HttpPost.cc +++ /dev/null @@ -1,277 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "debug/Debugger.h" -#include "log/Logger.h" -#include "steering/basic/HttpPost.h" - -using namespace std; - -namespace hemelb -{ - namespace steering - { - - void HttpPost::get_host_details(char* rank_0_host_details) - { - // This really does modify the strings passed in. Without - // checking their lengths, of course. - char hostname[256]; - - // On specific machines, just get the host name and insert it into the rank_0_host_details parameter -#ifdef NGS2Leeds - gethostname(hostname, 256); - std::sprintf(rank_0_host_details, "%s.ngs.leeds.ac.uk:%i", hostname, MYPORT); -#elif NGS2Manchester - gethostname(hostname, 256); - std::sprintf(rank_0_host_details, "%s.vidar.ngs.manchester.ac.uk:%i", hostname, MYPORT); -#elif CCS - gethostname(hostname, 256); - std::sprintf(rank_0_host_details, "%s.chem.ucl.ac.uk:%i", hostname, MYPORT); -#elif LONI - gethostname(hostname, 256); - std::sprintf(rank_0_host_details, "%s:%i", hostname, MYPORT); -#elif NCSA - gethostname(hostname, 256); - std::sprintf(rank_0_host_details, "%s.ncsa.teragrid.org:%i", hostname, MYPORT); -#else - - // If not on a specific known machine, need to do something more clever. - struct utsname name; - - if (uname(&name) < 0) - { - std::fprintf(stderr, "STEER: Get_fully_qualified_hostname: uname failed\n"); - exit(0x0); - } - - // Get information about our host. - struct hostent *host = gethostbyname(name.nodename); - char ip_addr[16]; - - // Now go through every associated IP adress, and use the first valid, public one. - for (int address_id = 0; address_id < 4; address_id++) - { - // If no address, break. - if (host->h_addr_list[address_id] == NULL) - break; - - std::printf("checking Address ID %i...\n", address_id); - - // If IP4, print details. - if (host->h_length != 4) - { - std::printf("address is not IP4..\n"); - continue; - } - - std::sprintf(ip_addr, - "%d.%d.%d.%d", - (unsigned char) (host->h_addr_list[address_id][0]), - (unsigned char) (host->h_addr_list[address_id][1]), - (unsigned char) (host->h_addr_list[address_id][2]), - (unsigned char) (host->h_addr_list[address_id][3])); - - std::printf("NAME %s IP %s\n", host->h_name, ip_addr); - - // Private addresses (see RFC-1918). - if ((unsigned char) (host->h_addr_list[address_id][0]) == 10) - { - std::printf("IP %s is not public..\n", ip_addr); - continue; - } - - // Loopback addresses. - if ((unsigned char) (host->h_addr_list[address_id][0]) == 127) - { - std::printf("IP %s is not public..\n", ip_addr); - continue; - } - - // Private addresses. - if ((unsigned char) (host->h_addr_list[address_id][0]) == 172 - && (unsigned char) (host->h_addr_list[address_id][1]) >= 16 - && (unsigned char) (host->h_addr_list[address_id][1]) < 32) - { - std::printf("IP %s is not public..\n", ip_addr); - continue; - } - - // Private IP addresses. - if ((unsigned char) (host->h_addr_list[address_id][0]) == 192 - && (unsigned char) (host->h_addr_list[address_id][1]) == 168) - { - std::printf("IP %s is not public..\n", ip_addr); - continue; - } - } - - std::sprintf(rank_0_host_details, "%s:%i (IP %s)", host->h_name, MYPORT, ip_addr); -#endif - - log::Logger::Log("MPI public interface details - %s", rank_0_host_details); - } - - ssize_t HttpPost::Send_Request(int iSocket, const char *iMessage) - { - return send(iSocket, iMessage, strlen(iMessage), 0); - } - - int HttpPost::request(const char* hostname, const in_port_t port, const char* api, const char* resourceid) - { - // Get the host name to communicate with. - char host_name[1024]; - get_host_details(host_name); - - // Attempt to get a socket to use. - int sock = socket(AF_INET, SOCK_STREAM, 0); - if (sock == -1) - { - return -100; - } - - sockaddr_in sin; - sin.sin_family = AF_INET; - sin.sin_port = htons((in_port_t) port); - - // Get name for the other end of the connection. - struct hostent * host_addr = gethostbyname(hostname); - - if (host_addr == NULL) - { - return -103; - } - - sin.sin_addr.s_addr = * ((int*) *host_addr->h_addr_list); - - // Attempt to connect, but don't try for too long. - timeval tv; - memset(&tv, 0, sizeof (tv)); // so valgrind knows the whole struct is initialised. - tv.tv_sec = 2; - tv.tv_usec = 0; - - setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof (tv)); - setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof (tv)); - - if (connect(sock, (const struct sockaddr *) &sin, sizeof(sockaddr_in)) == -1) - { - return -101; - } - - // Now we perform the actual sending. - Send_Request(sock, "POST "); - Send_Request(sock, api); - Send_Request(sock, resourceid); - Send_Request(sock, " HTTP/1.0\r\n"); - Send_Request(sock, "Accept: */*\r\n"); - Send_Request(sock, "User-Agent: Mozilla/4.0\r\n"); - - char content_header[100]; - std::sprintf(content_header, "Content-Length: %d\r\n", int (std::strlen(host_name))); - - Send_Request(sock, content_header); - Send_Request(sock, "Accept-Language: en-us\r\n"); - Send_Request(sock, "Accept-Encoding: gzip, deflate\r\n"); - Send_Request(sock, "Host: "); - Send_Request(sock, "hostname"); - Send_Request(sock, "\r\n"); - Send_Request(sock, "Content-Type: text/plain\r\n"); - Send_Request(sock, "\r\n"); - Send_Request(sock, host_name); - Send_Request(sock, "\r\n"); - - /* If you need to send a basic authorization - * string Auth = "username:password"; - * Figureout a way to encode test into base64 ! - * string AuthInfo = base64_encode(reinterpret_cast(Auth.c_str()),Auth.length()); - * string sPassReq = "Authorization: Basic " + AuthInfo; - * Send_Request(sock, sPassReq.c_str()); - * */ - - // Receive 1 character at a time until a whole response is recovered. - int line_length = 0; - bool loop = true; - bool bHeader = false; - - string message; - - while (loop) - { - char c1[1]; - - // receive 1 char from the socket - ssize_t l = recv(sock, c1, 1, 0); - - // An error occurred. - if (l < 0) - loop = false; - - if (c1[0] == '\n') - { - // Received a newline. If the only character on this line, we're at the end of the - // response. - if (line_length == 0) - loop = false; - - line_length = 0; - - // if the message contains a success response code, we are in the header. - if (message.find("201 Created") != string::npos) - bHeader = true; - } - else if (c1[0] != '\r') - { - // Didn't get newline and didn't get carriage return. - line_length++; - } - - message += c1[0]; - } - - message = ""; - - // If we are indeed reading a header, read the rest of the message. - if (bHeader) - { - char p[1024]; - - ssize_t l; - - while ( (l = recv(sock, p, 1023, 0)) > 0) - { - p[l] = '\0'; - message += p; - } - } - else - { - return -102; - } - - return 0; - } - - } -} - diff --git a/Code/steering/basic/HttpPost.h b/Code/steering/basic/HttpPost.h deleted file mode 100644 index 7775ca193..000000000 --- a/Code/steering/basic/HttpPost.h +++ /dev/null @@ -1,56 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_BASIC_HTTPPOST_H -#define HEMELB_STEERING_BASIC_HTTPPOST_H - -namespace hemelb -{ - namespace steering - { - - class HttpPost - { - public: - /** - * Send an HTTP Request. - * - * @param hostname The name of the host. - * @param port The port to send to. - * @param api The API name to include in the request. - * @param resourceid An identifier for the desired resource. - * @return Error code. 0 on success. - */ - static int request(const char* hostname, - const in_port_t port, - const char* api, - const char* resourceid); - - private: - /** - *Get details and ip address for this host. - * - * @param rank_0_host_details - * @param ip_addr - */ - static void get_host_details(char* rank_0_host_details); - - /** - * Helper method to send a request. - * - * @param iSocket - * @param iMessage - * @return - */ - static ssize_t Send_Request(int iSocket, const char *iMessage); - - static const int MYPORT = 65250; - }; - - } -} - -#endif // HEMELB_STEERING_BASIC_HTTPPOST_H diff --git a/Code/steering/basic/Network.cc b/Code/steering/basic/Network.cc deleted file mode 100644 index 370cce47b..000000000 --- a/Code/steering/basic/Network.cc +++ /dev/null @@ -1,257 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "debug/Debugger.h" -#include "log/Logger.h" -#include "steering/Network.h" -#include "util/utilityFunctions.h" - -namespace hemelb -{ - namespace steering - { - Network::Network(int steeringSessionId, reporting::Timers & timings) : - clientConnection(steeringSessionId, timings) - { - - } - - /** - * Receive a bytestream of known length from a socket into a buffer. - * - * @param sockid - * @param buf - * @param length - * @return Returns true if we have successfully provided that much data. - */ - bool Network::recv_all(char *buf, const int length) - { - // Get a socket - int socketToClient = clientConnection.GetWorkingSocket(); - - if (socketToClient < 0) - { - return false; - } - - ssize_t bytesGot = 0; - // If we have some buffered data to receive, include that in our count. - if (recvBuf.length() > 0) - { - bytesGot += recvBuf.length(); - } - - log::Logger::Log("Steering component will try to receive %d bytes, has %d so far", - length, - bytesGot); - // While some data left to be received... - while (bytesGot < length) - { - // Receive some data (up to the remaining length) - ssize_t n = recv(socketToClient, buf + bytesGot, length - bytesGot, 0); - - // If there was an error, report it and return. - if (n <= 0) - { - // If there was no data and it wasn't simply that the socket would block, - // raise an error. - if (errno != EAGAIN) - { - log::Logger::Log("Steering component: broken network pipe... (%s)", - strerror(errno)); - Break(socketToClient); - } - else - { - // If we'd already received some data, store this in the buffer for the start of the - // next receive. - if (bytesGot > 0) - { - // The newly-received-byte count is the total received byte count minus the length - // of the buffer. - long int numNewBytes = bytesGot - recvBuf.length(); - recvBuf.append(buf + recvBuf.length(), numNewBytes); - log::Logger::Log("Steering component: blocked socket"); - } - } - log::Logger::Log("Steering component exiting after incomplete reception"); - // We didn't fully receive. - return false; - } - else - { - bytesGot += n; - log::Logger::Log("Steering component: received bytes... (New total %d)", bytesGot); - } - } - log::Logger::Log("Steering component is happy with what it has received"); - // Successfully received what we needed to. Now use the buffer to fill in the gaps, if - // we were using the buffer at the front of the received data. - if (recvBuf.length() > 0) - { - // The length of buffer to use is the minimum of the buffer size and the length of - // the string. - int copyLength = util::NumericalFunctions::min((int) recvBuf.length(), length); - - memcpy(buf, recvBuf.c_str(), copyLength); - - // Empty that much of the buffer. - recvBuf.erase(0, copyLength); - } - - return true; - } - - void Network::PreReceive() - { - // Calling send_all will attempt to flush the send buffer. - send_all(NULL, 0); - } - - bool Network::IsConnected() - { - int res = clientConnection.GetWorkingSocket(); - return res > 0; - } - - /** - * Send all bytes from a buffer of known length over a socket. - * - * @param sockid - * @param buf - * @param length - * @return Returns the number of bytes sent or -1 on failure. - */ - bool Network::send_all(const char *buf, const int length) - { - // Get a socket. - int socketToClient = clientConnection.GetWorkingSocket(); - - // If there's no such socket, we don't have a connection. Nothing to do. - if (socketToClient < 0) - { - return false; - } - log::Logger::Log("Steering component will try to send %d new bytes and a buffer of %d", - length, - sendBuf.length()); - // If we have buffered strings to be sent, send those first. - if (sendBuf.length() > 0) - { - long sent = sendInternal(sendBuf.c_str(), sendBuf.length(), socketToClient); - - // Broken socket? - if (sent < 0) - { - return false; - } - - // Did sending block? We'll try again next time. - if (sent < (long) sendBuf.length()) - { - sendBuf.erase(0, sent); - - // If the sending blocks, just ignore this most recent frame. This way, we stop the - // memory usage of the steering process spiralling, and keep current the images delivered - // to the client. - // What we *would* do is sendBuf.append(buf, length); - - log::Logger::Log("Steering component could not send all buffer, managed %d bytes", - sent); - return true; - } - // If not, we sent the whole buffer. - else - { - sendBuf.clear(); - } - } - log::Logger::Log("Steering component sent all the buffer, sending new data"); - // If we sent the whole buffer, try to send the new data. - long sent_bytes = sendInternal(buf, length, socketToClient); - - // Is the socket broken? - if (sent_bytes < 0) - { - log::Logger::Log("Steering component socket broke sending new bytes"); - return false; - } - // Did the socket block? Still return true, because we'll try again next time. - else if (sent_bytes < length) - { - log::Logger::Log("Steering component socket blocked after sending %d bytes, adding %d bytes to buffer", - sent_bytes, - length - sent_bytes); - sendBuf.append(buf + sent_bytes, length - sent_bytes); - } - - // Otherwise we sent the whole thing. - return true; - } - - void Network::Break(int socket) - { - clientConnection.ReportBroken(socket); - - recvBuf.clear(); - sendBuf.clear(); - } - - /** - * Returns -1 for a broken socket, otherwise the number of bytes we could send before blocking. - * - * @param data - * @param length - * @param socket - * @return - */ - long Network::sendInternal(const char* data, long length, int socket) - { - long sent_bytes = 0; - - while (sent_bytes < length) - { - long n = send(socket, data + sent_bytes, length - sent_bytes, 0); - - if (n <= 0) - { - // Distinguish between cases where the pipe fails because it'd block - // (No problem, we'll try again later) or because the pipe is broken. - if (errno == EWOULDBLOCK) - { - return sent_bytes; - } - else - { - log::Logger::Log("Network send had broken pipe... (%s)", strerror(errno)); - Break(socket); - - return -1; - } - } - else - { - sent_bytes += n; - } - } - - return sent_bytes; - } - - } -} diff --git a/Code/steering/basic/SimulationParameters.cc b/Code/steering/basic/SimulationParameters.cc deleted file mode 100644 index a5b7a2f29..000000000 --- a/Code/steering/basic/SimulationParameters.cc +++ /dev/null @@ -1,41 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "io/writers/xdr/XdrMemWriter.h" - -#include "steering/basic/SimulationParameters.h" - -namespace hemelb -{ - namespace steering - { - - SimulationParameters::SimulationParameters() - { - // C'tor initialises to the following defaults. - - timeStep = 0; - time = 0.0; - nInlets = 3; - } - - SimulationParameters::~SimulationParameters() - { - } - - void SimulationParameters::pack(io::writers::xdr::XdrWriter* writer) - { - writer->operator <<(timeStep); - - writer->operator <<(time); - - writer->operator <<(0); // Cycle is always zero, leave this in to stop steering clients breaking. - writer->operator <<(nInlets); - - } - - } -} diff --git a/Code/steering/basic/SimulationParameters.h b/Code/steering/basic/SimulationParameters.h deleted file mode 100644 index 88aac3607..000000000 --- a/Code/steering/basic/SimulationParameters.h +++ /dev/null @@ -1,40 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_BASIC_SIMULATIONPARAMETERS_H -#define HEMELB_STEERING_BASIC_SIMULATIONPARAMETERS_H - -#include "io/writers/xdr/XdrMemWriter.h" -#include "lb/SimulationState.h" - -namespace hemelb -{ - namespace steering - { - - class SimulationParameters - { - - public: - - int timeStep; - double time; - int nInlets; - - static const u_int paramsSizeB = 3 * sizeof(int) + 1 * sizeof(double); - - SimulationParameters(); - ~SimulationParameters(); - void pack(io::writers::xdr::XdrWriter* writer); - - private: - - }; - - } -} - -#endif // HEMELB_STEERING_BASIC_SIMULATIONPARAMETERS_H diff --git a/Code/steering/basic/SteeringComponentB.cc b/Code/steering/basic/SteeringComponentB.cc deleted file mode 100644 index c6ae73989..000000000 --- a/Code/steering/basic/SteeringComponentB.cc +++ /dev/null @@ -1,97 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include - -#include "steering/SteeringComponent.h" -#include "steering/Network.h" -#include "io/writers/xdr/XdrMemReader.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace steering - { - SteeringComponent::SteeringComponent(Network* iNetwork, - net::Net * iNet, lb::SimulationState * iSimState, - configuration::SimConfig* iSimConfig, - const util::UnitConverter* iUnits, - reporting::Timers& timings) : - net::CollectiveAction(iNet->GetCommunicator(), timings[reporting::Timers::steeringWait]), - mNetwork(iNetwork), mSimState(iSimState), - privateSteeringParams(STEERABLE_PARAMETERS + 1), - mUnits(iUnits), simConfig(iSimConfig) - { - ClearValues(); - AssignValues(); - } - - bool SteeringComponent::RequiresSeparateSteeringCore() - { - return true; - } - - void SteeringComponent::PreSend() - { - if (collectiveComm->Rank() != RootRank) - return; - - // Create a buffer for the data received. - const int num_chars = STEERABLE_PARAMETERS * sizeof(float) / sizeof(char); - const int bytes = sizeof(char) * num_chars; - - char steeringRecvBuffer[bytes]; - - // Get the open socket. - isConnected = mNetwork->IsConnected(); - - if (!isConnected) - { - return; - } - - bool newSteeringDataExists = false; - - // Keep looping through all data that we can read - if multiple steering commands have been - // received we only want to act on the last set. - while (true) - { - // Try to receive all the data. - if (mNetwork->recv_all(steeringRecvBuffer, num_chars)) - { - // We read data! - newSteeringDataExists = true; - } - // If it wasn't readable, we've exhausted all the received data at the socket, so - // break and actually use the last read data. - else - { - break; - } - } - - if (newSteeringDataExists) - { - // Initialise the stream reader. - io::writers::xdr::XdrMemReader steeringStream(steeringRecvBuffer, bytes); - - for (int i = 0; i < STEERABLE_PARAMETERS; i++) - steeringStream.readFloat(privateSteeringParams[i]); - } - } - void SteeringComponent::Send() - { - collectiveReq = collectiveComm->Ibcast(privateSteeringParams, RootRank); - } - - void SteeringComponent::PostReceive() - { - // TODO we need to make sure that doing this doesn't overwrite the values in the config.xml file. - // At the moment, it definitely does. - AssignValues(); - } - } -} diff --git a/Code/steering/common/SteeringComponentC.cc b/Code/steering/common/SteeringComponentC.cc deleted file mode 100644 index 3c51e243e..000000000 --- a/Code/steering/common/SteeringComponentC.cc +++ /dev/null @@ -1,29 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "steering/SteeringComponent.h" - -namespace hemelb -{ - namespace steering - { - void SteeringComponent::AssignValues() - { - mSimState->SetIsTerminating(1 == (int) privateSteeringParams[SetIsTerminal]); - } - - void SteeringComponent::ClearValues() - { - isConnected = false; - - // signal useful to terminate the simulation - privateSteeringParams[SetIsTerminal] = 0.0F; - - // Vis_mode - privateSteeringParams[Mode] = 0.0F; - } - } -} diff --git a/Code/steering/none/ClientConnection.cc b/Code/steering/none/ClientConnection.cc deleted file mode 100644 index e186a1284..000000000 --- a/Code/steering/none/ClientConnection.cc +++ /dev/null @@ -1,47 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#include -#include -#include - -#include "steering/ClientConnection.h" - -namespace hemelb -{ - namespace steering - { - - /** - * In the 'no-steering' code, this should do nothing. - * - * @param iSteeringSessionId - * @return - */ - ClientConnection::ClientConnection(int iSteeringSessionId, reporting::Timers & timings): - mIsBusy(), timers(timings) - { - } - - ClientConnection::~ClientConnection() - { - } - - int ClientConnection::GetWorkingSocket() - { - return -1; - } - - void ClientConnection::ReportBroken(int iSocketNum) - { - } - - } -} - diff --git a/Code/steering/none/Network.cc b/Code/steering/none/Network.cc deleted file mode 100644 index f1ad3e166..000000000 --- a/Code/steering/none/Network.cc +++ /dev/null @@ -1,86 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "debug/Debugger.h" -#include "log/Logger.h" -#include "steering/Network.h" -#include "util/utilityFunctions.h" - -namespace hemelb -{ - namespace steering - { - Network::Network(int steeringSessionId, reporting::Timers & timings) : - clientConnection(steeringSessionId, timings) - { - - } - - /** - * Do nothing. - * - * @param sockid - * @param buf - * @param length - * @return Returns true if we have successfully provided that much data. - */ - bool Network::recv_all(char *buf, const int length) - { - return false; - } - - void Network::PreReceive() - { - } - - bool Network::IsConnected() - { - return false; - } - - /** - * Do nothing. - * - * @param sockid - * @param buf - * @param length - * @return Returns the number of bytes sent or -1 on failure. - */ - bool Network::send_all(const char *buf, const int length) - { - return false; - } - - void Network::Break(int socket) - { - } - - /** - * Do nothing. - * - * @param data - * @param length - * @param socket - * @return - */ - long Network::sendInternal(const char* data, long length, int socket) - { - return -1; - } - - } -} diff --git a/Code/steering/none/SteeringComponentN.cc b/Code/steering/none/SteeringComponentN.cc deleted file mode 100644 index f5c582550..000000000 --- a/Code/steering/none/SteeringComponentN.cc +++ /dev/null @@ -1,53 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "steering/SteeringComponent.h" - -namespace hemelb -{ - namespace steering - { - - /** - * In the 'no steering' version of the steering controller, we don't - * need to override any of the base classes methods, because we don't - * need to do anything. - * - * @param iNet - * @param iNetTop - * @param iSimState - * @return - */ - SteeringComponent::SteeringComponent(Network* network, - net::Net * iNet, lb::SimulationState * iSimState, - configuration::SimConfig* iSimConfig, - const util::UnitConverter* iUnits, - reporting::Timers& timings) : - net::CollectiveAction(iNet->GetCommunicator(), timings[reporting::Timers::steeringWait]), - mSimState(iSimState), - mUnits(iUnits), simConfig(iSimConfig), privateSteeringParams(STEERABLE_PARAMETERS + 1) - { - ClearValues(); - AssignValues(); - } - - bool SteeringComponent::RequiresSeparateSteeringCore() - { - return false; - } - - void SteeringComponent::PreSend() - { - } - void SteeringComponent::Send() - { - } - void SteeringComponent::PostReceive() - { - AssignValues(); - } - } -} diff --git a/deploy/compile_options.yml b/deploy/compile_options.yml index f6f623a17..fdd36e3c9 100644 --- a/deploy/compile_options.yml +++ b/deploy/compile_options.yml @@ -7,10 +7,6 @@ default: CMAKE_BUILD_TYPE: "Release" CMAKE_CXX_FLAGS_RELEASE: "-O4" -no_steering: - HEMELB_STEERING_LIB: "none" -basic_steering: - HEMELB_STEERING_LIB: "basic" debug: CMAKE_BUILD_TYPE: "Debug" HEMELB_LOG_LEVEL: "debug" diff --git a/deploy/fab.py b/deploy/fab.py index e13ffaf72..c1512d540 100644 --- a/deploy/fab.py +++ b/deploy/fab.py @@ -495,16 +495,13 @@ def hemelb(config, **args): config : config directory to use to define geometry, e.g. config=cylinder Keyword arguments: cores : number of compute cores to request - steering : steering session i.d. wall_time : wall-time job limit memory : memory per node """ with_config(config) execute(put_configs, config) job(dict(script='hemelb', - cores=4, steering=1111, wall_time='0:15:0', memory='2G'), args) - if args.get('steer', False): - execute(steer, env.name, retry=True, framerate=args.get('framerate'), orbit=args.get('orbit')) + cores=4, wall_time='0:15:0', memory='2G'), args) @task def multiscale_hemelb(config,**args): @@ -514,17 +511,13 @@ def multiscale_hemelb(config,**args): config : config directory to use to define geometry, e.g. config=cylinder Keyword arguments: cores : number of compute cores to request - steering : steering session i.d. wall_time : wall-time job limit memory : memory per node """ with_config(config) execute(put_configs,config) job(dict(script='multiscale_hemelb', - cores=4, steering=1111, wall_time='0:15:0',memory='2G'),args) - if args.get('steer',False): - execute(steer,env.name,retry=True,framerate=args.get('framerate'),orbit=args.get('orbit')) - + cores=4, wall_time='0:15:0',memory='2G'),args) @task def resubmit(name): @@ -541,7 +534,7 @@ def multijob(*names, **args): env.jobstorun = "\n".join(jobscriptpaths) # And then, submit it job(dict(script='multijob', job_name_template='multijob', - cores=4, steering=1111, wall_time='0:15:0', memory='2G'), args) + cores=4, wall_time='0:15:0', memory='2G'), args) @task def hemelbs(config, **args): @@ -552,7 +545,6 @@ def hemelbs(config, **args): config : config directory to use to define geometry, e.g. config=cylinder Keyword arguments: cores : number of compute cores to request - steering : steering session i.d. wall_time : wall-time job limit memory : memory per node """ @@ -572,7 +564,7 @@ def hemelb_benchmark(config, min_cores, max_cores, **args): with_config(config) execute(put_configs, config) job(dict(script='hemelb', - cores=cores_used, steering=1111, wall_time='0:15:0', memory='2G'), args) + cores=cores_used, wall_time='0:15:0', memory='2G'), args) cores_used *= 2 @task(alias='regress') @@ -582,7 +574,7 @@ def regression_test(**args): execute(copy_regression_tests) execute(build_python_tools) job(dict(job_name_template='regression_${build_number}_${machine_name}', cores=3, - wall_time='0:20:0', memory='2G', steering=1111, script='regression'), args) + wall_time='0:20:0', memory='2G', script='regression'), args) def calc_nodes(): # If we're not reserving whole nodes, then if we request less than one node's worth of cores, need to keep N<=n @@ -805,36 +797,6 @@ def vampir(original_job, *args): def vampir_tunnel(node, port): local("ssh hector -L 30070:nid%s:%s -N" % node, port) -@task -def steer(job, orbit=False, view=False, retry=False, framerate=None): - with_job(job) - if view: - env.steering_client = 'steering.py' - else: - env.steering_client = 'timing_client.py' - if orbit: - env.steering_options = "--orbit" - else: - env.steering_options = "" - if retry: - env.steering_options += " --retry" - if framerate: - env.steering_options += " --MaxFramerate=%s" % framerate - command_template = "python $repository_path/Tools/steering/python/hemelb_steering/${steering_client} ${steering_options} ${running_node} >> $job_results/steering_results.txt" - if retry: - while True: - try: - get_running_location() - run(template(command_template)) - break - except: - print "Couldn't connect. Will retry" - execute(stat) - time.sleep(10) - else: - get_running_location() - run(template(command_template)) - @task def ensemble(config,cores,wall_time): diff --git a/deploy/templates/multiscale_hemelb b/deploy/templates/multiscale_hemelb index 6c458caff..918887a5d 100644 --- a/deploy/templates/multiscale_hemelb +++ b/deploy/templates/multiscale_hemelb @@ -7,4 +7,4 @@ cd $job_results $run_prefix rm -rf results cp $job_config_path/* . -$run_command $install_path/bin/multiscale_hemelb -in config.xml -s $snapshots -ss $steering +$run_command $install_path/bin/multiscale_hemelb -in config.xml -s $snapshots diff --git a/deploy/templates/regression b/deploy/templates/regression index 79428a65a..c1cb2b426 100644 --- a/deploy/templates/regression +++ b/deploy/templates/regression @@ -6,5 +6,5 @@ cd $job_results $run_prefix rm -rf results -$run_command $install_path/bin/hemelb -in $regression_test_path/config.xml -out $job_results/results -ss $steering +$run_command $install_path/bin/hemelb -in $regression_test_path/config.xml -out $job_results/results $run_command_one_proc $regression_test_path/NumericalComparison $regression_test_path/CleanExtracted results/Extracted \ No newline at end of file diff --git a/deploy/test/fixtures/templates/hemelb b/deploy/test/fixtures/templates/hemelb index 2998a4beb..aef351b9b 100644 --- a/deploy/test/fixtures/templates/hemelb +++ b/deploy/test/fixtures/templates/hemelb @@ -2,4 +2,4 @@ cd $job_results $run_prefix rm -rf results cp $job_config_path/* . -mpirun -np $cores $install_path/bin/hemelb -in config.xml -ss $steering +mpirun -np $cores $install_path/bin/hemelb -in config.xml From 830a7fa18cfc65de26af5515e1ed05e05944ce5a Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 27 Oct 2016 11:23:06 +0100 Subject: [PATCH 77/99] remove the net object of horror --- Code/SimulationMaster.cc | 10 +- Code/SimulationMaster.h | 6 +- Code/colloids/ColloidController.h | 1 - Code/comm/AsyncConcern.h | 43 ++++++ .../neighbouring/NeighbouringDataManager.h | 1 - Code/lb/EntropyTester.h | 4 +- Code/net/BaseNet.cc | 75 ---------- Code/net/BaseNet.h | 84 ----------- Code/net/CMakeLists.txt | 7 +- Code/net/mixins/InterfaceDelegationNet.h | 65 --------- Code/net/mixins/StoringNet.cc | 33 ----- Code/net/mixins/StoringNet.h | 36 ----- Code/net/mixins/mixins.h | 15 -- .../mixins/pointpoint/CoalescePointPoint.cc | 138 ------------------ .../mixins/pointpoint/CoalescePointPoint.h | 49 ------- .../mixins/pointpoint/ImmediatePointPoint.cc | 46 ------ .../mixins/pointpoint/ImmediatePointPoint.h | 40 ----- .../mixins/pointpoint/SeparatedPointPoint.cc | 110 -------------- .../mixins/pointpoint/SeparatedPointPoint.h | 51 ------- Code/net/net.h | 30 ---- Code/net/phased/NetConcern.h | 51 ------- 21 files changed, 52 insertions(+), 843 deletions(-) create mode 100644 Code/comm/AsyncConcern.h delete mode 100644 Code/net/BaseNet.cc delete mode 100644 Code/net/BaseNet.h delete mode 100644 Code/net/mixins/InterfaceDelegationNet.h delete mode 100644 Code/net/mixins/StoringNet.cc delete mode 100644 Code/net/mixins/StoringNet.h delete mode 100644 Code/net/mixins/mixins.h delete mode 100644 Code/net/mixins/pointpoint/CoalescePointPoint.cc delete mode 100644 Code/net/mixins/pointpoint/CoalescePointPoint.h delete mode 100644 Code/net/mixins/pointpoint/ImmediatePointPoint.cc delete mode 100644 Code/net/mixins/pointpoint/ImmediatePointPoint.h delete mode 100644 Code/net/mixins/pointpoint/SeparatedPointPoint.cc delete mode 100644 Code/net/mixins/pointpoint/SeparatedPointPoint.h delete mode 100644 Code/net/net.h delete mode 100644 Code/net/phased/NetConcern.h diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index 817dc5345..f5f111e17 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -33,7 +33,7 @@ * object. */ SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, hemelb::comm::Communicator::ConstPtr ioComm) : - ioComms(ioComm), timings(ioComm), build_info(), communicationNet(ioComm), asyncCommQ(hemelb::comm::Async::New(ioComm)) + ioComms(ioComm), timings(ioComm), build_info(), asyncCommQ(hemelb::comm::Async::New(ioComm)) { timings[hemelb::reporting::Timers::total].Start(); @@ -251,7 +251,7 @@ void SimulationMaster::Initialise() stepManager = new hemelb::net::phased::StepManager(2, &timings, hemelb::net::separate_communications); - netConcern = new hemelb::net::phased::NetConcern(communicationNet); + netConcern = new hemelb::comm::AsyncConcern(asyncCommQ); stepManager->RegisterIteratedActorSteps(*neighbouringDataManager, 0); if (colloidController != NULL) { @@ -346,11 +346,7 @@ void SimulationMaster::Finalise() reporter->FillDictionary(); reporter->Write(); } - // DTMP: Logging output on communication as debug output for now. - hemelb::log::Logger::Log("sync points: %lld, bytes sent: %lld", - communicationNet.SyncPointsCounted, - communicationNet.BytesSent); - + hemelb::log::Logger::Log("Finish running simulation."); } diff --git a/Code/SimulationMaster.h b/Code/SimulationMaster.h index e5c3af678..ebab99a77 100644 --- a/Code/SimulationMaster.h +++ b/Code/SimulationMaster.h @@ -10,7 +10,6 @@ #include "extraction/PropertyActor.h" #include "lb/lb.hpp" #include "lb/StabilityTester.h" -#include "net/net.h" #include "lb/EntropyTester.h" #include "lb/iolets/BoundaryValues.h" #include "util/UnitConverter.h" @@ -22,7 +21,7 @@ #include "lb/IncompressibilityChecker.h" #include "colloids/ColloidController.h" #include "net/phased/StepManager.h" -#include "net/phased/NetConcern.h" +#include "comm/AsyncConcern.h" #include "geometry/neighbouring/NeighbouringDataManager.h" class SimulationMaster @@ -90,7 +89,6 @@ class SimulationMaster hemelb::lb::IncompressibilityChecker* incompressibilityChecker; hemelb::colloids::ColloidController* colloidController; - hemelb::net::Net communicationNet; hemelb::comm::Async::Ptr asyncCommQ; const hemelb::util::UnitConverter* unitConverter; @@ -98,7 +96,7 @@ class SimulationMaster hemelb::extraction::PropertyActor* propertyExtractor; hemelb::net::phased::StepManager* stepManager; - hemelb::net::phased::NetConcern* netConcern; + hemelb::comm::AsyncConcern* netConcern; static const hemelb::LatticeTimeStep FORCE_FLUSH_PERIOD=1000; }; diff --git a/Code/colloids/ColloidController.h b/Code/colloids/ColloidController.h index 9448c56be..04170f9bd 100644 --- a/Code/colloids/ColloidController.h +++ b/Code/colloids/ColloidController.h @@ -8,7 +8,6 @@ #define HEMELB_COLLOIDS_COLLOIDCONTROLLER_H #include -#include "net/net.h" #include "net/IteratedAction.h" #include "geometry/LatticeData.h" #include "geometry/Geometry.h" diff --git a/Code/comm/AsyncConcern.h b/Code/comm/AsyncConcern.h new file mode 100644 index 000000000..0bec98bc8 --- /dev/null +++ b/Code/comm/AsyncConcern.h @@ -0,0 +1,43 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_COMM_ASYNC_CONCERN_H +#define HEMELB_COMM_ASYNC_CONCERN_H + +#include "comm/Async.h" +#include "net/phased/Concern.h" +#include "net/phased/steps.h" + +namespace hemelb +{ + namespace comm + { + using namespace net::phased; + class AsyncConcern : public Concern + { + public: + AsyncConcern(Async::Ptr async) : mAsync(async) + { + } + bool CallAction(int action) + { + switch (static_cast(action)) + { + case steps::Wait: + mAsync->Wait(); + return true; + + default: + return false; + } + } + private: + Async::Ptr mAsync; + }; + + } +} + +#endif diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.h b/Code/geometry/neighbouring/NeighbouringDataManager.h index b0d19b1b3..88be667a9 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.h +++ b/Code/geometry/neighbouring/NeighbouringDataManager.h @@ -10,7 +10,6 @@ #include "geometry/LatticeData.h" #include "geometry/neighbouring/NeighbouringLatticeData.h" #include "geometry/neighbouring/RequiredSiteInformation.h" -#include "net/net.h" #include "net/IteratedAction.h" #include #include diff --git a/Code/lb/EntropyTester.h b/Code/lb/EntropyTester.h index 4d9e545c0..2d5fe1e33 100644 --- a/Code/lb/EntropyTester.h +++ b/Code/lb/EntropyTester.h @@ -21,9 +21,9 @@ namespace hemelb { public: EntropyTester(int* collisionTypes, unsigned int typesTested, - const geometry::LatticeData * iLatDat, net::Net* net, + const geometry::LatticeData * iLatDat, comm::Communicator::ConstPtr comm, SimulationState* simState, reporting::Timers& timings) : - net::CollectiveAction(net->GetCommunicator(), timings[reporting::Timers::mpiWait]), + net::CollectiveAction(comm, timings[reporting::Timers::mpiWait]), mLatDat(iLatDat), mHPreCollision(mLatDat->GetLocalFluidSiteCount()), workTimer(timings[reporting::Timers::monitoring]) { diff --git a/Code/net/BaseNet.cc b/Code/net/BaseNet.cc deleted file mode 100644 index e4b54adbc..000000000 --- a/Code/net/BaseNet.cc +++ /dev/null @@ -1,75 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -/*! \file net.cc - \brief In this file the functions useful to discover the topology used and - to create and delete the domain decomposition and the various - buffers are defined. - */ - -#include -#include -#include - -#include "net/BaseNet.h" -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" -#include "comm/Communicator.h" - -namespace hemelb -{ - namespace net - { - - void BaseNet::Dispatch() - { - Send(); - Receive(); - Wait(); - } - - BaseNet::BaseNet(comm::Communicator::ConstPtr commObject) : - BytesSent(0), SyncPointsCounted(0), communicator(commObject) - { - } - - void BaseNet::Receive() - { - // Ensure collectives are called before point-to-point, as some implementing mixins implement collectives via point-to-point - ReceivePointToPoint(); - } - - void BaseNet::Send() - { - - // Ensure collectives are called before point-to-point, as some implementing mixins implement collectives via point-to-point - SendPointToPoint(); - } - - void BaseNet::Wait() - { - SyncPointsCounted++; //DTMP: counter for monitoring purposes. - - WaitPointToPoint(); - - displacementsBuffer.clear(); - countsBuffer.clear(); - } - - std::vector & BaseNet::GetDisplacementsBuffer() - { - displacementsBuffer.push_back(std::vector()); - return displacementsBuffer.back(); - } - - std::vector & BaseNet::GetCountsBuffer() - { - countsBuffer.push_back(std::vector()); - return countsBuffer.back(); - } - - } -} diff --git a/Code/net/BaseNet.h b/Code/net/BaseNet.h deleted file mode 100644 index 4b8d52d1e..000000000 --- a/Code/net/BaseNet.h +++ /dev/null @@ -1,84 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_BASENET_H -#define HEMELB_NET_BASENET_H - -#include -#include -#include -#include - -#include "constants.h" -#include "comm/Communicator.h" - -namespace hemelb -{ - namespace net - { - - class BaseNet - { - public: - BaseNet(comm::Communicator::ConstPtr communicator); - - virtual ~BaseNet() - { - } - ; - - //DTMP: monitoring variables - long long int BytesSent; - long long int SyncPointsCounted; - - void Receive(); - void Send(); - virtual void Wait(); - - /*** - * Carry out a complete send-receive-wait - */ - void Dispatch(); - - inline comm::Communicator::ConstPtr GetCommunicator() const - { - return communicator; - } - - inline int Rank() const - { - return communicator->Rank(); - } - inline int Size() const - { - return communicator->Size(); - } - protected: - virtual void SendPointToPoint()=0; - - virtual void ReceivePointToPoint()=0; - - virtual void WaitPointToPoint()=0; - - // Interfaces exposing MPI_Datatype, not intended for client class use - virtual void RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type)=0; - virtual void RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type)=0; - - std::vector & GetDisplacementsBuffer(); - std::vector & GetCountsBuffer(); - - comm::Communicator::ConstPtr communicator; - private: - /*** - * Buffers which can be used to store displacements and counts for cleaning up interfaces - * These will be cleaned up following a Wait/Dispatch - */ - std::vector > displacementsBuffer; - std::vector > countsBuffer; - }; - } -} -#endif // HEMELB_NET_NET_H diff --git a/Code/net/CMakeLists.txt b/Code/net/CMakeLists.txt index 0d85f0141..4a4d8a2fb 100644 --- a/Code/net/CMakeLists.txt +++ b/Code/net/CMakeLists.txt @@ -4,12 +4,9 @@ # file AUTHORS. This software is provided under the terms of the # license in the file LICENSE. add_library(hemelb_net - IteratedAction.cc BaseNet.cc + IteratedAction.cc CollectiveAction.cc -mixins/pointpoint/CoalescePointPoint.cc -mixins/pointpoint/SeparatedPointPoint.cc -mixins/pointpoint/ImmediatePointPoint.cc -mixins/StoringNet.cc ProcComms.cc + ProcComms.cc phased/StepManager.cc) configure_file ( "${PROJECT_SOURCE_DIR}/net/BuildInfo.h.in" diff --git a/Code/net/mixins/InterfaceDelegationNet.h b/Code/net/mixins/InterfaceDelegationNet.h deleted file mode 100644 index d903c6b2d..000000000 --- a/Code/net/mixins/InterfaceDelegationNet.h +++ /dev/null @@ -1,65 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_INTERFACEDELEGATIONNET_H -#define HEMELB_NET_MIXINS_INTERFACEDELEGATIONNET_H -namespace hemelb -{ - namespace net - { - /*** - * Define the external template-based interface to be used by client classes. - * We define a series of templates with nice C++ style interfaces which delegate to C-style template interfaces - * And then, delegate the templated C-style interfaces to nontemplated interfaces taking an MPI datatype. - */ - class InterfaceDelegationNet : public virtual BaseNet - { - public: - InterfaceDelegationNet(comm::Communicator::ConstPtr comms) : - BaseNet(comms) - { - } - - template - void RequestSendV(const std::vector &payload, proc_t toRank) - { - RequestSend(&payload[0], payload.size(), toRank); - } - - template - void RequestSendR(const T& value, proc_t toRank) - { - RequestSend(&value, 1, toRank); - } - - template - void RequestReceiveR(T& value, proc_t fromRank) - { - RequestReceive(&value, 1, fromRank); - } - - template - void RequestReceiveV(std::vector &payload, proc_t toRank) - { - RequestReceive(&payload[0], payload.size(), toRank); - } - - template - void RequestSend(const T* pointer, int count, proc_t rank) - { - RequestSendImpl(const_cast(pointer), count, rank, comm::MpiDataType()); - } - - template - void RequestReceive(T* pointer, int count, proc_t rank) - { - RequestReceiveImpl(pointer, count, rank, comm::MpiDataType()); - } - - }; - } -} -#endif diff --git a/Code/net/mixins/StoringNet.cc b/Code/net/mixins/StoringNet.cc deleted file mode 100644 index 44dc6c617..000000000 --- a/Code/net/mixins/StoringNet.cc +++ /dev/null @@ -1,33 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/StoringNet.h" -namespace hemelb -{ - namespace net - { - StoringNet::StoringNet(comm::Communicator::ConstPtr comms) : - BaseNet(comms) - { - } - void StoringNet::RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type) - { - if (count > 0) - { - sendProcessorComms[rank].push_back(SimpleRequest(pointer, count, type, rank)); - } - } - - void StoringNet::RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type) - { - if (count > 0) - { - receiveProcessorComms[rank].push_back(SimpleRequest(pointer, count, type, rank)); - } - } - - } -} diff --git a/Code/net/mixins/StoringNet.h b/Code/net/mixins/StoringNet.h deleted file mode 100644 index 80a77fb45..000000000 --- a/Code/net/mixins/StoringNet.h +++ /dev/null @@ -1,36 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_STORINGNET_H -#define HEMELB_NET_MIXINS_STORINGNET_H -#include "net/BaseNet.h" -#include "log/Logger.h" -#include "net/ProcComms.h" -namespace hemelb -{ - namespace net - { - class StoringNet : public virtual BaseNet - { - public: - StoringNet(comm::Communicator::ConstPtr comms); - - virtual void RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); - virtual void RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); - - - protected: - /** - * Struct representing all that's needed to successfully communicate with another processor. - */ - - std::map sendProcessorComms; - std::map receiveProcessorComms; - - }; - } -} -#endif diff --git a/Code/net/mixins/mixins.h b/Code/net/mixins/mixins.h deleted file mode 100644 index 68c752031..000000000 --- a/Code/net/mixins/mixins.h +++ /dev/null @@ -1,15 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_MIXINS_H -#define HEMELB_NET_MIXINS_MIXINS_H - -#include "net/mixins/pointpoint/CoalescePointPoint.h" -#include "net/mixins/pointpoint/ImmediatePointPoint.h" -#include "net/mixins/pointpoint/SeparatedPointPoint.h" -#include "net/mixins/StoringNet.h" -#include "net/mixins/InterfaceDelegationNet.h" -#endif diff --git a/Code/net/mixins/pointpoint/CoalescePointPoint.cc b/Code/net/mixins/pointpoint/CoalescePointPoint.cc deleted file mode 100644 index 632e69d8a..000000000 --- a/Code/net/mixins/pointpoint/CoalescePointPoint.cc +++ /dev/null @@ -1,138 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/pointpoint/CoalescePointPoint.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace net - { - - void CoalescePointPoint::EnsureEnoughRequests(size_t count) - { - if (requests->size() < count) - { - requests->resize(count); - } - } - - void CoalescePointPoint::ReceivePointToPoint() - { - - // Make sure the MPI datatypes have been created. - EnsurePreparedToSendReceive(); - proc_t m = 0; - - for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); - ++it) - { - requests->set(m, communicator->IrecvImpl(it->second.front().Pointer, - 1, - it->second.Type, - it->first, - 10)); - ++m; - } - - - //if(m>1) { - // hemelb::log::Logger::Log("RecvPointToPoint() Neighbouring proc count: %i", m); - //} - - } - - // Makes sure the MPI_Datatypes for sending and receiving have been created for every neighbour. - void CoalescePointPoint::EnsurePreparedToSendReceive() - { - if (sendReceivePrepped) - { - return; - } - - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); ++it) - { - it->second.CreateMPIType(); - } - - for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); - ++it) - { - it->second.CreateMPIType(); - } - - EnsureEnoughRequests(receiveProcessorComms.size() + sendProcessorComms.size()); - - sendReceivePrepped = true; - } - - void CoalescePointPoint::SendPointToPoint() - { - - // Make sure the MPI datatypes have been created. - EnsurePreparedToSendReceive(); - proc_t m = 0; - - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); ++it) - { - - int TypeSizeStorage = 0; //DTMP:byte size tracking - MPI_Type_size(it->second.Type, &TypeSizeStorage); //DTMP: - BytesSent += TypeSizeStorage; //DTMP: - - requests->set(receiveProcessorComms.size() + m, - communicator->IsendImpl(it->second.front().Pointer, - 1, - it->second.Type, - it->first, - 10)); - ++m; - } - } - - /*! - Free the allocated data. - */ - CoalescePointPoint::~CoalescePointPoint() - { - if (sendReceivePrepped) - { - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); - ++it) - { - MPI_Type_free(&it->second.Type); - } - - for (std::map::iterator it = receiveProcessorComms.begin(); - it != receiveProcessorComms.end(); ++it) - { - MPI_Type_free(&it->second.Type); - } - - } - } - - void CoalescePointPoint::WaitPointToPoint() - { - requests->WaitAll(); - - for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); - ++it) - { - MPI_Type_free(&it->second.Type); - } - receiveProcessorComms.clear(); - - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); ++it) - { - MPI_Type_free(&it->second.Type); - } - sendProcessorComms.clear(); - sendReceivePrepped = false; - - } - } -} diff --git a/Code/net/mixins/pointpoint/CoalescePointPoint.h b/Code/net/mixins/pointpoint/CoalescePointPoint.h deleted file mode 100644 index 2bd51e31b..000000000 --- a/Code/net/mixins/pointpoint/CoalescePointPoint.h +++ /dev/null @@ -1,49 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_POINTPOINT_COALESCEPOINTPOINT_H -#define HEMELB_NET_MIXINS_POINTPOINT_COALESCEPOINTPOINT_H -#include "net/BaseNet.h" -#include "net/mixins/StoringNet.h" -#include "comm/Request.h" - -namespace hemelb -{ - namespace net - { - class CoalescePointPoint : public virtual StoringNet - { - - public: - CoalescePointPoint(comm::Communicator::ConstPtr comms) : - BaseNet(comms), StoringNet(comms), sendReceivePrepped(false), requests(comms->MakeRequestList()) - { - } - ~CoalescePointPoint(); - - void WaitPointToPoint(); - - protected: - void ReceivePointToPoint(); - void SendPointToPoint(); - - private: - void EnsureEnoughRequests(size_t count); - void EnsurePreparedToSendReceive(); - bool sendReceivePrepped; - - // Requests and statuses available for general communication within the Net object (both - // initialisation and during each iteration). Code using these must make sure - // there are enough available. We do this in a way to minimise the number created - // on each core, but also to minimise creation / deletion overheads. - //std::vector requests; - comm::RequestList::Ptr requests; - //std::vector statuses; - }; - } -} - -#endif diff --git a/Code/net/mixins/pointpoint/ImmediatePointPoint.cc b/Code/net/mixins/pointpoint/ImmediatePointPoint.cc deleted file mode 100644 index 10473fd1a..000000000 --- a/Code/net/mixins/pointpoint/ImmediatePointPoint.cc +++ /dev/null @@ -1,46 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/pointpoint/ImmediatePointPoint.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace net - { - void ImmediatePointPoint::RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type) - { - communicator->SsendImpl(pointer, count, type, rank, 10); - } - void ImmediatePointPoint::RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type) - { - communicator->RecvImpl(pointer, count, type, rank, 10, MPI_STATUS_IGNORE); - - } - - void ImmediatePointPoint::ReceivePointToPoint() - { - - } - - void ImmediatePointPoint::SendPointToPoint() - { - - } - - /*! - Free the allocated data. - */ - ImmediatePointPoint::~ImmediatePointPoint() - { - } - - void ImmediatePointPoint::WaitPointToPoint() - { - - } - } -} diff --git a/Code/net/mixins/pointpoint/ImmediatePointPoint.h b/Code/net/mixins/pointpoint/ImmediatePointPoint.h deleted file mode 100644 index ed257c7a4..000000000 --- a/Code/net/mixins/pointpoint/ImmediatePointPoint.h +++ /dev/null @@ -1,40 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_POINTPOINT_IMMEDIATEPOINTPOINT_H -#define HEMELB_NET_MIXINS_POINTPOINT_IMMEDIATEPOINTPOINT_H -#include "net/BaseNet.h" -#include "net/mixins/StoringNet.h" -namespace hemelb -{ - namespace net - { - // although ImmediatePointPoint does not use the StoringNet capabilities at all - // it needs to inherit it - // so that it becomes the unique final overrider - class ImmediatePointPoint : public virtual StoringNet - { - - public: - ImmediatePointPoint(comm::Communicator::ConstPtr comms) : - BaseNet(comms), StoringNet(comms) - { - } - ~ImmediatePointPoint(); - - void WaitPointToPoint(); - // we will *NOT* store the requests, so we must provide RequestSendImpl ourselves. - virtual void RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); - virtual void RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); - protected: - void ReceivePointToPoint(); //PASS - void SendPointToPoint(); //PASS - - }; - } -} - -#endif diff --git a/Code/net/mixins/pointpoint/SeparatedPointPoint.cc b/Code/net/mixins/pointpoint/SeparatedPointPoint.cc deleted file mode 100644 index 2d2dd433d..000000000 --- a/Code/net/mixins/pointpoint/SeparatedPointPoint.cc +++ /dev/null @@ -1,110 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/pointpoint/SeparatedPointPoint.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace net - { - - void SeparatedPointPoint::EnsureEnoughRequests(size_t count) - { - if (requests->size() < count) - { - requests->resize(count); - } - } - - void SeparatedPointPoint::ReceivePointToPoint() - { - EnsurePreparedToSendReceive(); - proc_t m = 0; - - for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); - ++it) - { - for (ProcComms::iterator request = it->second.begin(); request != it->second.end(); request++) - { - requests->set(m, - communicator->IrecvImpl(request->Pointer, - request->Count, - request->Type, - it->first, - 10)); - ++m; - } - } - - } - - // Makes sure the MPI_Datatypes for sending and receiving have been created for every neighbour. - void SeparatedPointPoint::EnsurePreparedToSendReceive() - { - if (sendReceivePrepped) - { - return; - } - - count_sends=0; - count_receives=0; - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); ++it) - { - count_sends += it->second.size(); - } - - for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); - ++it) - { - count_receives += it->second.size(); - } - - EnsureEnoughRequests(count_sends+count_receives); - - sendReceivePrepped = true; - } - - void SeparatedPointPoint::SendPointToPoint() - { - - // Make sure the MPI datatypes have been created. - EnsurePreparedToSendReceive(); - proc_t m = 0; - - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); ++it) - { - for (ProcComms::iterator request = it->second.begin(); request != it->second.end(); request++) - { - requests->set(count_receives+m, - communicator->IsendImpl(request->Pointer, - request->Count, - request->Type, - it->first, - 10)); - ++m; - } - } - } - - /*! - Free the allocated data. - */ - SeparatedPointPoint::~SeparatedPointPoint() - { - } - - void SeparatedPointPoint::WaitPointToPoint() - { - requests->WaitAll(); - - receiveProcessorComms.clear(); - sendProcessorComms.clear(); - sendReceivePrepped = false; - - } - } -} diff --git a/Code/net/mixins/pointpoint/SeparatedPointPoint.h b/Code/net/mixins/pointpoint/SeparatedPointPoint.h deleted file mode 100644 index ecf7a3525..000000000 --- a/Code/net/mixins/pointpoint/SeparatedPointPoint.h +++ /dev/null @@ -1,51 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_POINTPOINT_SEPARATEDPOINTPOINT_H -#define HEMELB_NET_MIXINS_POINTPOINT_SEPARATEDPOINTPOINT_H -#include "net/BaseNet.h" -#include "net/mixins/StoringNet.h" -#include "comm/Request.h" - -namespace hemelb -{ - namespace net - { - class SeparatedPointPoint : public virtual StoringNet - { - - public: - SeparatedPointPoint(comm::Communicator::ConstPtr comms) : - BaseNet(comms), StoringNet(comms), sendReceivePrepped(false), - requests(comms->MakeRequestList()), count_sends(0), - count_receives(0) - { - } - ~SeparatedPointPoint(); - - void WaitPointToPoint(); - - protected: - void ReceivePointToPoint(); - void SendPointToPoint(); - - private: - void EnsureEnoughRequests(size_t count); - void EnsurePreparedToSendReceive(); - bool sendReceivePrepped; - - // Requests and statuses available for general communication within the Net object (both - // initialisation and during each iteration). Code using these must make sure - // there are enough available. We do this in a way to minimise the number created - // on each core, but also to minimise creation / deletion overheads. - comm::RequestList::Ptr requests; - unsigned int count_sends; - unsigned int count_receives; - }; - } -} - -#endif diff --git a/Code/net/net.h b/Code/net/net.h deleted file mode 100644 index 3adeb023c..000000000 --- a/Code/net/net.h +++ /dev/null @@ -1,30 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_NET_H -#define HEMELB_NET_NET_H - -#include "net/BaseNet.h" -#include "net/mixins/mixins.h" -#include "net/BuildInfo.h" -namespace hemelb -{ - namespace net - { - class Net : public PointPointImpl, - public InterfaceDelegationNet - { - public: - Net(comm::Communicator::ConstPtr communicator) : - BaseNet(communicator), StoringNet(communicator), PointPointImpl(communicator), - InterfaceDelegationNet(communicator) - { - } - }; - } -} -#endif - diff --git a/Code/net/phased/NetConcern.h b/Code/net/phased/NetConcern.h deleted file mode 100644 index 2cfc38765..000000000 --- a/Code/net/phased/NetConcern.h +++ /dev/null @@ -1,51 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASED_NETCONCERN_H -#define HEMELB_NET_PHASED_NETCONCERN_H - -#include "net/phased/Concern.h" -#include "net/phased/steps.h" - -namespace hemelb -{ - namespace net - { - namespace phased - { - class NetConcern : public Concern - { - public: - NetConcern(net::BaseNet & net) : - net(net) - { - } - bool CallAction(int action) - { - switch (static_cast(action)) - { - case phased::steps::Send: - net.Send(); - return true; - case phased::steps::Receive: - net.Receive(); - return true; - case phased::steps::Wait: - net.Wait(); - return true; - - default: - return false; - } - } - private: - net::BaseNet & net; - }; - } - } -} - -#endif From 3d515bf5b183dee2a6e65767852de604645a9cca Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 27 Oct 2016 11:25:25 +0100 Subject: [PATCH 78/99] remove ProcComms and StoredRequest as redundant --- Code/net/CMakeLists.txt | 3 +-- Code/net/ProcComms.cc | 37 ---------------------------- Code/net/ProcComms.h | 44 ---------------------------------- Code/net/StoredRequest.h | 52 ---------------------------------------- 4 files changed, 1 insertion(+), 135 deletions(-) delete mode 100644 Code/net/ProcComms.cc delete mode 100644 Code/net/ProcComms.h delete mode 100644 Code/net/StoredRequest.h diff --git a/Code/net/CMakeLists.txt b/Code/net/CMakeLists.txt index 4a4d8a2fb..0ff25081e 100644 --- a/Code/net/CMakeLists.txt +++ b/Code/net/CMakeLists.txt @@ -6,8 +6,7 @@ add_library(hemelb_net IteratedAction.cc CollectiveAction.cc - ProcComms.cc -phased/StepManager.cc) + phased/StepManager.cc) configure_file ( "${PROJECT_SOURCE_DIR}/net/BuildInfo.h.in" "${PROJECT_BINARY_DIR}/net/BuildInfo.h" diff --git a/Code/net/ProcComms.cc b/Code/net/ProcComms.cc deleted file mode 100644 index 63639ff1d..000000000 --- a/Code/net/ProcComms.cc +++ /dev/null @@ -1,37 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/ProcComms.h" -namespace hemelb -{ - namespace net - { - void ProcComms::CreateMPIType() - { - std::vector displacements(size()); - std::vector lengths; - std::vector types; - - int location = 0; - - MPI_Aint offset; - MPI_Get_address(front().Pointer, &offset); - - for (iterator it = begin(); it != end(); ++it) - { - MPI_Get_address(it->Pointer, &displacements[location]); - displacements[location] -= offset; - - ++location; - lengths.push_back(it->Count); - types.push_back(it->Type); - } - // Create the type and commit it. - MPI_Type_create_struct(this->size(), &lengths.front(), &displacements.front(), &types.front(), &Type); - MPI_Type_commit(&Type); - } - } -} diff --git a/Code/net/ProcComms.h b/Code/net/ProcComms.h deleted file mode 100644 index 7031c81c6..000000000 --- a/Code/net/ProcComms.h +++ /dev/null @@ -1,44 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PROCCOMMS_H -#define HEMELB_NET_PROCCOMMS_H - -#include "constants.h" -#include "comm/MpiDataType.h" -#include "net/StoredRequest.h" -#include -#include - -namespace hemelb -{ - namespace net - { - template - class BaseProcComms : public std::deque - { - public: - MPI_Datatype Type; - }; - - class ProcComms : public BaseProcComms - { - public: - void CreateMPIType(); - }; - - class GatherProcComms : public BaseProcComms - { - - }; - - class GatherVReceiveProcComms : public BaseProcComms - { - - }; - } -} -#endif diff --git a/Code/net/StoredRequest.h b/Code/net/StoredRequest.h deleted file mode 100644 index f527de45f..000000000 --- a/Code/net/StoredRequest.h +++ /dev/null @@ -1,52 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_STOREDREQUEST_H -#define HEMELB_NET_STOREDREQUEST_H -#include -#include "constants.h" -#include "comm/MpiDataType.h" -namespace hemelb -{ - namespace net - { - - class SimpleRequest - { - public: - void * Pointer; - int Count; - MPI_Datatype Type; - proc_t Rank; - SimpleRequest(void *pointer, int count, MPI_Datatype type, proc_t rank) : - Pointer(pointer), Count(count), Type(type), Rank(rank) - { - } - }; - - class ScalarRequest : public SimpleRequest - { - public: - ScalarRequest(void *pointer, MPI_Datatype type, proc_t rank) : - SimpleRequest(pointer, 1, type, rank) - { - } - }; - - class GatherVReceiveRequest : public SimpleRequest - { - public: - int * Counts; - int * Displacements; - GatherVReceiveRequest(void *pointer, int *displacements, int *counts, MPI_Datatype type) : - SimpleRequest(pointer, 0, type, 0), Counts(counts), Displacements(displacements) - { - } - }; - } -} - -#endif From 361310464bed3f241588457c17d6fb1e8f9fd807 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 28 Oct 2016 12:39:07 +0100 Subject: [PATCH 79/99] get most unittests passing --- Code/comm/Async.h | 13 +- Code/unittests/SimulationMasterTests.h | 4 +- .../configuration/CommandLineTests.h | 4 +- Code/unittests/geometry/GeometryReaderTests.h | 3 +- Code/unittests/geometry/NeedsTests.h | 36 +- .../NeighbouringDataManagerTests.h | 154 +++--- Code/unittests/helpers/MockCommsHelper.h | 48 ++ Code/unittests/helpers/MockCommunicator.h | 469 ++++++++++++++++++ .../unittests/helpers/MockCommunicatorTests.h | 188 +++++++ Code/unittests/helpers/MockNetHelper.h | 178 ------- Code/unittests/helpers/helpers.h | 1 + Code/unittests/io/PathManagerTests.h | 4 +- .../lbtests/IncompressibilityCheckerTests.h | 8 +- .../multiscale/MockIntercommunicatorTests.h | 4 +- Code/unittests/net/LabelledRequest.h | 115 ----- Code/unittests/net/NetMock.h | 39 -- Code/unittests/net/RecordingNet.h | 185 ------- Code/unittests/reporting/ReporterTests.h | 5 +- 18 files changed, 825 insertions(+), 633 deletions(-) create mode 100644 Code/unittests/helpers/MockCommsHelper.h create mode 100644 Code/unittests/helpers/MockCommunicator.h create mode 100644 Code/unittests/helpers/MockCommunicatorTests.h delete mode 100644 Code/unittests/helpers/MockNetHelper.h delete mode 100644 Code/unittests/net/LabelledRequest.h delete mode 100644 Code/unittests/net/NetMock.h delete mode 100644 Code/unittests/net/RecordingNet.h diff --git a/Code/comm/Async.h b/Code/comm/Async.h index 1082ebe6b..0301c6c81 100644 --- a/Code/comm/Async.h +++ b/Code/comm/Async.h @@ -6,6 +6,7 @@ #ifndef HEMELB_COMM_ASYNC_H #define HEMELB_COMM_ASYNC_H +#include #include "comm/Communicator.h" #include "comm/Request.h" @@ -44,19 +45,19 @@ namespace hemelb } template - void Isend(Ts... args) + void Isend(Ts&&... args) { - q->push_back(comms->Isend(args...)); + q->push_back(comms->Isend(std::forward(args)...)); } template - void Issend(Ts... args) + void Issend(Ts&&... args) { - q->push_back(comms->Issend(args...)); + q->push_back(comms->Issend(std::forward(args)...)); } template - void Irecv(Ts... args) + void Irecv(Ts&&... args) { - q->push_back(comms->Irecv(args...)); + q->push_back(comms->Irecv(std::forward(args)...)); } private: diff --git a/Code/unittests/SimulationMasterTests.h b/Code/unittests/SimulationMasterTests.h index 678a0ffcd..e9964877f 100644 --- a/Code/unittests/SimulationMasterTests.h +++ b/Code/unittests/SimulationMasterTests.h @@ -27,12 +27,10 @@ namespace hemelb public: void setUp() { - argc = 5; + argc = 3; argv[0] = "hemelb"; argv[1] = "-in"; argv[2] = "four_cube.xml"; - argv[3] = "-ss"; - argv[4] = "1111"; FolderTestFixture::setUp(); CopyResourceToTempdir("four_cube.xml"); CopyResourceToTempdir("four_cube.gmy"); diff --git a/Code/unittests/configuration/CommandLineTests.h b/Code/unittests/configuration/CommandLineTests.h index 20b7d9510..671b4535c 100644 --- a/Code/unittests/configuration/CommandLineTests.h +++ b/Code/unittests/configuration/CommandLineTests.h @@ -31,12 +31,10 @@ namespace hemelb void setUp() { configFile = Resource("four_cube.xml").Path(); - argc = 5; + argc = 3; argv[0] = "hemelb"; argv[1] = "-in"; argv[2] = configFile.c_str(); - argv[3] = "-ss"; - argv[4] = "1111"; FolderTestFixture::setUp(); options = new hemelb::configuration::CommandLine(argc, argv); } diff --git a/Code/unittests/geometry/GeometryReaderTests.h b/Code/unittests/geometry/GeometryReaderTests.h index fbc64d901..8dba1e7c1 100644 --- a/Code/unittests/geometry/GeometryReaderTests.h +++ b/Code/unittests/geometry/GeometryReaderTests.h @@ -39,8 +39,7 @@ namespace hemelb { FolderTestFixture::setUp(); timings = new reporting::Timers(Comms()); - reader = new GeometryReader(false, - hemelb::lb::lattices::D3Q15::GetLatticeInfo(), + reader = new GeometryReader(hemelb::lb::lattices::D3Q15::GetLatticeInfo(), *timings, Comms()); lattice = NULL; fourCube = FourCubeLatticeData::Create(Comms()); diff --git a/Code/unittests/geometry/NeedsTests.h b/Code/unittests/geometry/NeedsTests.h index cc6b98c06..6d4214c7c 100644 --- a/Code/unittests/geometry/NeedsTests.h +++ b/Code/unittests/geometry/NeedsTests.h @@ -7,7 +7,7 @@ #ifndef HEMELB_UNITTESTS_GEOMETRY_NEEDSTESTS_H #define HEMELB_UNITTESTS_GEOMETRY_NEEDSTESTS_H #include "geometry/needs/Needs.h" -#include "unittests/helpers/MockNetHelper.h" +#include "unittests/helpers/MockCommsHelper.h" #include "unittests/helpers/CppUnitCompareVectors.h" #include @@ -19,7 +19,7 @@ namespace hemelb { using namespace hemelb::geometry; using namespace hemelb::unittests::helpers; - class NeedsTests : public CppUnit::TestFixture, MockNetHelper + class NeedsTests : public CppUnit::TestFixture, MockCommsHelper { CPPUNIT_TEST_SUITE (NeedsTests); CPPUNIT_TEST (TestReadingOne); @@ -38,22 +38,21 @@ namespace hemelb void tearDown() { delete mockedNeeds; - MockNetHelper::tearDown(); + MockCommsHelper::tearDown(); } void TestReadingOne() { SetupMocks(6, 2, 5, 0); - CPPUNIT_ASSERT_EQUAL(communicatorMock->Size(),5); + CPPUNIT_ASSERT_EQUAL(communicator->Size(), 5); // Start to record the expected communications calls. // First we gather the counts of blocks needed on each reading core - auto mock = std::dynamic_pointer_cast(communicatorMock); const std::vector block_size_reading_core_0 = {1, 2, 1, 2, 1}; // We don't really care about what the other ranks need from RC1 const std::vector block_size_reading_core_1 = {1, 0, 2, 0, 0}; - mock->AddGatherResult(block_size_reading_core_0); - mock->AddGatherResult(block_size_reading_core_1); + MockComms()->RequireGather(block_size_reading_core_0, 0); + MockComms()->RequireGather(block_size_reading_core_1, 1); // Now we list the blocks needed from reading core 0 (us) // Note counts match above @@ -64,10 +63,10 @@ namespace hemelb 2, 4, 4 }; - mock->AddGatherVResult(blocks_needed_from_reading_core_0); + MockComms()->RequireGatherV(blocks_needed_from_reading_core_0, block_size_reading_core_0, 0); // Blocks needed from RC1 - const std::vector blocks_needed_from_reading_core_1 = {1}; - mock->AddGatherVResult(blocks_needed_from_reading_core_1); + const std::vector blocks_needed_from_reading_core_1 = {1, 0, 0}; + MockComms()->RequireGatherV(blocks_needed_from_reading_core_1, block_size_reading_core_1, 1); ShareMockNeeds(); @@ -95,7 +94,6 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(needing_block_4, mockedNeeds->ProcessorsNeedingBlock(4)); CPPUNIT_ASSERT_EQUAL(needing_block_5, mockedNeeds->ProcessorsNeedingBlock(5)); // I guess this means that no comms went through the net - netMock->ExpectationsAllCompleted(); } void TestNonReading() @@ -103,10 +101,9 @@ namespace hemelb // We are core 2 - a no reading one SetupMocks(6, 2, 5, 2); - CPPUNIT_ASSERT_EQUAL(communicatorMock->Size(),5); + CPPUNIT_ASSERT_EQUAL(communicator->Size(),5); // Record the expected communications calls. - auto mock = std::dynamic_pointer_cast(communicatorMock); // First we gather the counts of blocks needed on each reading core // For RC0 we know this from above @@ -114,8 +111,8 @@ namespace hemelb // We don't really care about what the other ranks need from RC1 const std::vector block_size_reading_core_1 = {0, 0, 2, 0, 0}; - mock->AddGatherResult(block_size_reading_core_0); - mock->AddGatherResult(block_size_reading_core_1); + MockComms()->RequireGather(block_size_reading_core_0, 0); + MockComms()->RequireGather(block_size_reading_core_1, 1); // Now we list the blocks needed from RC0 // Note counts match above @@ -126,10 +123,10 @@ namespace hemelb 2, 4, 4 }; - mock->AddGatherVResult(blocks_needed_from_reading_core_0); + MockComms()->RequireGatherV(blocks_needed_from_reading_core_0, block_size_reading_core_0, 0); // Blocks needed from RC1 const std::vector blocks_needed_from_reading_core_1 = {1 ,3}; - mock->AddGatherVResult(blocks_needed_from_reading_core_1); + MockComms()->RequireGatherV(blocks_needed_from_reading_core_1, block_size_reading_core_1, 1); ShareMockNeeds(); // Finally, I would expect the resulting array of needs to be empty @@ -142,7 +139,6 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(empty_needs_array, mockedNeeds->ProcessorsNeedingBlock(4)); CPPUNIT_ASSERT_EQUAL(empty_needs_array, mockedNeeds->ProcessorsNeedingBlock(5)); - netMock->ExpectationsAllCompleted(); } void SetupMocks(const site_t block_count, @@ -154,7 +150,7 @@ namespace hemelb readingCores = reading_cores; rank = current_core; size = core_count; - MockNetHelper::setUp(core_count,current_core); + MockCommsHelper::setUp(core_count,current_core); inputNeededBlocks = std::vector(block_count); @@ -170,7 +166,7 @@ namespace hemelb mockedNeeds = new Needs(blockCount, inputNeededBlocks, readingCores, - communicatorMock, + communicator, false); } diff --git a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h index 77a8c5171..fbfda6e7d 100644 --- a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h +++ b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h @@ -7,10 +7,10 @@ #ifndef HEMELB_UNITTESTS_GEOMETRY_NEIGHBOURING_NEIGHBOURINGDATAMANAGERTESTS_H #define HEMELB_UNITTESTS_GEOMETRY_NEIGHBOURING_NEIGHBOURINGDATAMANAGERTESTS_H #include "net/phased/StepManager.h" -#include "net/phased/NetConcern.h" #include "geometry/neighbouring/NeighbouringDataManager.h" -#include "unittests/helpers/MockNetHelper.h" +#include "unittests/helpers/MockCommsHelper.h" #include "comm/MpiEnvironment.h" +#include "comm/AsyncConcern.h" namespace hemelb { @@ -22,7 +22,7 @@ namespace hemelb { using namespace hemelb::geometry::neighbouring; class NeighbouringDataManagerTests : public FourCubeBasedTestFixture, - public MockNetHelper + public MockCommsHelper { CPPUNIT_TEST_SUITE ( NeighbouringDataManagerTests); CPPUNIT_TEST ( TestConstruct); @@ -44,25 +44,24 @@ namespace hemelb { FourCubeBasedTestFixture::setUp(); data = &latDat->GetNeighbouringData(); - MockNetHelper::setUp(1, 0); - manager = new NeighbouringDataManager(*latDat, *data, *netMock); + MockCommsHelper::setUp(1, 0); + manager = new NeighbouringDataManager(*latDat, *data, commQ); } void UseRealCommunicator() { - delete manager; - delete netMock; + // delete manager; - communicatorMock = comm::MpiEnvironment::World(); - netMock = new net::NetMock(communicatorMock); - manager = new NeighbouringDataManager(*latDat, *data, *netMock); + // communicator = comm::MpiEnvironment::World(); + // commQ = comm::Async::New(communicator); + // manager = new NeighbouringDataManager(*latDat, *data, commQ); } void tearDown() { delete manager; FourCubeBasedTestFixture::tearDown(); - MockNetHelper::tearDown(); + MockCommsHelper::tearDown(); } void TestConstruct() @@ -101,11 +100,18 @@ namespace hemelb void TestShareNeedsOneProc() { - UseRealCommunicator(); - manager->RegisterNeededSite(43); - + std::vector neededsites = {43}; + manager->RegisterNeededSite(neededsites[0]); + + // The MapA2A needs a barrier but as the only needs are + // local this is true immediately + MockComms()->RequireIbarrier([]() { return true; }); + + // We then send this need to ourself + MockComms()->RequireSend(neededsites, 0, 0); + MockComms()->RequireRecv(neededsites, 0, 0); manager->ShareNeeds(); - + MockComms()->ExpectationsAllCompleted(); CPPUNIT_ASSERT_EQUAL(manager->GetNeedsForProc(0).size(), static_cast::size_type> (1)); CPPUNIT_ASSERT_EQUAL(manager->GetNeedsForProc(0).front(), static_cast (43)); @@ -113,8 +119,12 @@ namespace hemelb void TestShareConstantDataOneProc() { - UseRealCommunicator(); - manager->RegisterNeededSite(43); + std::vector neededsites = {43}; + manager->RegisterNeededSite(neededsites[0]); + + MockComms()->RequireIbarrier([]() { return true; }); + MockComms()->RequireSend(neededsites, 0, 0); + MockComms()->RequireRecv(neededsites, 0, 0); manager->ShareNeeds(); // Now, transfer the data about that site. @@ -126,39 +136,37 @@ namespace hemelb SiteData expectedData = exampleSite.GetSiteData(); SiteData fixtureData = exampleSite.GetSiteData(); - netMock->RequireSend(&expectedData.GetWallIntersectionData(), - 1, - 0, - "WallIntersectionDataToSelf"); - netMock->RequireReceive(&fixtureData.GetWallIntersectionData(), - 1, - 0, - "WallIntersectionDataFromSelf"); - netMock->RequireSend(&expectedData.GetIoletIntersectionData(), - 1, - 0, - "IoletIntersectionDataToSelf"); - netMock->RequireReceive(&fixtureData.GetIoletIntersectionData(), - 1, - 0, - "IoletIntersectionDataFromSelf"); - netMock->RequireSend(&expectedData.GetIoletId(), 1, 0, "IoletIdToSelf"); - netMock->RequireReceive(&fixtureData.GetIoletId(), 1, 0, "IoletIdFromSelf"); - netMock->RequireSend(&expectedData.GetSiteType(), 1, 0, "SiteTypeToSelf"); - netMock->RequireReceive(&fixtureData.GetSiteType(), 1, 0, "SiteTypeFromSelf"); - - netMock->RequireSend(exampleSite.GetWallDistances(), + MockComms()->RequireSend(expectedData.GetWallIntersectionData(), + 0, 0); + MockComms()->RequireRecv(fixtureData.GetWallIntersectionData(), + 0, 0); + MockComms()->RequireSend(&expectedData.GetIoletIntersectionData(), + 1, + 0, + 0); + MockComms()->RequireRecv(&fixtureData.GetIoletIntersectionData(), + 1, + 0, + 0); + MockComms()->RequireSend(&expectedData.GetIoletId(), 1, 0, 0); + MockComms()->RequireRecv(&fixtureData.GetIoletId(), 1, 0, 0); + MockComms()->RequireSend(&expectedData.GetSiteType(), 1, 0, 0); + MockComms()->RequireRecv(&fixtureData.GetSiteType(), 1, 0, 0); + + MockComms()->RequireSend(exampleSite.GetWallDistances(), lb::lattices::D3Q15::NUMVECTORS - 1, 0, - "WallToSelf"); - netMock->RequireReceive(exampleSite.GetWallDistances(), + 0); + MockComms()->RequireRecv(exampleSite.GetWallDistances(), lb::lattices::D3Q15::NUMVECTORS - 1, 0, - "WallFromSelf"); - netMock->RequireSend(&exampleSite.GetWallNormal(), 1, 0, "NormalToSelf"); - netMock->RequireReceive(&exampleSite.GetWallNormal(), 1, 0, "NormalFromSelf"); + 0); + MockComms()->RequireSend(&exampleSite.GetWallNormal(), 1, 0, 0); + MockComms()->RequireRecv(&exampleSite.GetWallNormal(), 1, 0, 0); manager->TransferNonFieldDependentInformation(); - netMock->ExpectationsAllCompleted(); + + MockComms()->ExpectationsAllCompleted(); + NeighbouringSite transferredSite = data->GetSite(43); CPPUNIT_ASSERT_EQUAL(exampleSite.GetSiteData(), transferredSite.GetSiteData()); for (unsigned int direction = 0; direction < lb::lattices::D3Q15::NUMVECTORS - 1; direction++) @@ -171,7 +179,7 @@ namespace hemelb void TestShareFieldDataOneProc() { - UseRealCommunicator(); + site_t targetGlobalOneDIdx = 43; LatticeVector targetGlobalThreeDIdx = latDat->GetSiteCoordsFromSiteId(targetGlobalOneDIdx); @@ -179,6 +187,11 @@ namespace hemelb for (unsigned int direction = 0; direction < 3; direction++) CPPUNIT_ASSERT_EQUAL(site_t(1), targetGlobalThreeDIdx[direction]); + + MockComms()->RequireIbarrier([]() { return true; }); + std::vector neededsites = {targetGlobalOneDIdx}; + MockComms()->RequireSend(neededsites, 0, 0); + MockComms()->RequireRecv(neededsites, 0, 0); manager->RegisterNeededSite(targetGlobalOneDIdx); manager->ShareNeeds(); @@ -188,19 +201,19 @@ namespace hemelb // It should arrive in the NeighbouringDataManager, from the values sent from the localLatticeData - netMock->RequireSend(const_cast (exampleSite.GetFOld ()), - lb::lattices::D3Q15::NUMVECTORS, - 0, - "IntersectionDataToSelf"); + MockComms()->RequireSend(const_cast (exampleSite.GetFOld ()), + lb::lattices::D3Q15::NUMVECTORS, + 0, + 0); std::vector receivedFOld(lb::lattices::D3Q15::NUMVECTORS, 53.0); - netMock->RequireReceive(&(receivedFOld[0]), - lb::lattices::D3Q15::NUMVECTORS, - 0, - "IntersectionDataFromSelf"); + MockComms()->RequireRecv(&(receivedFOld[0]), + lb::lattices::D3Q15::NUMVECTORS, + 0, + 0); manager->TransferFieldDependentInformation(); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); NeighbouringSite transferredSite = data->GetSite(targetGlobalOneDIdx); for (unsigned int direction = 0; direction < lb::lattices::D3Q15::NUMVECTORS; direction++) @@ -212,33 +225,38 @@ namespace hemelb void TestShareFieldDataOneProcViaIterableAction() { - UseRealCommunicator(); site_t targetGlobalOneDIdx = 43; site_t targetLocalIdx = latDat->GetLocalContiguousIdFromGlobalNoncontiguousId(targetGlobalOneDIdx); + + MockComms()->RequireIbarrier([]() { return true; }); + std::vector neededsites = {targetGlobalOneDIdx}; + MockComms()->RequireSend(neededsites, 0, 0); + MockComms()->RequireRecv(neededsites, 0, 0); manager->RegisterNeededSite(targetGlobalOneDIdx); manager->ShareNeeds(); // Now, transfer the data about that site. Site < LatticeData > exampleSite = latDat->GetSite(targetLocalIdx); - // It should arrive in the NeighbouringDataManager, from the values sent from the localLatticeData + // It should arrive in the NeighbouringDataManager, from + // the values sent from the localLatticeData - netMock->RequireSend(const_cast (exampleSite.GetFOld ()), - lb::lattices::D3Q15::NUMVECTORS, - 0, - "IntersectionDataToSelf"); + MockComms()->RequireSend(const_cast (exampleSite.GetFOld ()), + lb::lattices::D3Q15::NUMVECTORS, + 0, + 0); std::vector receivedFOld(lb::lattices::D3Q15::NUMVECTORS, 53.0); - netMock->RequireReceive(&(receivedFOld[0]), - lb::lattices::D3Q15::NUMVECTORS, - 0, - "IntersectionDataFromSelf"); - + MockComms()->RequireRecv(&(receivedFOld[0]), + lb::lattices::D3Q15::NUMVECTORS, + 0, + 0); + net::phased::StepManager stepManager; stepManager.RegisterIteratedActorSteps(*manager, 0); - net::phased::NetConcern netConcern = net::phased::NetConcern(*netMock); + comm::AsyncConcern netConcern = comm::AsyncConcern(commQ); stepManager.RegisterCommsForAllPhases(netConcern); stepManager.CallActions(); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); NeighbouringSite transferredSite = data->GetSite(targetGlobalOneDIdx); for (unsigned int direction = 0; direction < lb::lattices::D3Q15::NUMVECTORS; direction++) diff --git a/Code/unittests/helpers/MockCommsHelper.h b/Code/unittests/helpers/MockCommsHelper.h new file mode 100644 index 000000000..9f2fb87e5 --- /dev/null +++ b/Code/unittests/helpers/MockCommsHelper.h @@ -0,0 +1,48 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_UNITTESTS_HELPERS_MOCKCOMMSHELPER_H +#define HEMELB_UNITTESTS_HELPERS_MOCKCOMMSHELPER_H + +#include "unittests/helpers/MockCommunicator.h" + +namespace hemelb +{ + namespace unittests + { + namespace helpers + { + class MockCommsHelper + { + protected: + MockCommsHelper() + { + } + void setUp(const proc_t core_count, const proc_t current_core) + { + communicator = std::make_shared(current_core, core_count); + commQ = comm::Async::New(communicator); + //communicator = communicatorMock; + } + void tearDown() + { + } + + std::shared_ptr MockComms() { + auto ans = std::dynamic_pointer_cast(communicator); + CPPUNIT_ASSERT_MESSAGE("Cannot cast Communicator to a MockCommunicator", ans); + return ans; + } + //std::shared_ptr communicatorMock; + comm::Communicator::Ptr communicator; + comm::Async::Ptr commQ; + }; + + } + } +} + +#endif // HEMELB_UNITTESTS_HELPERS_MOCKCOMMSHELPER_H diff --git a/Code/unittests/helpers/MockCommunicator.h b/Code/unittests/helpers/MockCommunicator.h new file mode 100644 index 000000000..b7f15a85b --- /dev/null +++ b/Code/unittests/helpers/MockCommunicator.h @@ -0,0 +1,469 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATOR_H +#define HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATOR_H + +#include +#include "comm/Communicator.h" +#include "comm/Request.h" + +namespace hemelb +{ + namespace unittests + { + namespace helpers + { + using namespace comm; + + class MockRequest : public comm::Request + { + public: + MockRequest() : comm::Request(), testCount(0) { + } + virtual void Wait() { + + } + virtual bool Test() { + if (testCount > 1) { + Wait(); + return true; + } else { + ++testCount; + return false; + } + } + + private: + int testCount; + }; + + class MockIbarrierRequest : public MockRequest + { + public: + typedef std::function NullaryPredicate; + MockIbarrierRequest(NullaryPredicate fn) : done(fn){ + } + virtual void Wait() { + CPPUNIT_ASSERT_MESSAGE("Predicate must be done before waiting!", done()); + } + virtual bool Test() { + return done(); + } + private: + NullaryPredicate done; + }; + + class MockRequestList : public comm::RequestList + { + std::vector reqs; + public: + virtual size_t size() const { + return reqs.size(); + } + virtual void resize(size_t i) { + reqs.resize(i); + } + virtual void push_back(Request::Ptr rp) { + reqs.push_back(rp); + } + + virtual void clear() { + reqs.clear(); + } + + virtual void set(size_t i, Request::Ptr rp) { + reqs[i] = rp; + } + + virtual void WaitAll() { + for (auto r: reqs) { + r->Wait(); + } + } + virtual bool TestAll() { + bool alldone = true; + for (auto r: reqs) { + if (!r->Test()) { + alldone = false; + break; + } + } + return alldone; + } + }; + + // class MockSendReq : public MockRequest + // { + // MockSendReq(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + // int dest, int tag) : mSendbuf(sendbuf), mSendcount(sendcount), mSendtype(sendtype), + // mDest(dest), mTag(tag) + // { + // } + + // private: + // const void *mSendbuf; + // int mSendcount; + // MPI_Datatype mSendtype; + // int mDest; + // int mTag; + // }; + + template + struct is_std_vector : public std::false_type {}; + template + struct is_std_vector > : public std::true_type {}; + + class MockCommunicator : public comm::Communicator + { + public: + /*** + * Constructor for a dummy communicator + * Can be useful for testing but can't actually be used + * @param rank + * @param size + */ + MockCommunicator(int rank_, int size_) : + rank(rank_), size(size_) + { + + } + + virtual inline int Rank() const + { + return rank; + } + virtual inline int Size() const + { + return size; + } + +#define NOTIMPLEMENTED CPPUNIT_FAIL("Communicator function not implemented") + + virtual void Abort(int errCode) const { NOTIMPLEMENTED; } + + virtual Ptr Duplicate() const { NOTIMPLEMENTED; return nullptr; } + + virtual std::shared_ptr GetGroup() const { NOTIMPLEMENTED; return nullptr; } + virtual Ptr Create(std::shared_ptr grp) const { NOTIMPLEMENTED; return nullptr; } + virtual std::shared_ptr OpenFile(const std::string& filename, int mode, + const MPI_Info info = MPI_INFO_NULL) const { NOTIMPLEMENTED; return nullptr; } + + virtual std::shared_ptr MakeRequestList() const { + return std::make_shared(); + } + virtual void Barrier() const { NOTIMPLEMENTED; } + + virtual std::shared_ptr Ibarrier() const { + CPPUNIT_ASSERT(ibarrier_conditions.size()); + auto& cond = ibarrier_conditions.front(); + auto ans = std::make_shared(cond); + ibarrier_conditions.pop_front(); + return ans; + } + + virtual bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const { + // Look through the pending recvs and point to the first one that matches + auto maybe_match = recv_data.begin(); + auto end = recv_data.end(); + for (; maybe_match != end; ++maybe_match) { + if (source == MPI_ANY_SOURCE || source == maybe_match->src) { + if (tag == MPI_ANY_TAG || tag == maybe_match->tag) { + break; + } + } + } + + // No match - done + if (maybe_match == end) { + return false; + } + + // Got a match, setup the status then move it to the front + // of the queue + stat->MPI_SOURCE = maybe_match->src; + stat->MPI_TAG = maybe_match->tag; + auto tmp = std::move(*maybe_match); + recv_data.erase(maybe_match); + recv_data.push_front(std::move(tmp)); + return true; + } + + void ExpectationsAllCompleted() { + CPPUNIT_ASSERT_EQUAL(size_t(0), gather_data.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), gatherv_data.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), send_data.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), ssend_data.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), recv_data.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), ibarrier_conditions.size()); + } + + template + void RequireGather(const std::vector& data, int root) + { + CPPUNIT_ASSERT_EQUAL_MESSAGE("Number of elements must be a multiple of comm size", + 0, int(data.size()) % Size()); + int sendcount = data.size() / Size(); + gather_data.push_back({data.data() + sendcount*Rank(), sendcount, comm::MpiDataType(), + data.data(), sendcount, comm::MpiDataType(), + root}); + } + + template + void RequireGatherV(const std::vector& data, const std::vector& recvcounts, int root) + { + CPPUNIT_ASSERT_EQUAL(Size(), int(recvcounts.size())); + std::vector displs(Size()); + int total = 0; + for(size_t i = 0; i < Size(); ++i) { + displs[i] = total; + total += recvcounts[i]; + } + auto my_send_data = data.data() + displs[Rank()]; + CPPUNIT_ASSERT_EQUAL_MESSAGE("Size of data does not match sum of recvcounts", + size_t(total), data.size()); + gatherv_data.push_back({my_send_data, recvcounts[Rank()], comm::MpiDataType(), + data.data(), recvcounts.data(), std::move(displs), comm::MpiDataType(), + root}); + } + + template + void RequireSend(const T* valPtr, int count, int dest, int tag) + { + static_assert(!std::is_pointer::value, + "You are trying to require the sending of a pointer type"); + static_assert(!is_std_vector::value, + "You are trying to require the sending of a std::vector type"); + + CPPUNIT_ASSERT(dest >= 0); + CPPUNIT_ASSERT(dest < Size()); + send_data.push_back({valPtr, count, comm::MpiDataType(), dest, tag}); + } + template + void RequireSend(const T& val, int dest, int tag) + { + RequireSend(&val, 1, dest, tag); + } + template + void RequireSend(const std::vector& vals, int dest, int tag) + { + RequireSend(vals.data(), vals.size(), dest, tag); + } + + template + void RequireRecv(const T* valPtr, int count, int src, int tag) + { + static_assert(!std::is_pointer::value, + "You are trying to require the receiving of a pointer type"); + static_assert(!is_std_vector::value, + "You are trying to require the receiving of a std::vector type"); + CPPUNIT_ASSERT(src >= 0); + CPPUNIT_ASSERT(src < Size()); + recv_data.push_back({valPtr, count, comm::MpiDataType(), src, tag}); + } + template + void RequireRecv(const T& val, int src, int tag) + { + RequireRecv(&val, 1, src, tag); + } + template + void RequireRecv(const std::vector& vals, int src, int tag) + { + RequireRecv(vals.data(), vals.size(), src, tag); + } + + void RequireIbarrier(MockIbarrierRequest::NullaryPredicate cond) { + ibarrier_conditions.push_back(cond); + } + private: + int rank, size; + + struct SendData { + const void* expected_buf; + int count; + MPI_Datatype type; + int dest; + int tag; + }; + + struct RecvData { + const void* ans_buf; + int count; + MPI_Datatype type; + int src; + int tag; + }; + + struct GatherData { + const void* send_buf; + int sendcount; + MPI_Datatype sendtype; + const void* ans_buf; + int recvcount; + MPI_Datatype recvtype; + int root; + }; + + struct GatherVData { + const void* send_buf; + int sendcount; + MPI_Datatype sendtype; + const void* ans_buf; + const int* recvcounts; + const std::vector displs; + MPI_Datatype recvtype; + + int root; + }; + + mutable std::deque send_data; + mutable std::deque ssend_data; + mutable std::deque recv_data; + mutable std::deque gather_data; + mutable std::deque gatherv_data; + mutable std::deque ibarrier_conditions; + + virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const { NOTIMPLEMENTED; } + virtual std::shared_ptr IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const { NOTIMPLEMENTED; return nullptr; } + virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const { NOTIMPLEMENTED; } + virtual std::shared_ptr IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const { NOTIMPLEMENTED; return nullptr; } + virtual std::shared_ptr IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const { NOTIMPLEMENTED; return nullptr; } + virtual void ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const { NOTIMPLEMENTED; } + + virtual void GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype, + int root) const { + CPPUNIT_ASSERT(gather_data.size()); + + auto& gd = gather_data.front(); + CPPUNIT_ASSERT_EQUAL(gd.sendcount, sendcount); + CPPUNIT_ASSERT_EQUAL(gd.sendtype, sendtype); + CPPUNIT_ASSERT_EQUAL(gd.root, root); + int elSize; + MPI_Type_size(gd.sendtype, &elSize); + + { // assert that send[:] == relevant bit of stored data + int size_bytes = sendcount * elSize; + const char* expected = static_cast(gd.send_buf); + const char* actual = static_cast(send); + for (auto i = 0; i < size_bytes; ++i) + CPPUNIT_ASSERT_EQUAL(expected[i], actual[i]); + } + + if (Rank() == root) { + CPPUNIT_ASSERT_EQUAL(gd.recvcount, recvcount); + CPPUNIT_ASSERT_EQUAL(gd.recvtype, recvtype); + + std::memcpy(recv, gd.ans_buf, recvcount * elSize * Size()); + } + gather_data.pop_front(); + } + + virtual void GathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype, + int root) const { + CPPUNIT_ASSERT(gatherv_data.size()); + + auto& gd = gatherv_data.front(); + CPPUNIT_ASSERT_EQUAL(gd.sendcount, sendcount); + CPPUNIT_ASSERT_EQUAL(gd.sendtype, sendtype); + CPPUNIT_ASSERT_EQUAL(gd.root, root); + + int elSize; + MPI_Type_size(gd.sendtype, &elSize); + + { // assert that send[:] == relevant bit of stored data + int size_bytes = sendcount * elSize; + const char* expected = static_cast(gd.send_buf); + const char* actual = static_cast(sendbuf); + for (auto i = 0; i < size_bytes; ++i) + CPPUNIT_ASSERT_EQUAL(expected[i], actual[i]); + } + + if (Rank() == root) { + // The recv counts and displacements must match on root + for(auto i = 0; i < Size(); ++i) { + CPPUNIT_ASSERT_EQUAL(gd.recvcounts[i], recvcounts[i]); + CPPUNIT_ASSERT_EQUAL(gd.displs[i], displs[i]); + } + CPPUNIT_ASSERT_EQUAL(gd.recvtype, recvtype); + + int nElTotal = std::accumulate(recvcounts, recvcounts+Size(), 0); + std::memcpy(recvbuf, gd.ans_buf, nElTotal*elSize); + } + gatherv_data.pop_front(); + } + + virtual void AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } + virtual void AllgathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } + virtual void AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } + virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { + CPPUNIT_ASSERT(send_data.size()); + + auto& sd = send_data.front(); + CPPUNIT_ASSERT_EQUAL(sd.count, sendcount); + CPPUNIT_ASSERT_EQUAL(sd.type, sendtype); + CPPUNIT_ASSERT_EQUAL(sd.dest, dest); + CPPUNIT_ASSERT_EQUAL(sd.tag, tag); + send_data.pop_front(); + } + virtual void SsendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { + CPPUNIT_ASSERT(ssend_data.size()); + + auto& sd = ssend_data.front(); + CPPUNIT_ASSERT_EQUAL(sd.count, sendcount); + CPPUNIT_ASSERT_EQUAL(sd.type, sendtype); + CPPUNIT_ASSERT_EQUAL(sd.dest, dest); + CPPUNIT_ASSERT_EQUAL(sd.tag, tag); + ssend_data.pop_front(); + } + + virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int src, int tag, MPI_Status* stat) const { + CPPUNIT_ASSERT(recv_data.size()); + + auto& rd = recv_data.front(); + CPPUNIT_ASSERT_EQUAL(rd.count, recvcount); + CPPUNIT_ASSERT_EQUAL(rd.type, recvtype); + CPPUNIT_ASSERT_EQUAL(rd.src, src); + CPPUNIT_ASSERT_EQUAL(rd.tag, tag); + + int elSize; + MPI_Type_size(rd.type, &elSize); + + std::memcpy(recvbuf, rd.ans_buf, recvcount * elSize); + recv_data.pop_front(); + + } + + virtual std::shared_ptr IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { + SendImpl(sendbuf, sendcount, sendtype, dest, tag); + return std::make_shared(); + } + + virtual std::shared_ptr IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { + SsendImpl(sendbuf, sendcount, sendtype, dest, tag); + return std::make_shared(); + } + virtual std::shared_ptr IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int tag) const { + RecvImpl(recvbuf, recvcount, recvtype, source, tag, MPI_STATUS_IGNORE); + return std::make_shared(); + } + }; + + } + } +} + +#endif // HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATOR_H diff --git a/Code/unittests/helpers/MockCommunicatorTests.h b/Code/unittests/helpers/MockCommunicatorTests.h new file mode 100644 index 000000000..c7dd372d8 --- /dev/null +++ b/Code/unittests/helpers/MockCommunicatorTests.h @@ -0,0 +1,188 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATORTESTS_H +#define HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATORTESTS_H + +#include +#include "unittests/helpers/MockCommunicator.h" + +namespace hemelb +{ + namespace unittests + { + namespace helpers + { + class MockCommunicatorTests : public CppUnit::TestFixture + { + CPPUNIT_TEST_SUITE( MockCommunicatorTests); + CPPUNIT_TEST( TestSimple); + + CPPUNIT_TEST( TestSend); + CPPUNIT_TEST( TestIsend); + + CPPUNIT_TEST( TestRecv); + CPPUNIT_TEST( TestIrecv); + + CPPUNIT_TEST( TestGatherRoot); + CPPUNIT_TEST( TestGatherNonRoot); + + CPPUNIT_TEST( TestGatherV); + + CPPUNIT_TEST( TestIprobe); + CPPUNIT_TEST( TestIbarrier); + + CPPUNIT_TEST_SUITE_END(); + + public: + + void TestSimple() { + auto comm = std::make_shared(0, 1023); + CPPUNIT_ASSERT_EQUAL(0, comm->Rank()); + CPPUNIT_ASSERT_EQUAL(1023, comm->Size()); + } + + void TestSend() { + // Send a message 0 -> 1 + auto comm = std::make_shared(0, 2); + int number = 42; + comm->RequireSend(number, 1, 0); + comm->Send(42, 1); + comm->ExpectationsAllCompleted(); + } + + void TestIsend() { + // Send a message 0 -> 1 + auto comm = std::make_shared(0, 2); + int number = 42; + comm->RequireSend(number, 1, 0); + auto req = comm->Isend(42, 1); + req->Wait(); + comm->ExpectationsAllCompleted(); + } + + void TestRecv() { + // Send a message 0 -> 1 + auto comm = std::make_shared(1, 2); + int number = 42; + comm->RequireRecv(number, 1, 0); + int ans; + comm->Recv(ans, 1); + comm->ExpectationsAllCompleted(); + CPPUNIT_ASSERT_EQUAL(number, ans); + } + + void TestIrecv() { + // Send a message 0 -> 1 + auto comm = std::make_shared(1, 2); + int number = 42; + comm->RequireRecv(number, 1, 0); + int ans; + auto req = comm->Irecv(ans, 1); + req->Wait(); + comm->ExpectationsAllCompleted(); + CPPUNIT_ASSERT_EQUAL(number, ans); + } + + void TestGatherRoot() { + const int rank = 2; + const int size = 8; + auto comm = std::make_shared(rank, size); + + std::vector vals{9,8,7,6,5,4,3,2}; + comm->RequireGather(vals, 2); + + int myval = 7; + auto ans = comm->Gather(myval, 2); + + CPPUNIT_ASSERT_EQUAL(size, int(ans.size())); + comm->ExpectationsAllCompleted(); + } + void TestGatherNonRoot() { + const int rank = 0; + const int size = 8; + auto comm = std::make_shared(rank, size); + + std::vector vals{9,8,7,6,5,4,3,2}; + comm->RequireGather(vals, 2); + + int myval = 9; + auto ans = comm->Gather(myval, 2); + + CPPUNIT_ASSERT_EQUAL(0, int(ans.size())); + comm->ExpectationsAllCompleted(); + } + + void TestGatherV() { + const int rank = 0; + const int size = 4; + auto comm = std::make_shared(rank, size); + + std::vector counts{1,2,3,0}; + std::vector data{3, 4,5, 6,7,8}; + comm->RequireGatherV(data, counts, 0); + + std::vector mydata{3}; + auto ans = comm->GatherV(mydata, counts, 0); + CPPUNIT_ASSERT_EQUAL(data, ans); + comm->ExpectationsAllCompleted(); + } + + void TestIprobe() { + const int rank = 0; + const int size = 4; + auto comm = std::make_shared(rank, size); + + int val1 = 42; + int val2 = 24; + comm->RequireRecv(val1, 1, 0); + comm->RequireRecv(val2, 2, 0); + // message q has recvs ordered (1,2) + MPI_Status stat; + CPPUNIT_ASSERT(comm->Iprobe(2, MPI_ANY_TAG, &stat)); + // messages should now be ordered (2,1) + int ans; + comm->Recv(ans, 2); + CPPUNIT_ASSERT_EQUAL(val2, ans); + + comm->Recv(ans, 1); + CPPUNIT_ASSERT_EQUAL(val1, ans); + + CPPUNIT_ASSERT(!comm->Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, &stat)); + comm->ExpectationsAllCompleted(); + } + + void TestIbarrier() { + const int rank = 0; + const int size = 4; + auto comm = std::make_shared(rank, size); + + // Try a dead simple one first + comm->RequireIbarrier([]() { + return true; + }); + auto req = comm->Ibarrier(); + CPPUNIT_ASSERT(req->Test()); + + // Now this guy will count the calls and only work on the 3rd go + int i = 0; + comm->RequireIbarrier([&i]() { + return ++i == 3; + }); + req = comm->Ibarrier(); + CPPUNIT_ASSERT(!req->Test()); + CPPUNIT_ASSERT(!req->Test()); + req->Wait(); + + comm->ExpectationsAllCompleted(); + } + }; + CPPUNIT_TEST_SUITE_REGISTRATION( MockCommunicatorTests); + } + } +} + +#endif diff --git a/Code/unittests/helpers/MockNetHelper.h b/Code/unittests/helpers/MockNetHelper.h deleted file mode 100644 index 1b6ca4e1f..000000000 --- a/Code/unittests/helpers/MockNetHelper.h +++ /dev/null @@ -1,178 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_HELPERS_MOCKNETHELPER_H -#define HEMELB_UNITTESTS_HELPERS_MOCKNETHELPER_H - -#include "unittests/net/NetMock.h" -#include - -namespace hemelb -{ - namespace unittests - { - namespace helpers - { - using namespace comm; - - class MockMpiCommunicator : public comm::Communicator - { - public: - /*** - * Constructor for a dummy communicator - * Can be useful for testing but can't actually be used - * @param rank - * @param size - */ - MockMpiCommunicator(int rank_, int size_) : - rank(rank_), size(size_) - { - - } - - virtual inline int Rank() const - { - return rank; - } - virtual inline int Size() const - { - return size; - } - -#define NOTIMPLEMENTED CPPUNIT_FAIL("Communicator function not implemented") - - virtual void Abort(int errCode) const { NOTIMPLEMENTED; } - - virtual Ptr Duplicate() const { NOTIMPLEMENTED; } - - virtual std::shared_ptr GetGroup() const { NOTIMPLEMENTED; } - virtual Ptr Create(std::shared_ptr grp) const { NOTIMPLEMENTED; } - virtual std::shared_ptr OpenFile(const std::string& filename, int mode, - const MPI_Info info = MPI_INFO_NULL) const { NOTIMPLEMENTED; } - - virtual std::shared_ptr MakeRequestList() const { NOTIMPLEMENTED; } - virtual void Barrier() const { NOTIMPLEMENTED; } - virtual std::shared_ptr Ibarrier() const { NOTIMPLEMENTED; } - - virtual bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const { NOTIMPLEMENTED; } - template - void AddGatherResult(const std::vector& data) - { - CPPUNIT_ASSERT_EQUAL_MESSAGE("Number of elements must be a multiple of comm size", - 0, int(data.size()) % Size()); - gather_data.emplace_back(data.size() / Size(), sizeof(T), data.data()); - } - - template - void AddGatherVResult(const std::vector& data) - { - gatherv_data.emplace_back(data.size(), sizeof(T), data.data()); - } - private: - int rank, size; - typedef std::tuple gather_info; - mutable std::deque gather_data; - mutable std::deque gatherv_data; - - virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const { NOTIMPLEMENTED; } - virtual std::shared_ptr IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const { NOTIMPLEMENTED; } - virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const { NOTIMPLEMENTED; } - virtual std::shared_ptr IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const { NOTIMPLEMENTED; } - virtual std::shared_ptr IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const { NOTIMPLEMENTED; } - virtual void ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const { NOTIMPLEMENTED; } - - virtual void GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, - void* recv, int recvcount, MPI_Datatype recvtype, - int root) const { - CPPUNIT_ASSERT(gather_data.size()); - int nEl, elSize; - const void* ptr; - std::tie(nEl, elSize, ptr) = gather_data.front(); - CPPUNIT_ASSERT_EQUAL(nEl, sendcount); - - { // assert that send[:] == stored_data[sendcount*rank:sendcount*(rank+1)] - int size_bytes = nEl*elSize; - const char* expected = static_cast(ptr) + Rank()*size_bytes; - const char* actual = static_cast(send); - for (auto i = 0; i < size_bytes; ++i) - CPPUNIT_ASSERT_EQUAL(expected[i], actual[i]); - } - - if (Rank() == root) { - std::memcpy(recv, ptr, nEl*elSize*Size()); - } - gather_data.pop_front(); - } - - virtual void GathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, - void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype, - int root) const { - CPPUNIT_ASSERT(gatherv_data.size()); - int nElTotal, elSize; - const void* ptr; - std::tie(nElTotal, elSize, ptr) = gatherv_data.front(); - if (Rank() == root) - CPPUNIT_ASSERT_EQUAL_MESSAGE("This rank's count must match", - recvcounts[Rank()], sendcount); - - if (root == Rank()) { - // sum(recvcounts) == nElTotal (only significant on root of collective) - int sum_recvcounts = std::accumulate(recvcounts, recvcounts + Size(), 0); - CPPUNIT_ASSERT_EQUAL(nElTotal, sum_recvcounts); - } - // assert that send[:] == stored_data[sendcount*rank:sendcount*(rank+1)] - if (Rank() == root) { - std::memcpy(recvbuf, ptr, nElTotal*elSize); - } - gatherv_data.pop_front(); - } - - virtual void AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, - void* recv, int recvcount, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } - virtual void AllgathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, - void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } - virtual void AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, - void* recv, int recvcount, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } - virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, - int dest, int tag) const { NOTIMPLEMENTED; } - virtual void SsendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, - int dest, int tag) const { NOTIMPLEMENTED; } - virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, - int src, int tag, MPI_Status* stat) const { NOTIMPLEMENTED; } - virtual std::shared_ptr IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, - int dest, int tag) const { NOTIMPLEMENTED; } - virtual std::shared_ptr IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, - int dest, int tag) const { NOTIMPLEMENTED; } - virtual std::shared_ptr IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, - int source, int tag) const { NOTIMPLEMENTED; } - }; - - class MockNetHelper - { - protected: - MockNetHelper() : - communicatorMock(NULL), netMock(NULL) - { - } - void setUp(const proc_t core_count, const proc_t current_core) - { - communicatorMock = std::make_shared(current_core, core_count); - netMock = new net::NetMock(communicatorMock); - } - void tearDown() - { - delete netMock; - } - - comm::Communicator::Ptr communicatorMock; - net::NetMock *netMock; - }; - - } - } -} - -#endif // HEMELB_UNITTESTS_HELPERS_RANDOMSOURCE_H diff --git a/Code/unittests/helpers/helpers.h b/Code/unittests/helpers/helpers.h index 647f44859..aaa9708fc 100644 --- a/Code/unittests/helpers/helpers.h +++ b/Code/unittests/helpers/helpers.h @@ -8,5 +8,6 @@ #define HEMELB_UNITTESTS_HELPERS_HELPERS_H #include "unittests/helpers/RandomSourceTests.h" +#include "unittests/helpers/MockCommunicatorTests.h" #endif /* HEMELB_UNITTESTS_HELPERS_HELPERS_H */ diff --git a/Code/unittests/io/PathManagerTests.h b/Code/unittests/io/PathManagerTests.h index 5bbdb66c2..0c52a1580 100644 --- a/Code/unittests/io/PathManagerTests.h +++ b/Code/unittests/io/PathManagerTests.h @@ -29,13 +29,11 @@ namespace hemelb void setUp() { FolderTestFixture::setUp(); - argc = 5; + argc = 3; processorCount = 5; argv[0] = "hemelb"; argv[1] = "-in"; argv[2] = "config.xml"; - argv[3] = "-ss"; - argv[4] = "1111"; } void tearDown() diff --git a/Code/unittests/lbtests/IncompressibilityCheckerTests.h b/Code/unittests/lbtests/IncompressibilityCheckerTests.h index e8a9ca018..2806df875 100644 --- a/Code/unittests/lbtests/IncompressibilityCheckerTests.h +++ b/Code/unittests/lbtests/IncompressibilityCheckerTests.h @@ -55,7 +55,7 @@ namespace hemelb eps = 1e-9; timings = new hemelb::reporting::Timers(Comms()); - net = new net::Net(Comms()); + commQ = comm::Async::New(Comms()); } void tearDown() @@ -75,7 +75,7 @@ namespace hemelb void TestIncompressibilityCheckerRootNode() { hemelb::lb::IncompressibilityChecker incompChecker(latDat, - net, + Comms(), simState, *cache, *timings, @@ -138,7 +138,7 @@ namespace hemelb void TestIncompressibilityCheckerLeafNode() { hemelb::lb::IncompressibilityChecker incompChecker(latDat, - net, + Comms(), simState, *cache, *timings, @@ -169,7 +169,7 @@ namespace hemelb distribn_t largestDefaultDensity; distribn_t largestDefaultVelocityMagnitude; hemelb::reporting::Timers* timings; - net::Net* net; + comm::Async::Ptr commQ; distribn_t eps; }; diff --git a/Code/unittests/multiscale/MockIntercommunicatorTests.h b/Code/unittests/multiscale/MockIntercommunicatorTests.h index eb525c279..7ecbf4a8e 100644 --- a/Code/unittests/multiscale/MockIntercommunicatorTests.h +++ b/Code/unittests/multiscale/MockIntercommunicatorTests.h @@ -208,12 +208,10 @@ namespace hemelb LADD_FAIL(); int argc; const char* argv[5]; - argc = 5; + argc = 3; argv[0] = "hemelb"; argv[1] = "-in"; argv[2] = "four_cube_multiscale.xml"; - argv[3] = "-ss"; - argv[4] = "1111"; CopyResourceToTempdir("four_cube_multiscale.xml"); CopyResourceToTempdir("four_cube.gmy"); hemelb::configuration::CommandLine options(argc, argv); diff --git a/Code/unittests/net/LabelledRequest.h b/Code/unittests/net/LabelledRequest.h deleted file mode 100644 index c33044092..000000000 --- a/Code/unittests/net/LabelledRequest.h +++ /dev/null @@ -1,115 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_NET_LABELLEDREQUEST_H -#define HEMELB_UNITTESTS_NET_LABELLEDREQUEST_H -#include "net/StoredRequest.h" -namespace hemelb -{ - namespace unittests - { - namespace net - { - class LabelledRequest : public hemelb::net::SimpleRequest - { - public: - const std::string Label; - LabelledRequest(void *pointer, int count, MPI_Datatype type, proc_t rank, const std::string &label) : - SimpleRequest(pointer, count, type, rank), Label(label) - { - } - - virtual bool EnvelopeIdentical(const SimpleRequest & other) - { - bool this_ok = ( (Count == other.Count) && (Rank == other.Rank) && (Type == other.Type)); - if (!this_ok) - { - std::cerr << "Envelope different: " << Label << " R: " << Rank << " C: " << Count << " T: " << " : " - << " R" << other.Rank << " C " << other.Count << " T " << std::endl; - // uncomment the below for type diagnostics if your compiler supports them - /*std::cerr << "Types are:" << static_cast(Type) << " : " << static_cast(other.Type) << std::endl; - std::cerr << "Type candidates include" << " int: " << static_cast(MpiDataType()) - << " unsigned int :" << static_cast(MpiDataType()) - << " site_t: " << static_cast(MpiDataType()) - << " proc_t: " << static_cast(MpiDataType()) - << std::endl;*/ - } - /*else{ - std::cerr << "Envelope same: " << Label << " R: " << Rank << " C: " << Count << " T: " << " : " - << " R" << other.Rank << " C " << other.Count << " T " << std::endl; - }*/ - return this_ok; - } - - virtual bool PayloadIdentical(const SimpleRequest & other) - { - // reduction - bool ok = true; - for (int element = 0; element < Count; element++) - { - int size; - - MPI_Type_size(other.Type, &size); - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - bool this_ok = 0 - == std::memcmp(static_cast(other.Pointer) + size * element, - static_cast(Pointer) + size * element, - size); - if (!this_ok) - { - - std::cerr << "Unexpected data in request: " << Label << " R " << Rank << " C " << Count << " : " - << std::endl; - for (int i = 0; i < size; i++) - { - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - std::cerr << i << " : " - << static_cast(* (static_cast(Pointer) + size * element + i)) << " " - << static_cast(* (static_cast(other.Pointer) + size * element + i)) << std::endl; - } - std::cerr << std::endl; - } - /*// --- use this block for debugging - else - { - std::cerr << "Expected data in request: " << Label << " R " << Rank << " C " << Count << " : " - << std::endl; - for (int i = 0; i < size; i++) - { - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - std::cerr << i << " : " - << static_cast(* (static_cast(Pointer) + size * element + i)) << " " - << static_cast(* (static_cast(other.Pointer) + size * element + i)) << std::endl; - } - }*/ - ok = ok && this_ok; - } - return ok; - } - - virtual void Unpack(SimpleRequest & other) - { - for (int element = 0; element < Count; element++) - { - int size; - MPI_Type_size(other.Type, &size); - - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - std::memcpy(static_cast(other.Pointer) + size * element, - static_cast(Pointer) + size * element, - size); - } - } - }; - } - } -} - -#endif diff --git a/Code/unittests/net/NetMock.h b/Code/unittests/net/NetMock.h deleted file mode 100644 index e4c29d966..000000000 --- a/Code/unittests/net/NetMock.h +++ /dev/null @@ -1,39 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_NET_NETMOCK_H -#define HEMELB_UNITTESTS_NET_NETMOCK_H -#include -#include -#include -#include - -#include "constants.h" -#include "net/net.h" -#include "unittests/net/RecordingNet.h" - -namespace hemelb -{ - namespace unittests - { - namespace net - { - using namespace hemelb::net; - - class NetMock : public InterfaceDelegationNet,public RecordingNet - { - public: - NetMock(comm::Communicator::ConstPtr communicator) : - BaseNet(communicator), StoringNet(communicator), InterfaceDelegationNet(communicator), - RecordingNet(communicator) - { - } - - }; - } - } -} -#endif // ONCE diff --git a/Code/unittests/net/RecordingNet.h b/Code/unittests/net/RecordingNet.h deleted file mode 100644 index d3c7921f7..000000000 --- a/Code/unittests/net/RecordingNet.h +++ /dev/null @@ -1,185 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_NET_RECORDINGNET_H -#define HEMELB_UNITTESTS_NET_RECORDINGNET_H -#include -#include -#include -#include - -#include "constants.h" -#include "net/net.h" -#include "unittests/net/LabelledRequest.h" - -namespace hemelb -{ - namespace unittests - { - namespace net - { - using namespace hemelb::net; - /** - * A mock for the net class for testing. - * You must first give a complete, per-rank-ordered list of the sends and - * another of the receives that it is to carry out. - * - * For sends, it checks that the sent data is identical to that specified/ - * - * For recvs, it delivers the data that you specified. - * - * For both, it checks metadata. - * - * Once you have completed a "round" of communications, call - * ExpectationsAllCompleted to check that there are none outstanding. - * - */ - class RecordingNet : public virtual StoringNet - { - public: - RecordingNet(comm::Communicator::ConstPtr comms) : - BaseNet(comms), StoringNet(comms), requiredReceipts(), requiredSends() - { - } - - /** - * Specify that this rank should receive a message - * @param pointer - to the message data that will later be received - * @param count - of number of elements in the message - * @param rank - of the source of the message - * @param label - used in error reporting (should be unique) - */ - template void RequireReceive(T* pointer, unsigned int count, proc_t rank, - const std::string &label = "") - { - requiredReceipts[rank].push_back(LabelledRequest(pointer, - count, - comm::MpiDataType(), - rank, - label)); - } - /** - * Specify that this rank should send a message - * @param pointer - to the message data that should be sent - * @param count - of the number of elements in the message - * @param rank - of the destination of the message - * @param label - used in error reporting (should be unique) - */ - template void RequireSend(T* pointer, unsigned int count, proc_t rank, - const std::string &label = "") - { - requiredSends[rank].push_back(LabelledRequest(pointer, - count, - comm::MpiDataType(), - rank, - label)); - } - - /** - * Mock-execute queued receives. - * - * Does no actual communication, but copies in the mock data - * supplied to RequireReceive. It also checks that the receives - * match the required ones. - */ - void ReceivePointToPoint() - { - for (std::map::iterator it = receiveProcessorComms.begin(); - it != receiveProcessorComms.end(); ++it) - { - for (ProcComms::iterator message = it->second.begin(); message != it->second.end(); - message++) - { - - if (requiredReceipts[message->Rank].size() == 0) - { - // It should not be the case, that a request is made, but there is no recorded assertion in the list of recorded assertions. - // i.e., we've popped off all the recorded requests, and there are none left to match the request we just received. - CPPUNIT_ASSERT(requiredReceipts[message->Rank].size() != 0); - return; - } - - CPPUNIT_ASSERT(requiredReceipts[message->Rank].front().EnvelopeIdentical(*message)); - requiredReceipts[message->Rank].front().Unpack(*message); - - // This assertion just checks that "Unpack" has done it's job. - // It shouldn't fail due to problems in tested code - // Would only fail due to memory screwups or bad code in this class's Unpack method. - CPPUNIT_ASSERT(requiredReceipts[message->Rank].front().PayloadIdentical(*message)); - - requiredReceipts[message->Rank].pop_front(); - } - - } - - } - /** - * Mock-execute queued sends. - * - * Does no actual communication, but checks that the sent data - * matches the mock data supplied to RequireSend. It also checks that - * the sends match the required ones. - */ - void SendPointToPoint() - { - - for (std::map::iterator it = sendProcessorComms.begin(); - it != sendProcessorComms.end(); ++it) - { - for (ProcComms::iterator message = it->second.begin(); message != it->second.end(); - message++) - { - - if (requiredSends[message->Rank].size() == 0) - { - CPPUNIT_ASSERT(requiredSends[message->Rank].size() != 0); - return; - } - CPPUNIT_ASSERT(requiredSends[message->Rank].front().EnvelopeIdentical(*message)); - CPPUNIT_ASSERT(requiredSends[message->Rank].front().PayloadIdentical(*message)); - requiredSends[message->Rank].pop_front(); - } - - } - } - - /** - * Mock-wait - clears the message queue - */ - void WaitPointToPoint() - { - receiveProcessorComms.clear(); - sendProcessorComms.clear(); - } - - /** - * Assert that all required sends and receives have occurred. - */ - void ExpectationsAllCompleted() - { - for (std::map >::iterator receipts_from_core = - requiredReceipts.begin(); receipts_from_core != requiredReceipts.end(); - receipts_from_core++) - { - CPPUNIT_ASSERT(0 == receipts_from_core->second.size()); - } - for (std::map >::iterator sends_to_core = - requiredSends.begin(); sends_to_core != requiredSends.end(); sends_to_core++) - { - CPPUNIT_ASSERT(0 == sends_to_core->second.size()); - } - } - private: - - std::map > requiredReceipts; - std::map > requiredSends; - } - ; - - } - } -} -#endif diff --git a/Code/unittests/reporting/ReporterTests.h b/Code/unittests/reporting/ReporterTests.h index 19d14f46b..ef9392462 100644 --- a/Code/unittests/reporting/ReporterTests.h +++ b/Code/unittests/reporting/ReporterTests.h @@ -40,14 +40,13 @@ namespace hemelb realTimers = new reporting::Timers(Comms()); buildInfo = new reporting::BuildInfo(); state = new hemelb::lb::SimulationState(0.0001, 1000); - net = new net::Net(Comms()); latticeData = FourCubeLatticeData::Create(Comms(), 6, 5); // The 5 here is to match the topology size in the MPICommsMock lbtests::LbTestsHelper::InitialiseAnisotropicTestData(latticeData); latticeData->SwapOldAndNew(); //Needed since InitialiseAnisotropicTestData only initialises FOld cache = new lb::MacroscopicPropertyCache(*state, *latticeData); cache->densityCache.SetRefreshFlag(); lbtests::LbTestsHelper::UpdatePropertyCache(*latticeData, *cache, *state); - incompChecker = new IncompressibilityCheckerMock(latticeData, net, state, *cache, *realTimers, 10.0); + incompChecker = new IncompressibilityCheckerMock(latticeData, Comms(), state, *cache, *realTimers, 10.0); reporter = new Reporter("mock_path", "exampleinputfile"); reporter->AddReportable(incompChecker); reporter->AddReportable(mockTimers); @@ -63,7 +62,6 @@ namespace hemelb delete realTimers; delete cache; delete incompChecker; - delete net; delete buildInfo; helpers::HasCommsTestFixture::tearDown(); } @@ -148,7 +146,6 @@ namespace hemelb lb::MacroscopicPropertyCache* cache; IncompressibilityCheckerMock *incompChecker; - net::Net *net; hemelb::unittests::FourCubeLatticeData* latticeData; reporting::BuildInfo *buildInfo; }; From e732501d7fc16c7655d52106c773aa4497a3c3fe Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 28 Oct 2016 12:46:26 +0100 Subject: [PATCH 80/99] fix final unittests --- .../NeighbouringDataManagerTests.h | 8 -- Code/unittests/net/phased/StepManagerTests.h | 98 ++++++++++--------- 2 files changed, 51 insertions(+), 55 deletions(-) diff --git a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h index fbfda6e7d..50b42ce75 100644 --- a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h +++ b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h @@ -48,14 +48,6 @@ namespace hemelb manager = new NeighbouringDataManager(*latDat, *data, commQ); } - void UseRealCommunicator() - { - // delete manager; - - // communicator = comm::MpiEnvironment::World(); - // commQ = comm::Async::New(communicator); - // manager = new NeighbouringDataManager(*latDat, *data, commQ); - } void tearDown() { diff --git a/Code/unittests/net/phased/StepManagerTests.h b/Code/unittests/net/phased/StepManagerTests.h index 754852dbb..53823b249 100644 --- a/Code/unittests/net/phased/StepManagerTests.h +++ b/Code/unittests/net/phased/StepManagerTests.h @@ -7,10 +7,10 @@ #ifndef HEMELB_UNITTESTS_NET_PHASED_STEPMANAGERTESTS_H #define HEMELB_UNITTESTS_NET_PHASED_STEPMANAGERTESTS_H #include "net/phased/StepManager.h" -#include "unittests/helpers/MockNetHelper.h" +#include "unittests/helpers/MockCommsHelper.h" #include "unittests/net/phased/MockConcern.h" #include "unittests/net/phased/MockIteratedAction.h" -#include "net/phased/NetConcern.h" +#include "comm/AsyncConcern.h" #include namespace hemelb @@ -23,7 +23,7 @@ namespace hemelb { using namespace hemelb::net::phased; using namespace hemelb::unittests::helpers; - class StepManagerTests : public CppUnit::TestFixture, public MockNetHelper + class StepManagerTests : public CppUnit::TestFixture, public MockCommsHelper { CPPUNIT_TEST_SUITE (StepManagerTests); CPPUNIT_TEST (TestConstruct); @@ -50,7 +50,7 @@ namespace hemelb public: StepManagerTests() : - MockNetHelper(),stepManager(NULL), action(NULL), concern(NULL), netConcern(NULL), action2(NULL), concern2(NULL) + MockCommsHelper(),stepManager(NULL), action(NULL), concern(NULL), netConcern(NULL), action2(NULL), concern2(NULL) { } @@ -62,7 +62,7 @@ namespace hemelb void tearDown() { - MockNetHelper::tearDown(); + MockCommsHelper::tearDown(); delete stepManager; delete netConcern; delete action; @@ -139,17 +139,17 @@ namespace hemelb int payload0Expectation = 5; int payload1Expectation = 1; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); - + MockComms()->RequireSend(payload1Expectation, 1, 11); + MockComms()->RequireRecv(payload0Expectation, 1, 11); + stepManager->RegisterCommsSteps(*netConcern, 0); + commQ->Isend(payload1, 1, 11); + commQ->Irecv(payload0, 1, 11); stepManager->CallActionsForPhase(0); CPPUNIT_ASSERT_EQUAL(payload0, 5); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); } void TestCallNonCommsActions() @@ -203,11 +203,12 @@ namespace hemelb int payload0Expectation = 5; int payload1Expectation = 1; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); - + MockComms()->RequireSend(payload1Expectation, 1, 1); + MockComms()->RequireRecv(payload0Expectation, 1, 1); + + commQ->Isend(payload1, 1, 1); + commQ->Irecv(payload0, 1, 1); + action = new MockIteratedAction("mockOne"); concern = new MockConcern("mockTwo"); @@ -228,7 +229,7 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, "), action->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 5); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); } void TestCallAllActionsOnePhase() @@ -240,11 +241,12 @@ namespace hemelb int payload0Expectation = 5; int payload1Expectation = 1; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); + MockComms()->RequireSend(payload1Expectation, 1, 0); + MockComms()->RequireRecv(payload0Expectation, 1, 0); + commQ->Isend(payload1, 1); + commQ->Irecv(payload0, 1); + action = new MockIteratedAction("mockOne"); concern = new MockConcern("mockTwo"); @@ -267,7 +269,7 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, EndIteration, "), action->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 5); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); } void TestCallAllActionsManyPhases() @@ -279,10 +281,11 @@ namespace hemelb int payload0Expectation = 5; int payload1Expectation = 1; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); + MockComms()->RequireSend(payload1Expectation, 1, 0); + MockComms()->RequireRecv(payload0Expectation, 1, 0); + + commQ->Isend(payload1, 1); + commQ->Irecv(payload0, 1); action = new MockIteratedAction("mockOne"); concern = new MockConcern("mockTwo"); @@ -325,7 +328,7 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, EndIteration, "), action2->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 5); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); } void TestCallAllActionsPhaseByPhase() @@ -368,16 +371,17 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string(""), action->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(std::string(""), action2->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 0); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); //------------------Phase 0 --------------------------------------------- int payload0Expectation = 5; int payload1Expectation = 1; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); + MockComms()->RequireSend(payload1Expectation, 1, 0); + MockComms()->RequireRecv(payload0Expectation, 1, 0); + + commQ->Isend(payload1, 1); + commQ->Irecv(payload0, 1); stepManager->CallActionsForPhase(0); @@ -392,17 +396,17 @@ namespace hemelb action->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(std::string(""), action2->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 5); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); // ------------------- Phase 1 ------------------------------------------ payload0Expectation = 13; payload1Expectation = 4; payload1 = 4; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); + MockComms()->RequireSend(payload1Expectation, 1, 0); + MockComms()->RequireRecv(payload0Expectation, 1, 0); + commQ->Isend(payload1, 1); + commQ->Irecv(payload0, 1); stepManager->CallActionsForPhase(1); @@ -413,17 +417,17 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, "), action2->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 13); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); // ------------------------- Phase 2 ----------------------------------------- payload0Expectation = 77; payload1Expectation = 16; payload1 = 16; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); + MockComms()->RequireSend(payload1Expectation, 1, 0); + MockComms()->RequireRecv(payload0Expectation, 1, 0); + commQ->Isend(payload1, 1); + commQ->Irecv(payload0, 1); stepManager->CallActionsForPhase(2); @@ -436,7 +440,7 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, "), action2->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 77); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); // -------------------------- EndAll ----------------------------------------------- @@ -451,13 +455,13 @@ namespace hemelb action->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, EndIteration, "), action2->CallsSoFar()); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); } void SetupMocks(const proc_t core_count, const proc_t current_core) { - MockNetHelper::setUp(core_count,current_core); - netConcern = new NetConcern(*netMock); + MockCommsHelper::setUp(core_count,current_core); + netConcern = new AsyncConcern(commQ); } private: @@ -465,7 +469,7 @@ namespace hemelb StepManager *stepManager; MockIteratedAction *action; MockConcern *concern; - NetConcern *netConcern; + comm::AsyncConcern *netConcern; MockIteratedAction *action2; MockConcern *concern2; }; From 12a5e96b732da11f8b08a8ffb6dd0dd0a4f89524 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 31 Oct 2016 14:41:49 +0000 Subject: [PATCH 81/99] minor tweaks to the communicator for sanity check and speed of Rank()/Size() --- Code/comm/Communicator.h | 4 ++++ Code/comm/Communicator.hpp | 15 +++++++++++++++ Code/comm/MpiCommunicator.cc | 13 ++++++------- Code/comm/MpiCommunicator.h | 2 ++ Code/unittests/helpers/MockCommunicator.h | 5 ----- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/Code/comm/Communicator.h b/Code/comm/Communicator.h index 249a977bf..febf88c65 100644 --- a/Code/comm/Communicator.h +++ b/Code/comm/Communicator.h @@ -207,6 +207,10 @@ namespace hemelb // bool operator==(const Communicator& comm1, const Communicator& comm2); // bool operator!=(const Communicator& comm1, const Communicator& comm2); + template + struct is_std_vector : public std::false_type {}; + template + struct is_std_vector > : public std::true_type {}; } } diff --git a/Code/comm/Communicator.hpp b/Code/comm/Communicator.hpp index 21bd1c36f..47617181d 100644 --- a/Code/comm/Communicator.hpp +++ b/Code/comm/Communicator.hpp @@ -217,6 +217,11 @@ namespace hemelb template std::shared_ptr Communicator::Isend(const T* valPtr, int count, int dest, int tag) const { + static_assert(!std::is_pointer::value, + "You are trying to communicate a pointer type"); + static_assert(!is_std_vector::value, + "You are trying to communicate a std::vector type"); + return IsendImpl(valPtr, count, MpiDataType(), dest, tag); } template @@ -234,6 +239,11 @@ namespace hemelb template std::shared_ptr Communicator::Issend(const T* valPtr, int count, int dest, int tag) const { + static_assert(!std::is_pointer::value, + "You are trying to communicate a pointer type"); + static_assert(!is_std_vector::value, + "You are trying to communicate a std::vector type"); + return IssendImpl(valPtr, count, MpiDataType(), dest, tag); } template @@ -251,6 +261,11 @@ namespace hemelb template std::shared_ptr Communicator::Irecv(T* valPtr, int count, int source, int tag) const { + static_assert(!std::is_pointer::value, + "You are trying to communicate a pointer type"); + static_assert(!is_std_vector::value, + "You are trying to communicate a std::vector type"); + return IrecvImpl(valPtr, count, MpiDataType(), source, tag); } template diff --git a/Code/comm/MpiCommunicator.cc b/Code/comm/MpiCommunicator.cc index 1e6455672..3b299e82e 100644 --- a/Code/comm/MpiCommunicator.cc +++ b/Code/comm/MpiCommunicator.cc @@ -26,15 +26,15 @@ namespace hemelb } } - MpiCommunicator::MpiCommunicator() : commPtr() + MpiCommunicator::MpiCommunicator() : commPtr(), rank(0), size(0) { } - MpiCommunicator::MpiCommunicator(MPI_Comm communicator, bool owner) : commPtr() + MpiCommunicator::MpiCommunicator(MPI_Comm communicator, bool owner) : commPtr(), rank(0), size(0) { if (communicator == MPI_COMM_NULL) return; - + if (owner) { commPtr.reset(new MPI_Comm(communicator), Deleter); @@ -43,6 +43,9 @@ namespace hemelb { commPtr.reset(new MPI_Comm(communicator)); } + + HEMELB_MPI_CALL(MPI_Comm_rank, (*commPtr, &rank)); + HEMELB_MPI_CALL(MPI_Comm_size, (*commPtr, &size)); } MpiCommunicator::operator MPI_Comm() const @@ -55,15 +58,11 @@ namespace hemelb } int MpiCommunicator::Rank() const { - int rank; - HEMELB_MPI_CALL(MPI_Comm_rank, (*commPtr, &rank)); return rank; } int MpiCommunicator::Size() const { - int size; - HEMELB_MPI_CALL(MPI_Comm_size, (*commPtr, &size)); return size; } diff --git a/Code/comm/MpiCommunicator.h b/Code/comm/MpiCommunicator.h index 6c1cdcbce..9aa06b6c1 100644 --- a/Code/comm/MpiCommunicator.h +++ b/Code/comm/MpiCommunicator.h @@ -105,6 +105,8 @@ namespace hemelb int source, int tag) const; std::shared_ptr commPtr; + int rank; + int size; }; // bool operator==(const Communicator& comm1, const Communicator& comm2); diff --git a/Code/unittests/helpers/MockCommunicator.h b/Code/unittests/helpers/MockCommunicator.h index b7f15a85b..a1a3bbb65 100644 --- a/Code/unittests/helpers/MockCommunicator.h +++ b/Code/unittests/helpers/MockCommunicator.h @@ -111,11 +111,6 @@ namespace hemelb // int mDest; // int mTag; // }; - - template - struct is_std_vector : public std::false_type {}; - template - struct is_std_vector > : public std::true_type {}; class MockCommunicator : public comm::Communicator { From e2265ebf3a6767242167a3561ce12b67f04b940d Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 1 Nov 2016 11:32:26 +0000 Subject: [PATCH 82/99] start on replacing the net::phased stuff with new timestep ns --- Code/CMakeLists.txt | 4 +- Code/lb/EntropyTester.h | 6 +-- Code/lb/IncompressibilityChecker.cc | 2 +- Code/lb/IncompressibilityChecker.h | 23 ++++---- Code/lb/StabilityTester.h | 42 ++++++++------- Code/lb/StabilityTester.hpp | 13 ++++- Code/net/CollectiveAction.cc | 81 ----------------------------- Code/net/CollectiveAction.h | 64 ----------------------- Code/timestep/Actor.cc | 23 ++++++++ Code/timestep/Actor.h | 33 ++++++++++++ Code/timestep/CMakeLists.txt | 7 +++ Code/timestep/CollectiveActor.cc | 43 +++++++++++++++ Code/timestep/CollectiveActor.h | 58 +++++++++++++++++++++ Code/timestep/TimeStepManager.cc | 50 ++++++++++++++++++ Code/timestep/TimeStepManager.h | 75 ++++++++++++++++++++++++++ 15 files changed, 342 insertions(+), 182 deletions(-) delete mode 100644 Code/net/CollectiveAction.cc delete mode 100644 Code/net/CollectiveAction.h create mode 100644 Code/timestep/Actor.cc create mode 100644 Code/timestep/Actor.h create mode 100644 Code/timestep/CMakeLists.txt create mode 100644 Code/timestep/CollectiveActor.cc create mode 100644 Code/timestep/CollectiveActor.h create mode 100644 Code/timestep/TimeStepManager.cc create mode 100644 Code/timestep/TimeStepManager.h diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index ada5fb5ae..a88f3d2be 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -181,7 +181,7 @@ set(package_subdirs reporting lb geometry - net + timestep debug util io @@ -227,7 +227,7 @@ if (HEMELB_BUILD_MULTISCALE) reporting lb geometry - net + timestep debug util io diff --git a/Code/lb/EntropyTester.h b/Code/lb/EntropyTester.h index 2d5fe1e33..8a35a1f3f 100644 --- a/Code/lb/EntropyTester.h +++ b/Code/lb/EntropyTester.h @@ -7,7 +7,7 @@ #ifndef HEMELB_LB_ENTROPYTESTER_H #define HEMELB_LB_ENTROPYTESTER_H -#include "net/CollectiveAction.h" +#include "timestep/CollectiveActor.h" #include "geometry/LatticeData.h" #include "lb/HFunction.h" #include "log/Logger.h" @@ -17,13 +17,13 @@ namespace hemelb namespace lb { template - class EntropyTester : public net::CollectiveAction + class EntropyTester : public timestep::CollectiveActor { public: EntropyTester(int* collisionTypes, unsigned int typesTested, const geometry::LatticeData * iLatDat, comm::Communicator::ConstPtr comm, SimulationState* simState, reporting::Timers& timings) : - net::CollectiveAction(comm, timings[reporting::Timers::mpiWait]), + timestep::CollectiveActor(comm, timings[reporting::Timers::mpiWait]), mLatDat(iLatDat), mHPreCollision(mLatDat->GetLocalFluidSiteCount()), workTimer(timings[reporting::Timers::monitoring]) { diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index ee5eac9c8..06f2ce138 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -57,7 +57,7 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, reporting::Timers& timings, distribn_t maximumRelativeDensityDifferenceAllowed) : - net::CollectiveAction(comms, timings[reporting::Timers::mpiWait]), + timestep::CollectiveActor(comms, timings[reporting::Timers::mpiWait]), mLatDat(latticeData), propertyCache(propertyCache), mSimState(simState), maximumRelativeDensityDifferenceAllowed(maximumRelativeDensityDifferenceAllowed), workTimer(timings[reporting::Timers::monitoring]) diff --git a/Code/lb/IncompressibilityChecker.h b/Code/lb/IncompressibilityChecker.h index 17f15765b..861290909 100644 --- a/Code/lb/IncompressibilityChecker.h +++ b/Code/lb/IncompressibilityChecker.h @@ -7,7 +7,7 @@ #ifndef HEMELB_LB_INCOMPRESSIBILITYCHECKER_H #define HEMELB_LB_INCOMPRESSIBILITYCHECKER_H #include "comm/Communicator.h" -#include "net/CollectiveAction.h" +#include "timestep/CollectiveActor.h" #include "geometry/LatticeData.h" #include "lb/MacroscopicPropertyCache.h" #include "reporting/Reportable.h" @@ -29,7 +29,7 @@ namespace hemelb double maxVel; }; - class IncompressibilityChecker : public net::CollectiveAction, + class IncompressibilityChecker : public timestep::CollectiveActor, public reporting::Reportable { public: @@ -107,15 +107,16 @@ namespace hemelb static MPI_Datatype GetDensityMpiDatatype(); protected: - /** - * Compute the local state. - */ - void PreSend(void); - - /** - * Initiate the collective. - */ - void Send(void); + + inline virtual void BeginAll() {} + inline virtual void Begin() {} + // Compute the local state + virtual void PreSend(); + // Initiate the collective + virtual void Send(); + inline virtual void PreWait() {} + inline virtual void End() {} + inline virtual void EndAll() {} private: static void UpdateDensityEtc(const DensityEtc& in, DensityEtc& inout); diff --git a/Code/lb/StabilityTester.h b/Code/lb/StabilityTester.h index 73bb65b70..ab560a5bd 100644 --- a/Code/lb/StabilityTester.h +++ b/Code/lb/StabilityTester.h @@ -7,7 +7,7 @@ #ifndef HEMELB_LB_STABILITYTESTER_H #define HEMELB_LB_STABILITYTESTER_H -#include "net/CollectiveAction.h" +#include "timestep/CollectiveActor.h" #include "geometry/LatticeData.h" namespace hemelb @@ -18,9 +18,9 @@ namespace hemelb * Class to repeatedly assess the stability of the simulation, using non-blocking collective */ template - class StabilityTester : public net::CollectiveAction + class StabilityTester : public timestep::CollectiveActor { - public: + public: StabilityTester(const geometry::LatticeData * iLatDat, comm::Communicator::ConstPtr comms, SimulationState* simState, reporting::Timers& timings, bool checkForConvergence, double relativeTolerance); @@ -32,16 +32,27 @@ namespace hemelb void Reset(); protected: - /** - * Compute the local stability/convergence state. - */ - void PreSend(void); - - /** - * Initiate the collective. - */ - virtual void Send(void); - + + virtual void BeginAll() {} + + virtual void Begin() {} + + // Compute the local stability/convergence state + virtual void PreSend(); + + // Initiate the collective + virtual void Send(); + + virtual void PreWait() {} + + // Wait is defined in CollectiveActor + // virtual void Wait(); + + // Apply the stability value sent by the root node to the simulation logic. + virtual void End(); + + virtual void EndAll() {} + /** * Computes the relative difference between the densities at the beginning and end of a * timestep, i.e. |(rho_new - rho_old) / (rho_old - rho_0)|. @@ -53,11 +64,6 @@ namespace hemelb double ComputeRelativeDifference(const distribn_t* fNew, const distribn_t* fOld) const; - /** - * Apply the stability value sent by the root node to the simulation logic. - */ - void PostReceive(void); - private: const geometry::LatticeData* mLatDat; diff --git a/Code/lb/StabilityTester.hpp b/Code/lb/StabilityTester.hpp index 987072238..f6096eaec 100644 --- a/Code/lb/StabilityTester.hpp +++ b/Code/lb/StabilityTester.hpp @@ -24,7 +24,7 @@ namespace hemelb reporting::Timers& timings, bool checkForConvergence, double relativeTolerance) : - CollectiveAction(comms, timings[reporting::Timers::monitoring]), + CollectiveActor(comms, timings[reporting::Timers::monitoring]), mLatDat(iLatDat), mSimState(simState), checkForConvergence(checkForConvergence), relativeTolerance(relativeTolerance), workTimer(timings[reporting::Timers::monitoring]) { @@ -48,6 +48,9 @@ namespace hemelb template void StabilityTester::PreSend(void) { + if (isCollectiveRunning) + return; + workTimer.Start(); bool unconvergedSitePresent = false; bool checkConvThisTimeStep = checkForConvergence; @@ -103,6 +106,9 @@ namespace hemelb template void StabilityTester::Send(void) { + if (isCollectiveRunning) + return; + // Begin collective. collectiveReq = collectiveComm->Iallreduce(localStability, MPI_MIN, globalStability); } @@ -143,8 +149,11 @@ namespace hemelb * Apply the stability value sent by the root node to the simulation logic. */ template - void StabilityTester::PostReceive() + void StabilityTester::End() { + if (isCollectiveRunning) + return; + mSimState->SetStability((Stability) globalStability); } } diff --git a/Code/net/CollectiveAction.cc b/Code/net/CollectiveAction.cc deleted file mode 100644 index 13d3044af..000000000 --- a/Code/net/CollectiveAction.cc +++ /dev/null @@ -1,81 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#include "net/CollectiveAction.h" -#include "net/phased/steps.h" - -namespace hemelb -{ - namespace net - { - CollectiveAction::CollectiveAction(comm::Communicator::ConstPtr comm, reporting::Timer& wTimer) : - isCollectiveRunning(false), mustWait(false), collectiveComm(comm->Duplicate()), waitTimer(wTimer), collectiveReq() - { - } - - bool CollectiveAction::CallAction(int action) - { - if (isCollectiveRunning) - { - switch (static_cast(action)) - { - case phased::steps::Wait: - Wait(); - return true; - default: - return false; - } - } - else - { - switch (static_cast(action)) - { - case phased::steps::BeginPhase: - RequestComms(); - return true; - case phased::steps::PreSend: - PreSend(); - return true; - case phased::steps::Send: - Send(); - isCollectiveRunning = true; - return true; - case phased::steps::EndPhase: - PostReceive(); - return true; - case phased::steps::EndAll: - EndIteration(); - return true; - default: - return false; - } - } - } - - /** - * Wait on the collectives to finish. - */ - void CollectiveAction::Wait(void) - { - waitTimer.Start(); - if (mustWait) { - collectiveReq->Wait(); - mustWait = false; - isCollectiveRunning = false; - } - else - { - bool done = collectiveReq->Test(); - if (done) - isCollectiveRunning = false; - } - waitTimer.Stop(); - } - } -} diff --git a/Code/net/CollectiveAction.h b/Code/net/CollectiveAction.h deleted file mode 100644 index e36b81787..000000000 --- a/Code/net/CollectiveAction.h +++ /dev/null @@ -1,64 +0,0 @@ -// -// Copyright (C) University College London, 2007-2012, all rights reserved. -// -// This file is part of HemeLB and is CONFIDENTIAL. You may not work -// with, install, use, duplicate, modify, redistribute or share this -// file, or any part thereof, other than as allowed by any agreement -// specifically made by you with University College London. -// - -#ifndef HEMELB_NET_COLLECTIVEACTION_H -#define HEMELB_NET_COLLECTIVEACTION_H - -#include "net/IteratedAction.h" -#include "reporting/Timers.h" -#include "comm/Request.h" - -namespace hemelb -{ - namespace net - { - class CollectiveAction : public IteratedAction - { - public: - bool CallAction(int action); - - inline void MustFinishThisTimeStep() - { - - mustWait = true; - } - - protected: - CollectiveAction(comm::Communicator::ConstPtr comm, reporting::Timer& waitTimer); - /** - * Initiate the collective. - */ - virtual void Send(void) = 0; - - /** - * Progress the communication - */ - virtual void Wait(void); - - bool isCollectiveRunning; - bool mustWait; - - /** - * Private communicator for non-blocking collectives. - */ - comm::Communicator::Ptr collectiveComm; - /** - * Timings for the wait etc. - */ - reporting::Timer& waitTimer; - /** - * Request object for the collective - */ - comm::Request::Ptr collectiveReq; - }; - - } -} - -#endif /* HEMELB_NET_COLLECTIVEACTION_H */ diff --git a/Code/timestep/Actor.cc b/Code/timestep/Actor.cc new file mode 100644 index 000000000..d97d8c883 --- /dev/null +++ b/Code/timestep/Actor.cc @@ -0,0 +1,23 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#include "timestep/Actor.h" + +namespace hemelb +{ + namespace timestep + { + void Actor::BeginAll() {} + void Actor::Begin() {} + void Actor::Receive() {} + void Actor::PreSend() {} + void Actor::Send() {} + void Actor::PreWait() {} + void Actor::Wait() {} + void Actor::End() {} + void Actor::EndAll() {} + } +} + diff --git a/Code/timestep/Actor.h b/Code/timestep/Actor.h new file mode 100644 index 000000000..c87e981f3 --- /dev/null +++ b/Code/timestep/Actor.h @@ -0,0 +1,33 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_TIMESTEP_ACTOR_H +#define HEMELB_TIMESTEP_ACTOR_H + +namespace hemelb +{ + namespace timestep + { + + class Actor + { + protected: + friend class TimeStepManager; + + virtual void BeginAll() = 0; + virtual void Begin() = 0; + virtual void Receive() = 0; + virtual void PreSend() = 0; + virtual void Send() = 0; + virtual void PreWait() = 0; + virtual void Wait() = 0; + virtual void End() = 0; + virtual void EndAll() = 0; + + }; + } +} + +#endif diff --git a/Code/timestep/CMakeLists.txt b/Code/timestep/CMakeLists.txt new file mode 100644 index 000000000..60f8686f5 --- /dev/null +++ b/Code/timestep/CMakeLists.txt @@ -0,0 +1,7 @@ +# This file is part of HemeLB and is Copyright (C) +# the HemeLB team and/or their institutions, as detailed in the +# file AUTHORS. This software is provided under the terms of the +# license in the file LICENSE. + +add_library(hemelb_timestep + TimeStepManager.cc Actor.cc CollectiveActor.cc) diff --git a/Code/timestep/CollectiveActor.cc b/Code/timestep/CollectiveActor.cc new file mode 100644 index 000000000..5aef7777c --- /dev/null +++ b/Code/timestep/CollectiveActor.cc @@ -0,0 +1,43 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#include "timestep/CollectiveActor.h" +#include "Exception.h" + +namespace hemelb +{ + namespace timestep + { + CollectiveActor::CollectiveActor(comm::Communicator::ConstPtr comm, reporting::Timer& wTimer) : + isCollectiveRunning(false), mustWait(false), collectiveComm(comm->Duplicate()), waitTimer(wTimer), collectiveReq() + { + } + + /** + * Wait on the collectives to finish. + */ + void CollectiveActor::Wait(void) + { + if (!isCollectiveRunning) + throw Exception() << "Can only wait if the collective is running"; + + waitTimer.Start(); + if (mustWait) { + collectiveReq->Wait(); + mustWait = false; + isCollectiveRunning = false; + } else { + bool done = collectiveReq->Test(); + if (done) + isCollectiveRunning = false; + } + waitTimer.Stop(); + } + } +} diff --git a/Code/timestep/CollectiveActor.h b/Code/timestep/CollectiveActor.h new file mode 100644 index 000000000..59ed393dd --- /dev/null +++ b/Code/timestep/CollectiveActor.h @@ -0,0 +1,58 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_TIMESTEP_COLLECTIVEACTOR_H +#define HEMELB_TIMESTEP_COLLECTIVEACTOR_H + +#include "timestep/Actor.h" +#include "reporting/Timers.h" +#include "comm/Request.h" + +namespace hemelb +{ + namespace timestep + { + class CollectiveActor : public Actor + { + public: + inline void MustFinishThisTimeStep() + { + + mustWait = true; + } + + protected: + CollectiveActor(comm::Communicator::ConstPtr comm, reporting::Timer& waitTimer); + + // This should be a no-op for a collective + virtual void Receive(); + + // Subclass must supply the send method with the collective + // virtual void Send(); + + // Progress the communication + virtual void Wait(void) final; + + bool isCollectiveRunning; + bool mustWait; + + /** + * Private communicator for non-blocking collectives. + */ + comm::Communicator::Ptr collectiveComm; + /** + * Timings for the wait etc. + */ + reporting::Timer& waitTimer; + /** + * Request object for the collective + */ + comm::Request::Ptr collectiveReq; + }; + + } +} + +#endif /* HEMELB_NET_COLLECTIVEACTION_H */ diff --git a/Code/timestep/TimeStepManager.cc b/Code/timestep/TimeStepManager.cc new file mode 100644 index 000000000..34e77b966 --- /dev/null +++ b/Code/timestep/TimeStepManager.cc @@ -0,0 +1,50 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#include "timestep/TimeStepManager.h" +#include "timestep/Actor.h" + +namespace hemelb +{ + namespace timestep + { + TimeStepManager::TimeStepManager(unsigned nPhases) : + mPhases(nPhases) + { + } + // Register the actor for the phase + void TimeStepManager::AddToPhase(unsigned phase, Actor* actor) { + mPhases[phase].actors.push_back(actor); + } + + void TimeStepManager::DoStep() + { + // BeginAll + for (auto& ph: mPhases) + for (auto ap: ph.actors) + ap->BeginAll(); + + for (auto& ph: mPhases) + { + for (auto ap: ph.actors) + { + ap->Begin(); + ap->Receive(); + ap->PreSend(); + ap->Send(); + ap->PreWait(); + ap->Wait(); + ap->End(); + } + } + + for (auto& ph: mPhases) + for (auto ap: ph.actors) + ap->EndAll(); + + } + + } +} diff --git a/Code/timestep/TimeStepManager.h b/Code/timestep/TimeStepManager.h new file mode 100644 index 000000000..b9e5c4a2f --- /dev/null +++ b/Code/timestep/TimeStepManager.h @@ -0,0 +1,75 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_TIMESTEP_TIMESTEPMANAGER_H +#define HEMELB_TIMESTEP_TIMESTEPMANAGER_H + +#include + +namespace hemelb +{ + namespace timestep + { + + // A TimeStep is made up of one or more Phases of computation and + // communication. + + // A Phase has multiple Actions which follow the same pattern: + // + // * Begin - do any necessary computation/memory allocation before + // the receives begin + // + // * Receive - start non-blocking receive operations + // + // * PreSend - do any necessary computation before data can be sent + // + // * Send - initiate non-blocking sends (or collectives) + // + // * PreWait - do any available computation that does not depend + // on data to be received + // + // * Wait - wait for communication to complete + // + // * End - do computation that depends on received data + + // In addition there are two special Actions, BeginAll and EndAll + // that will be executed before and after the main phases. + + // Thus for e.g. 2 phases, the ordering of Actions will be: + // + // BeginAll0, BeginAll1 + // Begin0, Recv0, PreSend0, Send0, PreWait0, Wait0, End0, + // Begin1, Recv1, PreSend1, Send1, PreWait1, Wait1, End1, + // EndAll0, EndAll1 + + // It is important to note that each Action can have multiple + // Actors performing their tasks + class Actor; + + // This class will arrange the calling of the different components + // of HemeLB through a single timestep. + class TimeStepManager + { + public: + TimeStepManager(unsigned nPhases); + // Register the actor for the phase + void AddToPhase(unsigned phase, Actor* actor); + // Perform a timestep + void DoStep(); + private: + //friend class Actor; + + struct Phase + { + std::vector actors; + }; + + std::vector mPhases; + }; + } +} + +#endif // HEMELB_TIMESTEP_TIMESTEPMANAGER_H From 1e00030c81d268d361126997653a97098a5b9c80 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 7 Feb 2017 13:55:04 +0000 Subject: [PATCH 83/99] remove most traces of net; new stuff only part-functioning --- Code/SimulationMaster.cc | 42 +++--- Code/SimulationMaster.h | 6 +- Code/colloids/ColloidController.h | 27 +++- Code/comm/AsyncConcern.h | 38 +++--- Code/comm/Communicator.h | 5 - Code/extraction/PropertyActor.cc | 2 +- Code/extraction/PropertyActor.h | 29 ++++- Code/geometry/LatticeData.cc | 13 +- Code/geometry/LatticeData.h | 4 +- .../neighbouring/NeighbouringDataManager.cc | 9 +- .../neighbouring/NeighbouringDataManager.h | 24 +++- Code/lb/CMakeLists.txt | 2 +- Code/lb/IncompressibilityChecker.cc | 4 + Code/lb/StabilityTester.hpp | 1 + Code/lb/iolets/BoundaryCommunicator.h | 12 +- Code/lb/iolets/BoundaryValues.cc | 121 ++++++++++-------- Code/lb/iolets/BoundaryValues.h | 54 ++++++-- Code/lb/iolets/InOutLet.cc | 5 - Code/lb/iolets/InOutLet.h | 36 ++---- Code/lb/iolets/InOutLetMultiscale.cc | 53 +++----- Code/lb/iolets/InOutLetMultiscale.h | 8 +- Code/lb/lb.h | 29 +++-- Code/lb/lb.hpp | 53 ++++---- Code/multiscale/MultiscaleSimulationMaster.h | 21 ++- Code/timestep/Actor.h | 4 +- Code/timestep/CollectiveActor.cc | 4 + .../NeighbouringDataManagerTests.h | 8 +- Code/unittests/helpers/HasCommsTestFixture.h | 7 + .../lbtests/IncompressibilityCheckerTests.h | 14 +- Code/unittests/lbtests/StreamerTests.h | 6 +- .../lbtests/VirtualSiteIoletStreamerTests.h | 4 +- Code/unittests/lbtests/iolets/BoundaryTests.h | 14 +- Code/unittests/main.cc | 2 +- 33 files changed, 394 insertions(+), 267 deletions(-) diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index f5f111e17..3f1a3cf25 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -17,7 +17,6 @@ #include "lb/HFunction.h" #include "io/xml/XmlAbstractionLayer.h" #include "colloids/ColloidController.h" -#include "net/BuildInfo.h" #include "colloids/BodyForces.h" #include "colloids/BoundaryConditions.h" #include "comm/MpiEnvironment.h" @@ -45,7 +44,7 @@ SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, propertyExtractor = NULL; simulationState = NULL; stepManager = NULL; - netConcern = NULL; + asyncCommsManager = NULL; neighbouringDataManager = NULL; fileManager = new hemelb::io::PathManager(options, IsCurrentProcTheIOProc(), GetProcessorCount()); @@ -98,7 +97,7 @@ SimulationMaster::~SimulationMaster() delete reporter; } delete stepManager; - delete netConcern; + delete asyncCommsManager; } /** @@ -213,14 +212,14 @@ void SimulationMaster::Initialise() latticeData, simConfig->GetInlets(), simulationState, - ioComms, + asyncCommQ, *unitConverter); outletValues = new hemelb::lb::iolets::BoundaryValues(hemelb::geometry::OUTLET_TYPE, latticeData, simConfig->GetOutlets(), simulationState, - ioComms, + asyncCommQ, *unitConverter); latticeBoltzmannModel->Initialise(inletValues, outletValues, unitConverter); @@ -248,38 +247,37 @@ void SimulationMaster::Initialise() timings, ioComms); } - stepManager = new hemelb::net::phased::StepManager(2, - &timings, - hemelb::net::separate_communications); - netConcern = new hemelb::comm::AsyncConcern(asyncCommQ); - stepManager->RegisterIteratedActorSteps(*neighbouringDataManager, 0); + const int nPhase = 2; + stepManager = new hemelb::timestep::TimeStepManager(nPhase); + asyncCommsManager = new hemelb::comm::AsyncConcern(asyncCommQ); + for (auto i=0; iAddToPhase(i, asyncCommsManager); + + stepManager->AddToPhase(0, neighbouringDataManager); if (colloidController != NULL) { - stepManager->RegisterIteratedActorSteps(*colloidController, 1); + stepManager->AddToPhase(1, colloidController); } - stepManager->RegisterIteratedActorSteps(*latticeBoltzmannModel, 1); + stepManager->AddToPhase(1, latticeBoltzmannModel); - stepManager->RegisterIteratedActorSteps(*inletValues, 1); - stepManager->RegisterIteratedActorSteps(*outletValues, 1); - stepManager->RegisterIteratedActorSteps(*stabilityTester, 1); - stepManager->RegisterCommsSteps(*stabilityTester, 1); + stepManager->AddToPhase(1, inletValues); + stepManager->AddToPhase(1, outletValues); + stepManager->AddToPhase(1, stabilityTester); if (entropyTester != NULL) { - stepManager->RegisterIteratedActorSteps(*entropyTester, 1); + stepManager->AddToPhase(1, entropyTester); } if (monitoringConfig->doIncompressibilityCheck) { - stepManager->RegisterIteratedActorSteps(*incompressibilityChecker, 1); - stepManager->RegisterCommsSteps(*incompressibilityChecker, 1); + stepManager->AddToPhase(1, incompressibilityChecker); } if (propertyExtractor != NULL) { - stepManager->RegisterIteratedActorSteps(*propertyExtractor, 1); + stepManager->AddToPhase(1, propertyExtractor); } - stepManager->RegisterCommsForAllPhases(*netConcern); } unsigned int SimulationMaster::OutputPeriod(unsigned int frequency) @@ -294,7 +292,7 @@ unsigned int SimulationMaster::OutputPeriod(unsigned int frequency) void SimulationMaster::HandleActors() { - stepManager->CallActions(); + stepManager->DoStep(); } void SimulationMaster::OnUnstableSimulation() diff --git a/Code/SimulationMaster.h b/Code/SimulationMaster.h index ebab99a77..3e5bcbe80 100644 --- a/Code/SimulationMaster.h +++ b/Code/SimulationMaster.h @@ -20,7 +20,7 @@ #include "reporting/BuildInfo.h" #include "lb/IncompressibilityChecker.h" #include "colloids/ColloidController.h" -#include "net/phased/StepManager.h" +#include "timestep/TimeStepManager.h" #include "comm/AsyncConcern.h" #include "geometry/neighbouring/NeighbouringDataManager.h" @@ -95,8 +95,8 @@ class SimulationMaster hemelb::extraction::IterableDataSource* propertyDataSource; hemelb::extraction::PropertyActor* propertyExtractor; - hemelb::net::phased::StepManager* stepManager; - hemelb::comm::AsyncConcern* netConcern; + hemelb::timestep::TimeStepManager* stepManager; + hemelb::comm::AsyncConcern* asyncCommsManager; static const hemelb::LatticeTimeStep FORCE_FLUSH_PERIOD=1000; }; diff --git a/Code/colloids/ColloidController.h b/Code/colloids/ColloidController.h index 04170f9bd..92b595457 100644 --- a/Code/colloids/ColloidController.h +++ b/Code/colloids/ColloidController.h @@ -8,7 +8,6 @@ #define HEMELB_COLLOIDS_COLLOIDCONTROLLER_H #include -#include "net/IteratedAction.h" #include "geometry/LatticeData.h" #include "geometry/Geometry.h" #include "io/xml/XmlAbstractionLayer.h" @@ -16,13 +15,14 @@ #include "colloids/ParticleSet.h" #include "util/Vector3D.h" #include "units.h" +#include "timestep/Actor.h" namespace hemelb { namespace colloids { /** provides the control interface between colloid simulation and the rest of the system */ - class ColloidController : public net::IteratedAction + class ColloidController : public timestep::Actor { public: /** constructor - currently only initialises the neighbour list */ @@ -39,14 +39,31 @@ namespace hemelb /** destructor - releases resources allocated by this class */ ~ColloidController(); - /** overloaded from IteratedAction */ void RequestComms(); - /** overloaded from IteratedAction */ void EndIteration(); const void OutputInformation(const LatticeTimeStep timestep) const; - + inline virtual void BeginAll() { + } + inline virtual void Begin() { + } + inline virtual void Receive() { + } + inline virtual void PreSend() { + } + inline virtual void Send() { + } + inline virtual void PreWait() { + } + + inline virtual void Wait() { + } + + inline virtual void End() { + } + virtual void EndAll() { + } private: /** Main code communicator */ comm::Communicator::ConstPtr ioComms; diff --git a/Code/comm/AsyncConcern.h b/Code/comm/AsyncConcern.h index 0bec98bc8..2b0baf093 100644 --- a/Code/comm/AsyncConcern.h +++ b/Code/comm/AsyncConcern.h @@ -7,32 +7,40 @@ #define HEMELB_COMM_ASYNC_CONCERN_H #include "comm/Async.h" -#include "net/phased/Concern.h" -#include "net/phased/steps.h" +#include "timestep/Actor.h" namespace hemelb { namespace comm { - using namespace net::phased; - class AsyncConcern : public Concern + class AsyncConcern : public timestep::Actor { public: AsyncConcern(Async::Ptr async) : mAsync(async) { } - bool CallAction(int action) - { - switch (static_cast(action)) - { - case steps::Wait: - mAsync->Wait(); - return true; - - default: - return false; - } + inline virtual void BeginAll() { + } + inline virtual void Begin() { + } + inline virtual void Receive() { + } + inline virtual void PreSend() { + } + inline virtual void Send() { + } + inline virtual void PreWait() { + } + + inline virtual void Wait() { + mAsync->Wait(); + } + + inline virtual void End() { + } + inline virtual void EndAll() { } + private: Async::Ptr mAsync; }; diff --git a/Code/comm/Communicator.h b/Code/comm/Communicator.h index febf88c65..a487c68bf 100644 --- a/Code/comm/Communicator.h +++ b/Code/comm/Communicator.h @@ -167,11 +167,6 @@ namespace hemelb protected: - // TODO: remove when net is dead. - friend class net::CoalescePointPoint; - friend class net::SeparatedPointPoint; - friend class net::ImmediatePointPoint; - virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; virtual std::shared_ptr IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const = 0; diff --git a/Code/extraction/PropertyActor.cc b/Code/extraction/PropertyActor.cc index fd87f621c..cb1edbff0 100644 --- a/Code/extraction/PropertyActor.cc +++ b/Code/extraction/PropertyActor.cc @@ -82,7 +82,7 @@ namespace hemelb } } - void PropertyActor::EndIteration() + void PropertyActor::EndAll() { timers[reporting::Timers::extractionWriting].Start(); propertyWriter->Write(simulationState.GetTimeStep()); diff --git a/Code/extraction/PropertyActor.h b/Code/extraction/PropertyActor.h index 55c3e2ee3..ee7173742 100644 --- a/Code/extraction/PropertyActor.h +++ b/Code/extraction/PropertyActor.h @@ -11,13 +11,13 @@ #include "io/PathManager.h" #include "lb/MacroscopicPropertyCache.h" #include "lb/SimulationState.h" -#include "net/IteratedAction.h" +#include "timestep/Actor.h" namespace hemelb { namespace extraction { - class PropertyActor : public net::IteratedAction + class PropertyActor : public timestep::Actor { public: /** @@ -41,11 +41,26 @@ namespace hemelb */ void SetRequiredProperties(lb::MacroscopicPropertyCache& propertyCache); - /** - * Override the iterated actor end of iteration method to perform writing. - */ - void EndIteration(); - + inline virtual void BeginAll() { + } + inline virtual void Begin() { + } + inline virtual void Receive() { + } + inline virtual void PreSend() { + } + inline virtual void Send() { + } + inline virtual void PreWait() { + } + inline virtual void Wait() { + } + inline virtual void End() { + } + + // Override the iterated actor end of iteration method to perform writing. + virtual void EndAll(); + private: const lb::SimulationState& simulationState; PropertyWriter* propertyWriter; diff --git a/Code/geometry/LatticeData.cc b/Code/geometry/LatticeData.cc index 28325e8ba..a89c0fa5a 100644 --- a/Code/geometry/LatticeData.cc +++ b/Code/geometry/LatticeData.cc @@ -652,17 +652,24 @@ namespace hemelb blockCoords.x = blockIJData / blockCounts.y; } - void LatticeData::SendAndReceive(comm::Async::Ptr commQ) + void LatticeData::Receive(comm::Async::Ptr commQ) { for (std::vector::const_iterator it = neighbouringProcs.begin(); it != neighbouringProcs.end(); ++it) { - // Request the receive into the appropriate bit of FOld. + // Start the receive into the appropriate bit of FOld. commQ->Irecv(GetFOld( (*it).FirstSharedDistribution), (int) ( ( (*it).SharedDistributionCount)), (*it).Rank, 1); - // Request the send from the right bit of FNew. + } + } + void LatticeData::Send(comm::Async::Ptr commQ) + { + for (std::vector::const_iterator it = neighbouringProcs.begin(); + it != neighbouringProcs.end(); ++it) + { + // Start the send from the right bit of FNew. commQ->Isend(GetFNew( (*it).FirstSharedDistribution), (int) ( ( (*it).SharedDistributionCount)), (*it).Rank, diff --git a/Code/geometry/LatticeData.h b/Code/geometry/LatticeData.h index 11eba4515..5f82f2a46 100644 --- a/Code/geometry/LatticeData.h +++ b/Code/geometry/LatticeData.h @@ -51,7 +51,9 @@ namespace hemelb oldDistributions.swap(newDistributions); } - void SendAndReceive(comm::Async::Ptr commQ); + void Receive(comm::Async::Ptr commQ); + void Send(comm::Async::Ptr commQ); + void CopyReceived(); /** diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.cc b/Code/geometry/neighbouring/NeighbouringDataManager.cc index 1903d12ac..fdb635d3e 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.cc +++ b/Code/geometry/neighbouring/NeighbouringDataManager.cc @@ -99,11 +99,12 @@ namespace hemelb void NeighbouringDataManager::TransferFieldDependentInformation() { - RequestComms(); + Receive(); + Send(); commQ->Wait(); } - void NeighbouringDataManager::RequestComms() + void NeighbouringDataManager::Receive() { /*if (needsHaveBeenShared == false) { @@ -125,7 +126,9 @@ namespace hemelb source); } - + } + void NeighbouringDataManager::Send() + { const unsigned Q = localLatticeData.GetLatticeInfo().GetNumVectors(); for (IdsMap::const_iterator iter = needsEachProcHasFromMe.begin(); iter != needsEachProcHasFromMe.end(); diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.h b/Code/geometry/neighbouring/NeighbouringDataManager.h index 88be667a9..e3e49d0b3 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.h +++ b/Code/geometry/neighbouring/NeighbouringDataManager.h @@ -10,7 +10,7 @@ #include "geometry/LatticeData.h" #include "geometry/neighbouring/NeighbouringLatticeData.h" #include "geometry/neighbouring/RequiredSiteInformation.h" -#include "net/IteratedAction.h" +#include "timestep/Actor.h" #include #include @@ -23,7 +23,7 @@ namespace hemelb namespace neighbouring { - class NeighbouringDataManager : public net::IteratedAction + class NeighbouringDataManager : public timestep::Actor { public: NeighbouringDataManager(const LatticeData & localLatticeData, @@ -38,6 +38,7 @@ namespace hemelb // This is collective across the communicator used by the // communication queue given to the constructor void ShareNeeds(); + std::vector &GetNeedsForProc(proc_t proc) { return needsEachProcHasFromMe[proc]; @@ -51,7 +52,24 @@ namespace hemelb // NB this is virtual so that the class can be tested. virtual proc_t ProcForSite(site_t site); protected: - void RequestComms(); + inline virtual void BeginAll() { + } + inline virtual void Begin() { + } + // Implemented + virtual void Receive(); + inline virtual void PreSend() { + } + // Implemented + virtual void Send(); + inline virtual void PreWait() { + } + inline virtual void Wait() { + } + inline virtual void End() { + } + inline virtual void EndAll() { + } private: const LatticeData & localLatticeData; NeighbouringLatticeData & neighbouringLatticeData; diff --git a/Code/lb/CMakeLists.txt b/Code/lb/CMakeLists.txt index 8bacdd357..2f2addf00 100644 --- a/Code/lb/CMakeLists.txt +++ b/Code/lb/CMakeLists.txt @@ -4,7 +4,7 @@ # file AUTHORS. This software is provided under the terms of the # license in the file LICENSE. add_library(hemelb_lb - iolets/BoundaryCommunicator.cc iolets/BoundaryComms.cc iolets/BoundaryValues.cc + iolets/BoundaryValues.cc iolets/InOutLet.cc iolets/InOutLetCosine.cc iolets/InOutLetFile.cc iolets/InOutLetMultiscale.cc diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index 06f2ce138..bf25381c3 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -100,6 +100,9 @@ namespace hemelb void IncompressibilityChecker::PreSend() { + if (isCollectiveRunning) + return; + for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) { DensityEtc etc; @@ -117,6 +120,7 @@ namespace hemelb { // Begin collective. collectiveReq = collectiveComm->Iallreduce(localDensity, reduction, globalDensity); + isCollectiveRunning = true; } bool IncompressibilityChecker::IsDensityDiffWithinRange() const diff --git a/Code/lb/StabilityTester.hpp b/Code/lb/StabilityTester.hpp index f6096eaec..99c7f4f5e 100644 --- a/Code/lb/StabilityTester.hpp +++ b/Code/lb/StabilityTester.hpp @@ -111,6 +111,7 @@ namespace hemelb // Begin collective. collectiveReq = collectiveComm->Iallreduce(localStability, MPI_MIN, globalStability); + isCollectiveRunning = true; } /** diff --git a/Code/lb/iolets/BoundaryCommunicator.h b/Code/lb/iolets/BoundaryCommunicator.h index d19c6da05..63635d499 100644 --- a/Code/lb/iolets/BoundaryCommunicator.h +++ b/Code/lb/iolets/BoundaryCommunicator.h @@ -17,13 +17,13 @@ namespace hemelb { class BoundaryCommunicator { - public: + public: BoundaryCommunicator(comm::Communicator::ConstPtr parent); - bool IsCurrentProcTheBCProc() const; - int GetBCProcRank() const; - comm::Communicator::ConstPtr GetComm() const; - private: - comm::Communicator::Ptr comm; + bool IsCurrentProcTheBCProc() const; + int GetBCProcRank() const; + comm::Communicator::ConstPtr GetComm() const; + private: + comm::Communicator::Ptr comm; }; } diff --git a/Code/lb/iolets/BoundaryValues.cc b/Code/lb/iolets/BoundaryValues.cc index 12c59a5de..c2b8b7c40 100644 --- a/Code/lb/iolets/BoundaryValues.cc +++ b/Code/lb/iolets/BoundaryValues.cc @@ -5,7 +5,6 @@ // license in the file LICENSE. #include "lb/iolets/BoundaryValues.h" -#include "lb/iolets/BoundaryComms.h" #include "util/utilityFunctions.h" #include "util/fileutils.h" #include @@ -21,13 +20,12 @@ namespace hemelb geometry::LatticeData* latticeData, const std::vector &incoming_iolets, SimulationState* simulationState, - comm::Communicator::ConstPtr comms, + comm::Async::Ptr commQ, const util::UnitConverter& units) : - net::IteratedAction(), ioletType(ioletType), totalIoletCount(incoming_iolets.size()), localIoletCount(0), - state(simulationState), unitConverter(units), bcComms(comms) + ioletType(ioletType), totalIoletCount(incoming_iolets.size()), localIoletCount(0), + procsForIolet(incoming_iolets.size()), + state(simulationState), unitConverter(units), asyncCommsQ(commQ) { - std::vector *procsList = new std::vector[totalIoletCount]; - // Determine which iolets need comms and create them for (int ioletIndex = 0; ioletIndex < totalIoletCount; ioletIndex++) { @@ -40,28 +38,20 @@ namespace hemelb bool isIOletOnThisProc = IsIOletOnThisProc(ioletType, latticeData, ioletIndex); hemelb::log::Logger::Log("BOUNDARYVALUES.CC - isioletonthisproc? : %d", isIOletOnThisProc); - procsList[ioletIndex] = GatherProcList(isIOletOnThisProc); + procsForIolet[ioletIndex] = GatherProcList(isIOletOnThisProc); // With information on whether a proc has an IOlet and the list of procs for each IOlte // on the BC task we can create the comms - if (isIOletOnThisProc || bcComms.IsCurrentProcTheBCProc()) + if (isIOletOnThisProc || IsCurrentProcTheBCProc()) { localIoletCount++; localIoletIDs.push_back(ioletIndex); -// hemelb::log::Logger::Log("BOUNDARYVALUES.H - ioletIndex: %d", ioletIndex); - -// if (iolet->IsCommsRequired()) //DEREK: POTENTIAL MULTISCALE ISSUE (this if-statement) -// { - iolet->SetComms(new BoundaryComms(state, procsList[ioletIndex], bcComms, isIOletOnThisProc)); -// } } } // Send out initial values Reset(); - // Clear up - delete[] procsList; hemelb::log::Logger::Log("BOUNDARYVALUES.H - ioletCount: %d, first iolet ID %d", localIoletCount, localIoletIDs[0]); @@ -97,18 +87,20 @@ namespace hemelb { std::vector processorsNeedingIoletList(0); - // This is where the info about whether a proc contains the given inlet/outlet is sent - // If it does contain the given inlet/outlet it sends a true value, else it sends a false. - int isIOletOnThisProc = hasBoundary; // true if inlet i is on this proc + // This is where the info about whether a proc contains the + // given inlet/outlet is sent If it does contain the given + // inlet/outlet it sends a true value, else it sends a false. + // Ideally bools, but MPI doesn't support them :( + char isIOletOnThisProc = hasBoundary; // true if inlet i is on this proc - // These should be bool, but MPI only supports MPI_INT - // For each inlet/outlet there is an array of length equal to total number of procs. - // Each stores true/false value. True if proc of rank equal to the index contains - // the given inlet/outlet. + // For each inlet/outlet there is an array of length equal to + // total number of procs. Each stores true/false value. True + // if proc of rank equal to the index contains the given + // inlet/outlet. - std::vector processorsNeedingIoletFlags = bcComms.GetComm()->Gather(isIOletOnThisProc, bcComms.GetBCProcRank()); + auto processorsNeedingIoletFlags = asyncCommsQ->GetComm()->Gather(isIOletOnThisProc, GetBCProcRank()); - if (bcComms.IsCurrentProcTheBCProc()) + if (IsCurrentProcTheBCProc()) { // Now we have an array for each IOlet with true (1) at indices corresponding to // processes that are members of that group. We have to convert this into arrays @@ -122,60 +114,81 @@ namespace hemelb } } - return processorsNeedingIoletList; // return by copy + return processorsNeedingIoletList; // return by move } - void BoundaryValues::RequestComms() + void BoundaryValues::Begin() { - for (int i = 0; i < localIoletCount; i++) + for (int i = 0; i < localIoletCount; i++) { - HandleComms(GetLocalIolet(i)); + auto iolet = GetLocalIolet(i); + if (iolet->IsCommsRequired()) + { + iolet->Begin(this); + } } } - - void BoundaryValues::HandleComms(iolets::InOutLet* iolet) + void BoundaryValues::Receive() { - - if (iolet->IsCommsRequired()) + for (int i = 0; i < localIoletCount; i++) { - iolet->DoComms(bcComms, state->GetTimeStep()); + auto iolet = GetLocalIolet(i); + if (iolet->IsCommsRequired()) + { + iolet->Receive(this, asyncCommsQ); + } } - } - - void BoundaryValues::EndIteration() + void BoundaryValues::Send() { - for (int i = 0; i < localIoletCount; i++) + for (int i = 0; i < localIoletCount; i++) { - if (GetLocalIolet(i)->IsCommsRequired()) - { - GetLocalIolet(i)->GetComms()->FinishSend(); - } + auto iolet = GetLocalIolet(i); + if (iolet->IsCommsRequired()) + { + iolet->Send(this, asyncCommsQ); + } } } - - void BoundaryValues::FinishReceive() + // Wait done by AsyncActor + void BoundaryValues::End() { - for (int i = 0; i < localIoletCount; i++) + for (int i = 0; i < localIoletCount; i++) { - if (GetLocalIolet(i)->IsCommsRequired()) - { - GetLocalIolet(i)->GetComms()->Wait(); - } + auto iolet = GetLocalIolet(i); + if (iolet->IsCommsRequired()) + { + iolet->CommsComplete(this); + } } } + + void BoundaryValues::ForceCommunication() + { + Begin(); + { + auto tmpQ = comm::Async::New(asyncCommsQ->GetComm()); + for (int i = 0; i < localIoletCount; i++) + { + auto iolet = GetLocalIolet(i); + if (iolet->IsCommsRequired()) + { + iolet->Receive(this, tmpQ); + iolet->Send(this, tmpQ); + } + } + } + End(); + } void BoundaryValues::Reset() { for (int i = 0; i < localIoletCount; i++) { GetLocalIolet(i)->Reset(*state); - if (GetLocalIolet(i)->IsCommsRequired()) - { - GetLocalIolet(i)->GetComms()->WaitAllComms(); - - } } + // TODO: Are we sure we need to force a comms cycle? + ForceCommunication(); } // This assumes the program has already waited for comms to finish before diff --git a/Code/lb/iolets/BoundaryValues.h b/Code/lb/iolets/BoundaryValues.h index 0f4001bda..cd4f2cc9e 100644 --- a/Code/lb/iolets/BoundaryValues.h +++ b/Code/lb/iolets/BoundaryValues.h @@ -8,10 +8,9 @@ #define HEMELB_LB_IOLETS_BOUNDARYVALUES_H #include "comm/Communicator.h" -#include "net/IteratedAction.h" +#include "timestep/Actor.h" #include "lb/iolets/InOutLet.h" #include "geometry/LatticeData.h" -#include "lb/iolets/BoundaryCommunicator.h" namespace hemelb { @@ -20,27 +19,47 @@ namespace hemelb namespace iolets { - class BoundaryValues : public net::IteratedAction + class BoundaryValues : public timestep::Actor { public: BoundaryValues(geometry::SiteType ioletType, geometry::LatticeData* latticeData, const std::vector &iolets, SimulationState* simulationState, - comm::Communicator::ConstPtr comms, + comm::Async::Ptr commQ, const util::UnitConverter& units); ~BoundaryValues(); - void RequestComms(); - void EndIteration(); + // timestep::Actor interface + virtual void BeginAll() { /* not needed */ } + virtual void Begin(); + virtual void Receive(); + virtual void PreSend() { /* not needed */ } + virtual void Send(); + virtual void PreWait() { /* not needed */ } + virtual void Wait() { /* not needed */ } + virtual void End(); + virtual void EndAll() { /* not needed */ } + + + void ForceCommunication(); + // void RequestComms(); + // void EndIteration(); void Reset(); - void FinishReceive(); + // void FinishReceive(); LatticeDensity GetBoundaryDensity(const int index); - static proc_t GetBCProcRank(); - iolets::InOutLet* GetLocalIolet(unsigned int index) + static constexpr int GetBCProcRank() { + return 0; + } + + inline bool IsCurrentProcTheBCProc() { + return asyncCommsQ->GetComm()->Rank() == GetBCProcRank(); + } + + iolets::InOutLet* GetLocalIolet(unsigned int index) { return iolets[localIoletIDs[index]]; } @@ -57,10 +76,20 @@ namespace hemelb return ioletType; } + inline const std::vector& GetProcsForIolet(int i) const { + return procsForIolet[i]; + } + inline const std::vector& GetProcsForIolet(const InOutLet* io) const { + auto iter = std::find(iolets.begin(), iolets.end(), io); + if (iter == iolets.end()) + throw Exception() << "Can't find iolet in BoundaryValues"; + return GetProcsForIolet(iter - iolets.begin()); + } + private: bool IsIOletOnThisProc(geometry::SiteType ioletType, geometry::LatticeData* latticeData, int boundaryId); std::vector GatherProcList(bool hasBoundary); - void HandleComms(iolets::InOutLet* iolet); + // void HandleComms(iolets::InOutLet* iolet); geometry::SiteType ioletType; int totalIoletCount; // Number of IOlets and vector of their indices for communication purposes @@ -68,10 +97,11 @@ namespace hemelb std::vector localIoletIDs; // Has to be a vector of pointers for InOutLet polymorphism std::vector iolets; - + // For each iolet, lists the procs that have it (only populated on BCProc rank) + std::vector> procsForIolet; SimulationState* state; const util::UnitConverter& unitConverter; - BoundaryCommunicator bcComms; + comm::Async::Ptr asyncCommsQ; } ; } diff --git a/Code/lb/iolets/InOutLet.cc b/Code/lb/iolets/InOutLet.cc index 6add96f56..b209438d8 100644 --- a/Code/lb/iolets/InOutLet.cc +++ b/Code/lb/iolets/InOutLet.cc @@ -11,11 +11,6 @@ namespace hemelb { namespace iolets { - void InOutLet::DoComms(const BoundaryCommunicator& bcComms, const LatticeTimeStep timeStep) - { - // pass - } - namespace { unsigned SmallestMagnitudeComponent(const LatticeVector r) diff --git a/Code/lb/iolets/InOutLet.h b/Code/lb/iolets/InOutLet.h index 8b75c282a..01b4a1af6 100644 --- a/Code/lb/iolets/InOutLet.h +++ b/Code/lb/iolets/InOutLet.h @@ -10,6 +10,7 @@ #include "util/Vector3D.h" #include "util/UnitConverter.h" #include "lb/SimulationState.h" +#include "comm/Async.h" namespace hemelb { @@ -17,11 +18,8 @@ namespace hemelb { namespace iolets { - - //forward declare boundary comms class - class BoundaryComms; - class BoundaryCommunicator; - + class BoundaryValues; + /** * Base class for extra data needed by LB BC implementations. * Makes "Iolet coordinates" available. @@ -56,8 +54,7 @@ namespace hemelb class InOutLet { public: - InOutLet() : - comms(NULL), extraData(NULL) + InOutLet() : extraData(NULL) { } virtual ~InOutLet() @@ -80,6 +77,15 @@ namespace hemelb return false; } + virtual void Begin(BoundaryValues*) { + } + virtual void Receive(BoundaryValues*, comm::Async::Ptr) { + } + virtual void Send(BoundaryValues* , comm::Async::Ptr) { + } + virtual void CommsComplete(BoundaryValues*) { + } + /*** * This is a castable? virtual method, which is perhaps an anti-pattern * We should potentially use dynamic cast checks instead. @@ -89,20 +95,7 @@ namespace hemelb { return false; } - void SetComms(BoundaryComms * boundaryComms) - { - comms = boundaryComms; - } - BoundaryComms * GetComms() const - { - return comms; - } - /*** - * Carry out communication necessary - * @param isIoProcess Is the process the master process? - */ - virtual void DoComms(const BoundaryCommunicator& bcComms, const LatticeTimeStep timeStep); - + /*** * Set up the Iolet. * @param units a UnitConverter instance. @@ -164,7 +157,6 @@ namespace hemelb LatticeDensity minimumSimulationDensity; LatticePosition position; util::Vector3D normal; - BoundaryComms* comms; IoletExtraData* extraData; friend class IoletExtraData; }; diff --git a/Code/lb/iolets/InOutLetMultiscale.cc b/Code/lb/iolets/InOutLetMultiscale.cc index ea9f153b2..711868192 100644 --- a/Code/lb/iolets/InOutLetMultiscale.cc +++ b/Code/lb/iolets/InOutLetMultiscale.cc @@ -6,7 +6,6 @@ #include "lb/iolets/InOutLetMultiscale.h" #include "configuration/SimConfig.h" -#include "lb/iolets/BoundaryComms.h" #include "lb/iolets/BoundaryValues.h" #include "comm/Async.h" @@ -109,43 +108,29 @@ namespace hemelb commsRequired = b; } - /* Distribution of internal pressure values */ - void InOutLetMultiscale::DoComms(const BoundaryCommunicator& bcComms, LatticeTimeStep time_step) - { - bool isIoProc = bcComms.IsCurrentProcTheBCProc(); - hemelb::log::Logger::Log("DoComms in IoletMultiscale triggered: %s", - isIoProc - ? "true" - : "false"); - double pressure_array[3]; + void InOutLetMultiscale::Begin(BoundaryValues* bv) { //TODO: Change these operators on SharedValue. pressure_array[0] = pressure.GetPayload(); pressure_array[1] = minPressure.GetPayload(); pressure_array[2] = maxPressure.GetPayload(); - + } + void InOutLetMultiscale::Receive(BoundaryValues* bv, comm::Async::Ptr commQ) { + // everyone receives from BC master proc + commQ->Irecv(pressure_array, 3, bv->GetBCProcRank(), 0); + } + void InOutLetMultiscale::Send(BoundaryValues* bv, comm::Async::Ptr commQ) { + if(bv->IsCurrentProcTheBCProc()) { - comm::Async commsNet(bcComms.GetComm()); - const std::vector& procList = comms->GetListOfProcs(); //TODO: CHECK + IMPROVE! - - // If this proc is to do IO, send the pressure array list to all cores that require it. - if (isIoProc && procList[0] != bcComms.GetBCProcRank()) - { - for (std::vector::const_iterator it = procList.begin(); it != procList.end(); it++) - { - commsNet.Isend(pressure_array, 3, *it, 0); - } - } - // Otherwise, receive the pressure array list from the core. - else if (procList[0] != bcComms.GetBCProcRank()) - { - commsNet.Irecv(pressure_array, 3, bcComms.GetBCProcRank(), 0); - } + const std::vector& procList = bv->GetProcsForIolet(this); //TODO: CHECK + IMPROVE! - // Perform the send / receive when the Async destructs - } - - if (!isIoProc) - { + for (auto dest_rank: procList) + commQ->Isend(pressure_array, 3, dest_rank, 0); + } + } + + void InOutLetMultiscale::CommsComplete(BoundaryValues* bv) { + if (!bv->IsCurrentProcTheBCProc()) + { pressure.SetPayload(static_cast (pressure_array[0])); minPressure.SetPayload(static_cast (pressure_array[1])); maxPressure.SetPayload(static_cast (pressure_array[2])); @@ -153,8 +138,8 @@ namespace hemelb pressure.GetPayload(), minPressure.GetPayload(), maxPressure.GetPayload()); - } - } + } + } } } } diff --git a/Code/lb/iolets/InOutLetMultiscale.h b/Code/lb/iolets/InOutLetMultiscale.h index dc2e2df18..11b9dc5d7 100644 --- a/Code/lb/iolets/InOutLetMultiscale.h +++ b/Code/lb/iolets/InOutLetMultiscale.h @@ -11,7 +11,6 @@ #include "multiscale/Intercommunicand.h" #include "multiscale/SharedValue.h" #include "log/Logger.h" -#include "lb/iolets/BoundaryCommunicator.h" namespace hemelb { @@ -84,7 +83,11 @@ namespace hemelb virtual bool IsCommsRequired() const; virtual void SetCommsRequired(bool b); - void DoComms(const BoundaryCommunicator& bcComms, const LatticeTimeStep timeStep); + + virtual void Begin(BoundaryValues*); + virtual void Receive(BoundaryValues*, comm::Async::Ptr); + virtual void Send(BoundaryValues* bv, comm::Async::Ptr); + virtual void CommsComplete(BoundaryValues*); private: std::string label; @@ -95,6 +98,7 @@ namespace hemelb multiscale::SharedValue minPressure; multiscale::SharedValue maxPressure; mutable multiscale::SharedValue velocity; + double pressure_array[3]; }; } } diff --git a/Code/lb/lb.h b/Code/lb/lb.h index aaa28f719..b234fcb7c 100644 --- a/Code/lb/lb.h +++ b/Code/lb/lb.h @@ -7,8 +7,8 @@ #ifndef HEMELB_LB_LB_H #define HEMELB_LB_LB_H -#include "net/IteratedAction.h" #include "comm/Communicator.h" +#include "timestep/Actor.h" #include "lb/SimulationState.h" #include "lb/iolets/BoundaryValues.h" #include "lb/MacroscopicPropertyCache.h" @@ -27,10 +27,10 @@ namespace hemelb { /** * Class providing core Lattice Boltzmann functionality. - * Implements the IteratedAction interface. + * Implements the timestep::Actor interface. */ template - class LBM : public net::IteratedAction + class LBM : public timestep::Actor { private: // Use the kernel specified through the build system. This will select one of the above classes. @@ -62,11 +62,24 @@ namespace hemelb geometry::neighbouring::NeighbouringDataManager *neighbouringDataManager); ~LBM(); - void RequestComms(); ///< part of IteratedAction interface. - void PreSend(); ///< part of IteratedAction interface. - void PreReceive(); ///< part of IteratedAction interface. - void PostReceive(); ///< part of IteratedAction interface. - void EndIteration(); ///< part of IteratedAction interface. + // Actor interface + inline virtual void BeginAll() { + // No-op + } + virtual void Begin() { + // No-op + } + virtual void Receive(); + virtual void PreSend(); + virtual void Send(); + virtual void PreWait(); + inline virtual void Wait() + { + // This is a no-op as waiting is delegated to the async + // comms actor + } + virtual void End(); + virtual void EndAll(); site_t TotalFluidSiteCount() const; void SetTotalFluidSiteCount(site_t); diff --git a/Code/lb/lb.hpp b/Code/lb/lb.hpp index 7e79baf2f..133ac3451 100644 --- a/Code/lb/lb.hpp +++ b/Code/lb/lb.hpp @@ -7,7 +7,6 @@ #ifndef HEMELB_LB_LB_HPP #define HEMELB_LB_LB_HPP -#include "io/writers/xdr/XdrMemWriter.h" #include "lb/lb.h" namespace hemelb @@ -167,16 +166,11 @@ namespace hemelb } template - void LBM::RequestComms() + void LBM::Receive() { timings[hemelb::reporting::Timers::lb].Start(); - - // Delegate to the lattice data object to post the asynchronous sends and receives - // (via the Net object). - // NOTE that this doesn't actually *perform* the sends and receives, it asks the Net - // to include them in the ISends and IRecvs that happen later. - mLatDat->SendAndReceive(mCommQ); - + // Delegate to the lattice data object + mLatDat->Receive(mCommQ); timings[hemelb::reporting::Timers::lb].Stop(); } @@ -187,10 +181,12 @@ namespace hemelb timings[hemelb::reporting::Timers::lb_calc].Start(); /** - * In the PreSend phase, we do LB on all the sites that need to have results sent to - * neighbouring ranks ('domainEdge' sites). In site id terms, this means we start at the - * end of the sites whose neighbours all lie on this rank ('midDomain'), then progress - * through the sites of each type in turn. + * In the PreSend phase, we do LB on all the sites that need to + * have results sent to neighbouring ranks ('domainEdge' + * sites). In site id terms, this means we start at the end of + * the sites whose neighbours all lie on this rank + * ('midDomain'), then progress through the sites of each type + * in turn. */ site_t offset = mLatDat->GetMidDomainSiteCount(); @@ -200,11 +196,9 @@ namespace hemelb StreamAndCollide(mWallCollision, offset, mLatDat->GetDomainEdgeCollisionCount(1)); offset += mLatDat->GetDomainEdgeCollisionCount(1); - mInletValues->FinishReceive(); StreamAndCollide(mInletCollision, offset, mLatDat->GetDomainEdgeCollisionCount(2)); offset += mLatDat->GetDomainEdgeCollisionCount(2); - mOutletValues->FinishReceive(); StreamAndCollide(mOutletCollision, offset, mLatDat->GetDomainEdgeCollisionCount(3)); offset += mLatDat->GetDomainEdgeCollisionCount(3); @@ -216,20 +210,31 @@ namespace hemelb timings[hemelb::reporting::Timers::lb_calc].Stop(); timings[hemelb::reporting::Timers::lb].Stop(); } - + + template + void LBM::Send() + { + timings[hemelb::reporting::Timers::lb].Start(); + // Delegate to the lattice data + mLatDat->Send(mCommQ); + timings[hemelb::reporting::Timers::lb].Stop(); + } + template - void LBM::PreReceive() + void LBM::PreWait() { timings[hemelb::reporting::Timers::lb].Start(); timings[hemelb::reporting::Timers::lb_calc].Start(); /** - * In the PreReceive phase, we perform LB for all the sites whose neighbours lie on this - * rank ('midDomain' rather than 'domainEdge' sites). Ideally this phase is the longest bit (maximising time for the asynchronous sends - * and receives to complete). + * In the PreWait phase, we perform LB for all the sites whose + * neighbours lie on this rank ('midDomain' rather than + * 'domainEdge' sites). Ideally this phase is the longest bit + * (maximising time for the asynchronous sends and receives to + * complete). * - * In site id terms, this means starting at the first site and progressing through the - * midDomain sites, one type at a time. + * In site id terms, this means starting at the first site and + * progressing through the midDomain sites, one type at a time. */ site_t offset = 0; @@ -255,7 +260,7 @@ namespace hemelb } template - void LBM::PostReceive() + void LBM::End() { timings[hemelb::reporting::Timers::lb].Start(); @@ -311,7 +316,7 @@ namespace hemelb } template - void LBM::EndIteration() + void LBM::EndAll() { timings[hemelb::reporting::Timers::lb].Start(); timings[hemelb::reporting::Timers::lb_calc].Start(); diff --git a/Code/multiscale/MultiscaleSimulationMaster.h b/Code/multiscale/MultiscaleSimulationMaster.h index 8330cc64b..06d196e52 100644 --- a/Code/multiscale/MultiscaleSimulationMaster.h +++ b/Code/multiscale/MultiscaleSimulationMaster.h @@ -225,6 +225,14 @@ namespace hemelb * (it's hard enough to get the physics right with a consistent * state ;)). */ + // TODO: Derek to please comment on this. + + // With the rejigged BoundarValues, the above comment is + // probably defunct. Hence I have removed the RequestComms + // calls below. (They may have in fact not quite worked as + // intented as they was no call to the net->Dispatch + // method to actually cause communication) + hemelb::log::Logger::Log("inlet and outlet count: %d and %d", inletValues->GetLocalIoletCount(), outletValues->GetLocalIoletCount()); @@ -232,14 +240,13 @@ namespace hemelb inletValues->GetLocalIolet(0)->IsCommsRequired()); hemelb::log::Logger::Log("outlets: %d", outletValues->GetLocalIolet(0)->IsCommsRequired()); + // SetCommsRequired(inletValues, true); + // SetCommsRequired(outletValues, true); - SetCommsRequired(inletValues, true); - SetCommsRequired(outletValues, true); - - inletValues->RequestComms(); - outletValues->RequestComms(); - SetCommsRequired(inletValues, false); - SetCommsRequired(outletValues, false); + // inletValues->RequestComms(); + // outletValues->RequestComms(); + // SetCommsRequired(inletValues, false); + // SetCommsRequired(outletValues, false); for (unsigned int i = 0; i < inletValues->GetLocalIoletCount(); i++) { diff --git a/Code/timestep/Actor.h b/Code/timestep/Actor.h index c87e981f3..7e4108e20 100644 --- a/Code/timestep/Actor.h +++ b/Code/timestep/Actor.h @@ -15,7 +15,9 @@ namespace hemelb { protected: friend class TimeStepManager; - + + inline virtual ~Actor() { + } virtual void BeginAll() = 0; virtual void Begin() = 0; virtual void Receive() = 0; diff --git a/Code/timestep/CollectiveActor.cc b/Code/timestep/CollectiveActor.cc index 5aef7777c..bd1b0e436 100644 --- a/Code/timestep/CollectiveActor.cc +++ b/Code/timestep/CollectiveActor.cc @@ -19,6 +19,10 @@ namespace hemelb { } + void CollectiveActor::Receive() + { + } + /** * Wait on the collectives to finish. */ diff --git a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h index 50b42ce75..19c0aab72 100644 --- a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h +++ b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h @@ -243,11 +243,11 @@ namespace hemelb 0, 0); - net::phased::StepManager stepManager; - stepManager.RegisterIteratedActorSteps(*manager, 0); + timestep::TimeStepManager stepManager(1); + stepManager.AddToPhase(0, manager); comm::AsyncConcern netConcern = comm::AsyncConcern(commQ); - stepManager.RegisterCommsForAllPhases(netConcern); - stepManager.CallActions(); + stepManager.AddToPhase(0, &netConcern); + stepManager.DoStep(); MockComms()->ExpectationsAllCompleted(); NeighbouringSite transferredSite = data->GetSite(targetGlobalOneDIdx); diff --git a/Code/unittests/helpers/HasCommsTestFixture.h b/Code/unittests/helpers/HasCommsTestFixture.h index 27e34cd84..fb38ca024 100644 --- a/Code/unittests/helpers/HasCommsTestFixture.h +++ b/Code/unittests/helpers/HasCommsTestFixture.h @@ -22,11 +22,13 @@ namespace hemelb void setUp() { //hemelbCommunicator = net::IOCommunicator::Instance(); + asyncCommQ = comm::Async::New(hemelbCommunicator); } void tearDown() { //hemelbCommunicator = NULL; + asyncCommQ = nullptr; } static void Init(comm::Communicator::ConstPtr inst) @@ -39,8 +41,13 @@ namespace hemelb { return hemelbCommunicator; } + comm::Async::Ptr Async() + { + return asyncCommQ; + } private: static comm::Communicator::ConstPtr hemelbCommunicator; + comm::Async::Ptr asyncCommQ; }; comm::Communicator::ConstPtr HasCommsTestFixture::hemelbCommunicator = NULL; diff --git a/Code/unittests/lbtests/IncompressibilityCheckerTests.h b/Code/unittests/lbtests/IncompressibilityCheckerTests.h index 2806df875..7264c517b 100644 --- a/Code/unittests/lbtests/IncompressibilityCheckerTests.h +++ b/Code/unittests/lbtests/IncompressibilityCheckerTests.h @@ -11,6 +11,7 @@ #include "lb/IncompressibilityChecker.h" #include "net/phased/steps.h" +#include "timestep/TimeStepManager.h" #include "unittests/FourCubeLatticeData.h" #include "unittests/reporting/Mocks.h" @@ -55,7 +56,6 @@ namespace hemelb eps = 1e-9; timings = new hemelb::reporting::Timers(Comms()); - commQ = comm::Async::New(Comms()); } void tearDown() @@ -64,13 +64,12 @@ namespace hemelb FourCubeBasedTestFixture::tearDown(); } - void AdvanceActorOneTimeStep(net::IteratedAction& actor) + void AdvanceActorOneTimeStep(timestep::Actor& actor) { - for (int phase = net::phased::steps::BeginPhase; phase <= net::phased::steps::EndPhase; ++phase) - { - actor.CallAction(phase); - } - } + timestep::TimeStepManager tsm(1); + tsm.AddToPhase(0, &actor); + tsm.DoStep(); + } void TestIncompressibilityCheckerRootNode() { @@ -169,7 +168,6 @@ namespace hemelb distribn_t largestDefaultDensity; distribn_t largestDefaultVelocityMagnitude; hemelb::reporting::Timers* timings; - comm::Async::Ptr commQ; distribn_t eps; }; diff --git a/Code/unittests/lbtests/StreamerTests.h b/Code/unittests/lbtests/StreamerTests.h index 33d4d8c02..52885010c 100644 --- a/Code/unittests/lbtests/StreamerTests.h +++ b/Code/unittests/lbtests/StreamerTests.h @@ -760,11 +760,12 @@ namespace hemelb void TestNashZerothOrderPressureIolet() { + auto async = comm::Async::New(Comms()); lb::iolets::BoundaryValues inletBoundary(geometry::INLET_TYPE, latDat, simConfig->GetInlets(), simState, - Comms(), + async, *unitConverter); initParams.boundaryObject = &inletBoundary; @@ -870,11 +871,12 @@ namespace hemelb void TestNashZerothOrderPressureBB() { + auto async = comm::Async::New(Comms()); lb::iolets::BoundaryValues inletBoundary(geometry::INLET_TYPE, latDat, simConfig->GetInlets(), simState, - Comms(), + async, *unitConverter); initParams.boundaryObject = &inletBoundary; diff --git a/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h b/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h index 2b8ad0f96..49d6b8646 100644 --- a/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h +++ b/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h @@ -91,7 +91,7 @@ namespace hemelb latDat, simConfig->GetInlets(), simState, - Comms(), + Async(), *unitConverter); InOutLetCosine* inlet = GetIolet(inletBoundary); // We have to make the outlet sane and consistent with the geometry now. @@ -128,7 +128,7 @@ namespace hemelb latDat, simConfig->GetOutlets(), simState, - Comms(), + Async(), *unitConverter); InOutLetCosine* outlet = GetIolet(outletBoundary); diff --git a/Code/unittests/lbtests/iolets/BoundaryTests.h b/Code/unittests/lbtests/iolets/BoundaryTests.h index 85a70684c..e96e59bfb 100644 --- a/Code/unittests/lbtests/iolets/BoundaryTests.h +++ b/Code/unittests/lbtests/iolets/BoundaryTests.h @@ -34,15 +34,15 @@ namespace hemelb CPPUNIT_TEST(TestUpdate); CPPUNIT_TEST(TestUpdateFile); CPPUNIT_TEST_SUITE_END(); - + public: void TestConstruct() { - double targetStartDensity = unitConverter->ConvertPressureToLatticeUnits(80.0 - 1.0) / Cs2; + double targetStartDensity = unitConverter->ConvertPressureToLatticeUnits(80.0 - 1.0) / Cs2; inlets = new BoundaryValues(hemelb::geometry::INLET_TYPE, latDat, simConfig->GetInlets(), simState, - Comms(), + Async(), *unitConverter); CPPUNIT_ASSERT_DOUBLES_EQUAL(targetStartDensity, inlets->GetBoundaryDensity(0), 1e-9); delete inlets; @@ -53,7 +53,7 @@ namespace hemelb latDat, simConfig->GetInlets(), simState, - Comms(), + Async(), *unitConverter); CPPUNIT_ASSERT_DOUBLES_EQUAL(pressureToDensity(80.0 - 1.0), inlets->GetBoundaryDensity(0), 1e-9); @@ -91,7 +91,7 @@ namespace hemelb latDat, fileInletConfig->GetInlets(), simState, - Comms(), + Async(), *unitConverter); CPPUNIT_ASSERT_DOUBLES_EQUAL(pressureToDensity(78.0), inlets->GetBoundaryDensity(0), 1e-6); @@ -120,8 +120,10 @@ namespace hemelb + (pressure - REFERENCE_PRESSURE_mmHg) * mmHg_TO_PASCAL * inverseVelocity * inverseVelocity / (Cs2 * BLOOD_DENSITY_Kg_per_m3); } + + private: BoundaryValues *inlets; - }; + }; //BoundaryTests CPPUNIT_TEST_SUITE_REGISTRATION(BoundaryTests); } // iolets diff --git a/Code/unittests/main.cc b/Code/unittests/main.cc index 63bafec7f..666bab43e 100644 --- a/Code/unittests/main.cc +++ b/Code/unittests/main.cc @@ -19,7 +19,7 @@ #include "unittests/geometry/geometry.h" #include "unittests/SimulationMasterTests.h" #include "unittests/extraction/extraction.h" -#include "unittests/net/net.h" + #include "unittests/multiscale/multiscale.h" #ifdef HEMELB_BUILD_MULTISCALE #include "unittests/multiscale/mpwide/mpwide.h" From 9935c886d4c1eddb420ed5e9a34129fe1949c55d Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 21 Feb 2017 13:02:08 +0000 Subject: [PATCH 84/99] fix ordering bug in step manager --- Code/geometry/LatticeData.cc | 1 - Code/timestep/TimeStepManager.cc | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/Code/geometry/LatticeData.cc b/Code/geometry/LatticeData.cc index a89c0fa5a..65bc392d9 100644 --- a/Code/geometry/LatticeData.cc +++ b/Code/geometry/LatticeData.cc @@ -7,7 +7,6 @@ #include #include -#include "debug/Debugger.h" #include "log/Logger.h" #include "comm/Communicator.h" #include "geometry/BlockTraverser.h" diff --git a/Code/timestep/TimeStepManager.cc b/Code/timestep/TimeStepManager.cc index 34e77b966..b40a669d0 100644 --- a/Code/timestep/TimeStepManager.cc +++ b/Code/timestep/TimeStepManager.cc @@ -5,6 +5,7 @@ #include "timestep/TimeStepManager.h" #include "timestep/Actor.h" +#include "debug/Debugger.h" namespace hemelb { @@ -29,15 +30,25 @@ namespace hemelb for (auto& ph: mPhases) { for (auto ap: ph.actors) - { ap->Begin(); + + for (auto ap: ph.actors) ap->Receive(); + + for (auto ap: ph.actors) ap->PreSend(); + + for (auto ap: ph.actors) ap->Send(); + + for (auto ap: ph.actors) ap->PreWait(); + + for (auto ap: ph.actors) ap->Wait(); + + for (auto ap: ph.actors) ap->End(); - } } for (auto& ph: mPhases) From 8e38972f1206ef1ab3dcc73121fb9699ff0cd7d2 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 21 Feb 2017 14:21:32 +0000 Subject: [PATCH 85/99] add missing include --- Code/reporting/Timers.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Code/reporting/Timers.hpp b/Code/reporting/Timers.hpp index e22cb04d5..ff0c89aa2 100644 --- a/Code/reporting/Timers.hpp +++ b/Code/reporting/Timers.hpp @@ -8,6 +8,7 @@ #define HEMELB_REPORTING_TIMERS_HPP #include "reporting/Timers.h" +#include namespace hemelb { From 4216145ed34fb4b91d74290e33feb34b11a37eaa Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 21 Feb 2017 16:09:57 +0000 Subject: [PATCH 86/99] missing include --- Code/comm/Request.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Code/comm/Request.h b/Code/comm/Request.h index ceb80359a..5011e19b1 100644 --- a/Code/comm/Request.h +++ b/Code/comm/Request.h @@ -11,6 +11,7 @@ #define HEMELB_COMM_REQUEST_H #include +#include namespace hemelb { From 618f01f94ba9505c7a78d209edd1daf9d54fd0a8 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 21 Feb 2017 16:15:17 +0000 Subject: [PATCH 87/99] move linux debugger to new comms --- Code/debug/linux/LinuxDebugger.cc | 2 +- Code/debug/linux/LinuxDebugger.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Code/debug/linux/LinuxDebugger.cc b/Code/debug/linux/LinuxDebugger.cc index 8f8aa66d4..269127154 100644 --- a/Code/debug/linux/LinuxDebugger.cc +++ b/Code/debug/linux/LinuxDebugger.cc @@ -14,7 +14,7 @@ namespace hemelb { namespace debug { - LinuxDebugger::LinuxDebugger(const char* const executable, const net::MpiCommunicator& comm) : + LinuxDebugger::LinuxDebugger(const char* const executable, comm::Communicator::ConstPtr comm) : ActiveDebugger(executable, comm) {} const std::string LinuxDebugger::GetBinaryPath(void) const diff --git a/Code/debug/linux/LinuxDebugger.h b/Code/debug/linux/LinuxDebugger.h index 06e44eecb..590ac7b15 100644 --- a/Code/debug/linux/LinuxDebugger.h +++ b/Code/debug/linux/LinuxDebugger.h @@ -22,7 +22,7 @@ namespace hemelb const std::string GetPlatformScript(void) const; // C'tor... - LinuxDebugger(const char* const executable, const net::MpiCommunicator& comm); + LinuxDebugger(const char* const executable, comm::Communicator::ConstPtr comm); // ... which the factory function needs to be able to get at. friend class Debugger; From 2775de66ee940f0c2760e5e840cebfc33f48f263 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 21 Feb 2017 16:23:14 +0000 Subject: [PATCH 88/99] tidy the debugger instantiation --- Code/debug/Debugger.cc | 6 +++--- Code/debug/linux/LinuxDebugger.h | 2 -- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Code/debug/Debugger.cc b/Code/debug/Debugger.cc index 283d4f105..db719c72c 100644 --- a/Code/debug/Debugger.cc +++ b/Code/debug/Debugger.cc @@ -16,9 +16,9 @@ namespace hemelb Debugger* Debugger::Init(bool active, const char * const executable, comm::Communicator::ConstPtr comm) { /* Static member function that implements the singleton pattern. - * Use the namespace function PlatformDebuggerFactory to - * actually construct the instance. It should be defined in the - * appropriate platform subdirectory. + * + * PlatformDebugger.h is configured to have an appropriate + * typedef for the current platform. */ if (Debugger::singleton == NULL) { diff --git a/Code/debug/linux/LinuxDebugger.h b/Code/debug/linux/LinuxDebugger.h index 590ac7b15..0bb626a0d 100644 --- a/Code/debug/linux/LinuxDebugger.h +++ b/Code/debug/linux/LinuxDebugger.h @@ -28,8 +28,6 @@ namespace hemelb }; - // Factory. Don't be calling this. - Debugger* PlatformDebuggerFactory(const char* const executable, const net::MpiCommunicator& comm); } } From b6a1f8bf92e78b5575fe2d15eebaab2f8537ddeb Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 21 Feb 2017 16:33:11 +0000 Subject: [PATCH 89/99] missing include --- Code/debug/linux/LinuxDebugger.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/Code/debug/linux/LinuxDebugger.cc b/Code/debug/linux/LinuxDebugger.cc index 269127154..ea9e2bbbb 100644 --- a/Code/debug/linux/LinuxDebugger.cc +++ b/Code/debug/linux/LinuxDebugger.cc @@ -7,6 +7,7 @@ #include #include "debug/linux/LinuxDebugger.h" +#include "Exception.h" #include #include From 2801a6d25d13d55daedab55b26038a45c0de3980 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 21 Feb 2017 16:50:47 +0000 Subject: [PATCH 90/99] missing include --- Code/unittests/helpers/MockCommunicator.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Code/unittests/helpers/MockCommunicator.h b/Code/unittests/helpers/MockCommunicator.h index a1a3bbb65..dc36dc023 100644 --- a/Code/unittests/helpers/MockCommunicator.h +++ b/Code/unittests/helpers/MockCommunicator.h @@ -8,6 +8,8 @@ #define HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATOR_H #include +#include + #include "comm/Communicator.h" #include "comm/Request.h" From ffbc38fc4deace6a9dddab13de50b4df2915cb0c Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Thu, 23 Feb 2017 12:08:53 +0000 Subject: [PATCH 91/99] remove empty StabiltiyTester.cc --- Code/lb/CMakeLists.txt | 2 +- Code/lb/StabilityTester.cc | 17 ----------------- 2 files changed, 1 insertion(+), 18 deletions(-) delete mode 100644 Code/lb/StabilityTester.cc diff --git a/Code/lb/CMakeLists.txt b/Code/lb/CMakeLists.txt index 2f2addf00..4d38a949e 100644 --- a/Code/lb/CMakeLists.txt +++ b/Code/lb/CMakeLists.txt @@ -15,5 +15,5 @@ add_library(hemelb_lb kernels/rheologyModels/AbstractRheologyModel.cc kernels/rheologyModels/CarreauYasudaRheologyModel.cc kernels/rheologyModels/CassonRheologyModel.cc kernels/rheologyModels/TruncatedPowerLawRheologyModel.cc lattices/LatticeInfo.cc lattices/D3Q15.cc lattices/D3Q19.cc lattices/D3Q27.cc lattices/D3Q15i.cc - MacroscopicPropertyCache.cc SimulationState.cc StabilityTester.cc + MacroscopicPropertyCache.cc SimulationState.cc ) diff --git a/Code/lb/StabilityTester.cc b/Code/lb/StabilityTester.cc deleted file mode 100644 index 29b890cf7..000000000 --- a/Code/lb/StabilityTester.cc +++ /dev/null @@ -1,17 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "lb/StabilityTester.h" -#include "lb/LbmParameters.h" - -namespace hemelb -{ - namespace lb - { - - - } -} From 072e2acfa122c3b48f98c836783350cfdc958771 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 28 Feb 2017 15:56:18 +0000 Subject: [PATCH 92/99] ARCHER/fabric tweaks --- deploy/machines.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/deploy/machines.yml b/deploy/machines.yml index 1b5448f9a..54649e79b 100644 --- a/deploy/machines.yml +++ b/deploy/machines.yml @@ -62,7 +62,9 @@ archer: #the ARCHER supercomputer at EPCC make_jobs: 4 verbose: 1 job_dispatch: "qsub" - run_command: "aprun -n $cores -N $corespernode" + # -n : total number of MPI ranks + # -N : MPI ranks per node (must be <= n) default is min(n, 24) + run_command: "aprun -n $cores" batch_header: pbs-archer remote: "login.archer.ac.uk" # On ARCHER, *all files* which are needed at runtime, must be on the /work filesystem, so we must make the install location be on the /work filesystem @@ -70,7 +72,7 @@ archer: #the ARCHER supercomputer at EPCC home_path_template: "/home/$project/$project/$username" runtime_path_template: "/work/$project/$project/$username" fabric_dir: "FabricHemeLb" - modules: ["load cmake/3.2.3", "swap PrgEnv-cray PrgEnv-gnu", "load boost", "load cray-hdf5-parallel", "load cray-tpsl"] + modules: ["load cmake/3.2.3", "swap PrgEnv-cray PrgEnv-gnu", "load boost", "load cray-hdf5-parallel", "load cray-tpsl", "load anaconda/2.2.0-python2"] # Tell autoconf for dependencies where the compilers are build_prefix_commands: ["export LDFLAGS=-dynamic","export CXX=CC","export CC=cc", "export LD=CC", "export XTPE_LINK_TYPE=dynamic" ] temp_path_template: "$work_path/tmp" From 938fd04db78d479391b05d275c1cbbd97c450cca Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Tue, 14 Mar 2017 17:07:54 +0000 Subject: [PATCH 93/99] further tidy the cmake/fabric and ensure correct MPI is found on ARCHER --- dependencies/Modules/FindMPI.cmake | 622 ----------------------------- deploy/fab.py | 3 +- deploy/machines.yml | 14 +- 3 files changed, 5 insertions(+), 634 deletions(-) delete mode 100644 dependencies/Modules/FindMPI.cmake diff --git a/dependencies/Modules/FindMPI.cmake b/dependencies/Modules/FindMPI.cmake deleted file mode 100644 index d7436af3b..000000000 --- a/dependencies/Modules/FindMPI.cmake +++ /dev/null @@ -1,622 +0,0 @@ - -# This file is part of HemeLB and is Copyright (C) -# the HemeLB team and/or their institutions, as detailed in the -# file AUTHORS. This software is provided under the terms of the -# license in the file LICENSE. -# - Find a Message Passing Interface (MPI) implementation -# The Message Passing Interface (MPI) is a library used to write -# high-performance distributed-memory parallel applications, and -# is typically deployed on a cluster. MPI is a standard interface -# (defined by the MPI forum) for which many implementations are -# available. All of them have somewhat different include paths, -# libraries to link against, etc., and this module tries to smooth -# out those differences. -# -# === Variables === -# -# This module will set the following variables per language in your project, -# where is one of C, CXX, or Fortran: -# MPI__FOUND TRUE if FindMPI found MPI flags for -# MPI__COMPILER MPI Compiler wrapper for -# MPI__COMPILE_FLAGS Compilation flags for MPI programs -# MPI__INCLUDE_PATH Include path(s) for MPI header -# MPI__LINK_FLAGS Linking flags for MPI programs -# MPI__LIBRARIES All libraries to link MPI programs against -# Additionally, FindMPI sets the following variables for running MPI -# programs from the command line: -# MPIEXEC Executable for running MPI programs -# MPIEXEC_NUMPROC_FLAG Flag to pass to MPIEXEC before giving -# it the number of processors to run on -# MPIEXEC_PREFLAGS Flags to pass to MPIEXEC directly -# before the executable to run. -# MPIEXEC_POSTFLAGS Flags to pass to MPIEXEC after other flags -# === Usage === -# -# To use this module, simply call FindMPI from a CMakeLists.txt file, or -# run find_package(MPI), then run CMake. If you are happy with the auto- -# detected configuration for your language, then you're done. If not, you -# have two options: -# 1. Set MPI__COMPILER to the MPI wrapper (mpicc, etc.) of your -# choice and reconfigure. FindMPI will attempt to determine all the -# necessary variables using THAT compiler's compile and link flags. -# 2. If this fails, or if your MPI implementation does not come with -# a compiler wrapper, then set both MPI__LIBRARIES and -# MPI__INCLUDE_PATH. You may also set any other variables -# listed above, but these two are required. This will circumvent -# autodetection entirely. -# When configuration is successful, MPI__COMPILER will be set to the -# compiler wrapper for , if it was found. MPI__FOUND and other -# variables above will be set if any MPI implementation was found for , -# regardless of whether a compiler was found. -# -# When using MPIEXEC to execute MPI applications, you should typically use -# all of the MPIEXEC flags as follows: -# ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} PROCS -# ${MPIEXEC_PREFLAGS} EXECUTABLE ${MPIEXEC_POSTFLAGS} ARGS -# where PROCS is the number of processors on which to execute the program, -# EXECUTABLE is the MPI program, and ARGS are the arguments to pass to the -# MPI program. -# -# === Backward Compatibility === -# -# For backward compatibility with older versions of FindMPI, these -# variables are set, but deprecated: -# MPI_FOUND MPI_COMPILER MPI_LIBRARY -# MPI_COMPILE_FLAGS MPI_INCLUDE_PATH MPI_EXTRA_LIBRARY -# MPI_LINK_FLAGS MPI_LIBRARIES -# In new projects, please use the MPI__XXX equivalents. - -#============================================================================= -# Copyright 2001-2011 Kitware, Inc. -# Copyright 2010-2011 Todd Gamblin tgamblin@llnl.gov -# Copyright 2001-2009 Dave Partyka -# -# Distributed under the OSI-approved BSD License (the "License"); -# see accompanying file Copyright.txt for details. -# -# This software is distributed WITHOUT ANY WARRANTY; without even the -# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the License for more information. -#============================================================================= -# (To distribute this file outside of CMake, substitute the full -# License text for the above reference.) - -# include this to handle the QUIETLY and REQUIRED arguments -include(FindPackageHandleStandardArgs) # Modified by JH -include(GetPrerequisites) - -# -# This part detects MPI compilers, attempting to wade through the mess of compiler names in -# a sensible way. -# -# The compilers are detected in this order: -# -# 1. Try to find the most generic availble MPI compiler, as this is usually set up by -# cluster admins. e.g., if plain old mpicc is available, we'll use it and assume it's -# the right compiler. -# -# 2. If a generic mpicc is NOT found, then we attempt to find one that matches -# CMAKE__COMPILER_ID. e.g. if you are using XL compilers, we'll try to find mpixlc -# and company, but not mpiicc. This hopefully prevents toolchain mismatches. -# -# If you want to force a particular MPI compiler other than what we autodetect (e.g. if you -# want to compile regular stuff with GNU and parallel stuff with Intel), you can always set -# your favorite MPI__COMPILER explicitly and this stuff will be ignored. -# - -# Start out with the generic MPI compiler names, as these are most commonly used. -set(_MPI_C_COMPILER_NAMES mpicc mpcc mpicc_r mpcc_r) -set(_MPI_CXX_COMPILER_NAMES mpicxx mpiCC mpcxx mpCC mpic++ mpc++ - mpicxx_r mpiCC_r mpcxx_r mpCC_r mpic++_r mpc++_r) -set(_MPI_Fortran_COMPILER_NAMES mpif95 mpif95_r mpf95 mpf95_r - mpif90 mpif90_r mpf90 mpf90_r - mpif77 mpif77_r mpf77 mpf77_r) - -# GNU compiler names -set(_MPI_GNU_C_COMPILER_NAMES mpigcc mpgcc mpigcc_r mpgcc_r) -set(_MPI_GNU_CXX_COMPILER_NAMES mpig++ mpg++ mpig++_r mpg++_r) -set(_MPI_GNU_Fortran_COMPILER_NAMES mpigfortran mpgfortran mpigfortran_r mpgfortran_r - mpig77 mpig77_r mpg77 mpg77_r) - -# Intel MPI compiler names -set(_MPI_Intel_C_COMPILER_NAMES mpiicc) -set(_MPI_Intel_CXX_COMPILER_NAMES mpiicpc mpiicxx mpiic++ mpiiCC) -set(_MPI_Intel_Fortran_COMPILER_NAMES mpiifort mpiif95 mpiif90 mpiif77) - -# PGI compiler names -set(_MPI_PGI_C_COMPILER_NAMES mpipgcc mppgcc) -set(_MPI_PGI_CXX_COMPILER_NAMES mpipgCC mppgCC) -set(_MPI_PGI_Fortran_COMPILER_NAMES mpipgf95 mpipgf90 mppgf95 mppgf90 mpipgf77 mppgf77) - -# XLC MPI Compiler names -set(_MPI_XL_C_COMPILER_NAMES mpxlc mpxlc_r mpixlc mpixlc_r) -set(_MPI_XL_CXX_COMPILER_NAMES mpixlcxx mpixlC mpixlc++ mpxlcxx mpxlc++ mpixlc++ mpxlCC - mpixlcxx_r mpixlC_r mpixlc++_r mpxlcxx_r mpxlc++_r mpixlc++_r mpxlCC_r) -set(_MPI_XL_Fortran_COMPILER_NAMES mpixlf95 mpixlf95_r mpxlf95 mpxlf95_r - mpixlf90 mpixlf90_r mpxlf90 mpxlf90_r - mpixlf77 mpixlf77_r mpxlf77 mpxlf77_r - mpixlf mpixlf_r mpxlf mpxlf_r) - -# append vendor-specific compilers to the list if we either don't know the compiler id, -# or if we know it matches the regular compiler. -foreach (lang C CXX Fortran) - foreach (id GNU Intel PGI XL) - if (NOT CMAKE_${lang}_COMPILER_ID OR "${CMAKE_${lang}_COMPILER_ID}" STREQUAL "${id}") - list(APPEND _MPI_${lang}_COMPILER_NAMES ${_MPI_${id}_${lang}_COMPILER_NAMES}) - endif() - unset(_MPI_${id}_${lang}_COMPILER_NAMES) # clean up the namespace here - endforeach() -endforeach() - - -# Names to try for MPI exec -set(_MPI_EXEC_NAMES mpiexec mpirun lamexec srun) - -# Grab the path to MPI from the registry if we're on windows. -set(_MPI_PREFIX_PATH) -if(WIN32) - list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH\\SMPD;binary]/..") - list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH2;Path]") - list(APPEND _MPI_PREFIX_PATH "$ENV{ProgramW6432}/MPICH2/") -endif() - -# Build a list of prefixes to search for MPI. -foreach(SystemPrefixDir ${CMAKE_SYSTEM_PREFIX_PATH}) - foreach(MpiPackageDir ${_MPI_PREFIX_PATH}) - if(EXISTS ${SystemPrefixDir}/${MpiPackageDir}) - list(APPEND _MPI_PREFIX_PATH "${SystemPrefixDir}/${MpiPackageDir}") - endif() - endforeach() -endforeach() - - -# -# interrogate_mpi_compiler(lang try_libs) -# -# Attempts to extract compiler and linker args from an MPI compiler. The arguments set -# by this function are: -# -# MPI__INCLUDE_PATH MPI__LINK_FLAGS MPI__FOUND -# MPI__COMPILE_FLAGS MPI__LIBRARIES -# -# MPI__COMPILER must be set beforehand to the absolute path to an MPI compiler for -# . Additionally, MPI__INCLUDE_PATH and MPI__LIBRARIES may be set -# to skip autodetection. -# -# If try_libs is TRUE, this will also attempt to find plain MPI libraries in the usual -# way. In general, this is not as effective as interrogating the compilers, as it -# ignores language-specific flags and libraries. However, some MPI implementations -# (Windows implementations) do not have compiler wrappers, so this approach must be used. -# -function (interrogate_mpi_compiler lang try_libs) - # MPI_${lang}_NO_INTERROGATE will be set to a compiler name when the *regular* compiler was - # discovered to be the MPI compiler. This happens on machines like the Cray XE6 that use - # modules to set cc, CC, and ftn to the MPI compilers. If the user force-sets another MPI - # compiler, MPI_${lang}_COMPILER won't be equal to MPI_${lang}_NO_INTERROGATE, and we'll - # inspect that compiler anew. This allows users to set new compilers w/o rm'ing cache. - string(COMPARE NOTEQUAL "${MPI_${lang}_NO_INTERROGATE}" "${MPI_${lang}_COMPILER}" interrogate) - - # If MPI is set already in the cache, don't bother with interrogating the compiler. - if (interrogate AND ((NOT MPI_${lang}_INCLUDE_PATH) OR (NOT MPI_${lang}_LIBRARIES))) - if (MPI_${lang}_COMPILER) - # Check whether the -showme:compile option works. This indicates that we have either OpenMPI - # or a newer version of LAM-MPI, and implies that -showme:link will also work. - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -showme:compile - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_COMPILE_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - - if (MPI_COMPILER_RETURN EQUAL 0) - # If we appear to have -showme:compile, then we should - # also have -showme:link. Try it. - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -showme:link - OUTPUT_VARIABLE MPI_LINK_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_LINK_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - - if (MPI_COMPILER_RETURN EQUAL 0) - # We probably have -showme:incdirs and -showme:libdirs as well, - # so grab that while we're at it. - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -showme:incdirs - OUTPUT_VARIABLE MPI_INCDIRS OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_INCDIRS ERROR_STRIP_TRAILING_WHITESPACE) - - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -showme:libdirs - OUTPUT_VARIABLE MPI_LIBDIRS OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_LIBDIRS ERROR_STRIP_TRAILING_WHITESPACE) - - else() - # reset things here if something went wrong. - set(MPI_COMPILE_CMDLINE) - set(MPI_LINK_CMDLINE) - endif() - endif () - - # Older versions of LAM-MPI have "-showme". Try to find that. - if (NOT MPI_COMPILER_RETURN EQUAL 0) - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -showme - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_COMPILE_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - endif() - - # MVAPICH uses -compile-info and -link-info. Try them. - if (NOT MPI_COMPILER_RETURN EQUAL 0) - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -compile-info - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_COMPILE_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - - # If we have compile-info, also have link-info. - if (MPI_COMPILER_RETURN EQUAL 0) - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -link-info - OUTPUT_VARIABLE MPI_LINK_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_LINK_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - endif() - - # make sure we got compile and link. Reset vars if something's wrong. - if (NOT MPI_COMPILER_RETURN EQUAL 0) - set(MPI_COMPILE_CMDLINE) - set(MPI_LINK_CMDLINE) - endif() - endif() - - # MPICH just uses "-show". Try it. - if (NOT MPI_COMPILER_RETURN EQUAL 0) - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -show - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_COMPILE_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - endif() - - if (MPI_COMPILER_RETURN EQUAL 0) - # We have our command lines, but we might need to copy MPI_COMPILE_CMDLINE - # into MPI_LINK_CMDLINE, if we didn't find the link line. - if (NOT MPI_LINK_CMDLINE) - set(MPI_LINK_CMDLINE ${MPI_COMPILE_CMDLINE}) - endif() - else() - message(STATUS "Unable to determine MPI from MPI driver ${MPI_${lang}_COMPILER}") - set(MPI_COMPILE_CMDLINE) - set(MPI_LINK_CMDLINE) - endif() - - # Here, we're done with the interrogation part, and we'll try to extract args we care - # about from what we learned from the compiler wrapper scripts. - - # If interrogation came back with something, extract our variable from the MPI command line - if (MPI_COMPILE_CMDLINE OR MPI_LINK_CMDLINE) - # Extract compile flags from the compile command line. - string(REGEX MATCHALL "(^| )-[Df]([^\" ]+|\"[^\"]+\")" MPI_ALL_COMPILE_FLAGS "${MPI_COMPILE_CMDLINE}") - set(MPI_COMPILE_FLAGS_WORK) - - foreach(FLAG ${MPI_ALL_COMPILE_FLAGS}) - if (MPI_COMPILE_FLAGS_WORK) - set(MPI_COMPILE_FLAGS_WORK "${MPI_COMPILE_FLAGS_WORK} ${FLAG}") - else() - set(MPI_COMPILE_FLAGS_WORK ${FLAG}) - endif() - endforeach() - - # Extract include paths from compile command line - string(REGEX MATCHALL "(^| )-I([^\" ]+|\"[^\"]+\")" MPI_ALL_INCLUDE_PATHS "${MPI_COMPILE_CMDLINE}") - foreach(IPATH ${MPI_ALL_INCLUDE_PATHS}) - string(REGEX REPLACE "^ ?-I" "" IPATH ${IPATH}) - string(REGEX REPLACE "//" "/" IPATH ${IPATH}) - list(APPEND MPI_INCLUDE_PATH_WORK ${IPATH}) - endforeach() - - # try using showme:incdirs if extracting didn't work. - if (NOT MPI_INCLUDE_PATH_WORK) - set(MPI_INCLUDE_PATH_WORK ${MPI_INCDIRS}) - separate_arguments(MPI_INCLUDE_PATH_WORK) - endif() - - # If all else fails, just search for mpi.h in the normal include paths. - if (NOT MPI_INCLUDE_PATH_WORK) - set(MPI_HEADER_PATH "MPI_HEADER_PATH-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - find_path(MPI_HEADER_PATH mpi.h - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES include) - set(MPI_INCLUDE_PATH_WORK ${MPI_HEADER_PATH}) - endif() - - # Extract linker paths from the link command line - string(REGEX MATCHALL "(^| |-Wl,)-L([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_PATHS "${MPI_LINK_CMDLINE}") - set(MPI_LINK_PATH) - foreach(LPATH ${MPI_ALL_LINK_PATHS}) - string(REGEX REPLACE "^(| |-Wl,)-L" "" LPATH ${LPATH}) - string(REGEX REPLACE "//" "/" LPATH ${LPATH}) - list(APPEND MPI_LINK_PATH ${LPATH}) - endforeach() - - # try using showme:libdirs if extracting didn't work. - if (NOT MPI_LINK_PATH) - set(MPI_LINK_PATH ${MPI_LIBDIRS}) - separate_arguments(MPI_LINK_PATH) - endif() - - # Extract linker flags from the link command line - string(REGEX MATCHALL "(^| )-Wl,([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_FLAGS "${MPI_LINK_CMDLINE}") - set(MPI_LINK_FLAGS_WORK) - foreach(FLAG ${MPI_ALL_LINK_FLAGS}) - if (MPI_LINK_FLAGS_WORK) - set(MPI_LINK_FLAGS_WORK "${MPI_LINK_FLAGS_WORK} ${FLAG}") - else() - set(MPI_LINK_FLAGS_WORK ${FLAG}) - endif() - endforeach() - - # Extract the set of libraries to link against from the link command - # line - string(REGEX MATCHALL "(^| )-l([^\" ]+|\"[^\"]+\")" MPI_LIBNAMES "${MPI_LINK_CMDLINE}") - - # Determine full path names for all of the libraries that one needs - # to link against in an MPI program - foreach(LIB ${MPI_LIBNAMES}) - string(REGEX REPLACE "^ ?-l" "" LIB ${LIB}) - # MPI_LIB is cached by find_library, but we don't want that. Clear it first. - set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - find_library(MPI_LIB NAMES ${LIB} HINTS ${MPI_LINK_PATH}) - - if (MPI_LIB) - list(APPEND MPI_LIBRARIES_WORK ${MPI_LIB}) - elseif (NOT MPI_FIND_QUIETLY) - message(WARNING "Unable to find MPI library ${LIB}") - endif() - endforeach() - - # Sanity check MPI_LIBRARIES to make sure there are enough libraries - list(LENGTH MPI_LIBRARIES_WORK MPI_NUMLIBS) - list(LENGTH MPI_LIBNAMES MPI_NUMLIBS_EXPECTED) - if (NOT MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED) - set(MPI_LIBRARIES_WORK "MPI_${lang}_LIBRARIES-NOTFOUND") - endif() - endif() - - elseif(try_libs) - # If we didn't have an MPI compiler script to interrogate, attempt to find everything - # with plain old find functions. This is nasty because MPI implementations have LOTS of - # different library names, so this section isn't going to be very generic. We need to - # make sure it works for MS MPI, though, since there are no compiler wrappers for that. - find_path(MPI_HEADER_PATH mpi.h - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES include Inc) - set(MPI_INCLUDE_PATH_WORK ${MPI_HEADER_PATH}) - - # Decide between 32-bit and 64-bit libraries for Microsoft's MPI - if("${CMAKE_SIZEOF_VOID_P}" EQUAL 8) - set(MS_MPI_ARCH_DIR amd64) - else() - set(MS_MPI_ARCH_DIR i386) - endif() - - set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - find_library(MPI_LIB - NAMES mpi mpich mpich2 msmpi - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES lib lib/${MS_MPI_ARCH_DIR} Lib Lib/${MS_MPI_ARCH_DIR}) - set(MPI_LIBRARIES_WORK ${MPI_LIB}) - - # Right now, we only know about the extra libs for C++. - # We could add Fortran here (as there is usually libfmpich, etc.), but - # this really only has to work with MS MPI on Windows. - # Assume that other MPI's are covered by the compiler wrappers. - if (${lang} STREQUAL CXX) - set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - find_library(MPI_LIB - NAMES mpi++ mpicxx cxx mpi_cxx - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES lib) - if (MPI_LIBRARIES_WORK AND MPI_LIB) - set(MPI_LIBRARIES_WORK "${MPI_LIBRARIES_WORK} ${MPI_LIB}") - endif() - endif() - - if (NOT MPI_LIBRARIES_WORK) - set(MPI_LIBRARIES_WORK "MPI_${lang}_LIBRARIES-NOTFOUND") - endif() - endif() - - # If we found MPI, set up all of the appropriate cache entries - set(MPI_${lang}_COMPILE_FLAGS ${MPI_COMPILE_FLAGS_WORK} CACHE STRING "MPI ${lang} compilation flags" FORCE) - set(MPI_${lang}_INCLUDE_PATH ${MPI_INCLUDE_PATH_WORK} CACHE STRING "MPI ${lang} include path" FORCE) - set(MPI_${lang}_LINK_FLAGS ${MPI_LINK_FLAGS_WORK} CACHE STRING "MPI ${lang} linking flags" FORCE) - set(MPI_${lang}_LIBRARIES ${MPI_LIBRARIES_WORK} CACHE STRING "MPI ${lang} libraries to link against" FORCE) - mark_as_advanced(MPI_${lang}_COMPILE_FLAGS MPI_${lang}_INCLUDE_PATH MPI_${lang}_LINK_FLAGS MPI_${lang}_LIBRARIES) - - # clear out our temporary lib/header detectionv variable here. - set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI lib detection" FORCE) - set(MPI_HEADER_PATH "MPI_HEADER_PATH-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI header detection" FORCE) - endif() - - # finally set a found variable for each MPI language - if (MPI_${lang}_INCLUDE_PATH AND MPI_${lang}_LIBRARIES) - set(MPI_${lang}_FOUND TRUE PARENT_SCOPE) - else() - set(MPI_${lang}_FOUND FALSE PARENT_SCOPE) - endif() -endfunction() - - -# This function attempts to compile with the regular compiler, to see if MPI programs -# work with it. This is a last ditch attempt after we've tried interrogating mpicc and -# friends, and after we've tried to find generic libraries. Works on machines like -# Cray XE6, where the modules environment changes what MPI version cc, CC, and ftn use. -function(try_regular_compiler lang success) - set(scratch_directory ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}) - if (${lang} STREQUAL Fortran) - set(test_file ${scratch_directory}/cmake_mpi_test.f90) - file(WRITE ${test_file} - "program hello\n" - "include 'mpif.h'\n" - "integer ierror\n" - "call MPI_INIT(ierror)\n" - "call MPI_FINALIZE(ierror)\n" - "end\n") - else() - if (${lang} STREQUAL CXX) - set(test_file ${scratch_directory}/cmake_mpi_test.cpp) - else() - set(test_file ${scratch_directory}/cmake_mpi_test.c) - endif() - file(WRITE ${test_file} - "#include \n" - "int main(int argc, char **argv) {\n" - " MPI_Init(&argc, &argv);\n" - " MPI_Finalize();\n" - "}\n") - endif() - try_compile(compiler_has_mpi ${scratch_directory} ${test_file}) - if (compiler_has_mpi) - set(MPI_${lang}_NO_INTERROGATE ${CMAKE_${lang}_COMPILER} CACHE STRING "Whether to interrogate MPI ${lang} compiler" FORCE) - set(MPI_${lang}_COMPILER ${CMAKE_${lang}_COMPILER} CACHE STRING "MPI ${lang} compiler" FORCE) - set(MPI_${lang}_COMPILE_FLAGS "" CACHE STRING "MPI ${lang} compilation flags" FORCE) - set(MPI_${lang}_INCLUDE_PATH "" CACHE STRING "MPI ${lang} include path" FORCE) - set(MPI_${lang}_LINK_FLAGS "" CACHE STRING "MPI ${lang} linking flags" FORCE) - set(MPI_${lang}_LIBRARIES "" CACHE STRING "MPI ${lang} libraries to link against" FORCE) - endif() - set(${success} ${compiler_has_mpi} PARENT_SCOPE) - unset(compiler_has_mpi CACHE) -endfunction() - -# End definitions, commence real work here. - -# Most mpi distros have some form of mpiexec which gives us something we can reliably look for. -find_program(MPIEXEC - NAMES ${_MPI_EXEC_NAMES} - PATHS ${_MPI_PREFIX_PATH} - PATH_SUFFIXES bin - DOC "Executable for running MPI programs.") - -# call get_filename_component twice to remove mpiexec and the directory it exists in (typically bin). -# This gives us a fairly reliable base directory to search for /bin /lib and /include from. -get_filename_component(_MPI_BASE_DIR "${MPIEXEC}" PATH) -get_filename_component(_MPI_BASE_DIR "${_MPI_BASE_DIR}" PATH) - -set(MPIEXEC_NUMPROC_FLAG "-np" CACHE STRING "Flag used by MPI to specify the number of processes for MPIEXEC; the next option will be the number of processes.") -set(MPIEXEC_PREFLAGS "" CACHE STRING "These flags will be directly before the executable that is being run by MPIEXEC.") -set(MPIEXEC_POSTFLAGS "" CACHE STRING "These flags will come after all flags given to MPIEXEC.") -set(MPIEXEC_MAX_NUMPROCS "2" CACHE STRING "Maximum number of processors available to run MPI applications.") -mark_as_advanced(MPIEXEC MPIEXEC_NUMPROC_FLAG MPIEXEC_PREFLAGS MPIEXEC_POSTFLAGS MPIEXEC_MAX_NUMPROCS) - - -#============================================================================= -# Backward compatibility input hacks. Propagate the FindMPI hints to C and -# CXX if the respective new versions are not defined. Translate the old -# MPI_LIBRARY and MPI_EXTRA_LIBRARY to respective MPI_${lang}_LIBRARIES. -# -# Once we find the new variables, we translate them back into their old -# equivalents below. -foreach (lang C CXX) - # Old input variables. - set(_MPI_OLD_INPUT_VARS COMPILER COMPILE_FLAGS INCLUDE_PATH LINK_FLAGS) - - # Set new vars based on their old equivalents, if the new versions are not already set. - foreach (var ${_MPI_OLD_INPUT_VARS}) - if (NOT MPI_${lang}_${var} AND MPI_${var}) - set(MPI_${lang}_${var} "${MPI_${var}}") - endif() - endforeach() - - # Special handling for MPI_LIBRARY and MPI_EXTRA_LIBRARY, which we nixed in the - # new FindMPI. These need to be merged into MPI__LIBRARIES - if (NOT MPI_${lang}_LIBRARIES AND (MPI_LIBRARY OR MPI_EXTRA_LIBRARY)) - set(MPI_${lang}_LIBRARIES ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY}) - endif() -endforeach() -#============================================================================= - - -# This loop finds the compilers and sends them off for interrogation. -foreach (lang C CXX Fortran) - if (CMAKE_${lang}_COMPILER_WORKS) - # If the user supplies a compiler *name* instead of an absolute path, assume that we need to find THAT compiler. - if (MPI_${lang}_COMPILER) - is_file_executable(MPI_${lang}_COMPILER MPI_COMPILER_IS_EXECUTABLE) - if (NOT MPI_COMPILER_IS_EXECUTABLE) - # Get rid of our default list of names and just search for the name the user wants. - set(_MPI_${lang}_COMPILER_NAMES ${MPI_${lang}_COMPILER}) - set(MPI_${lang}_COMPILER "MPI_${lang}_COMPILER-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - # If the user specifies a compiler, we don't want to try to search libraries either. - set(try_libs FALSE) - endif() - else() - set(try_libs TRUE) - endif() - - find_program(MPI_${lang}_COMPILER - NAMES ${_MPI_${lang}_COMPILER_NAMES} - PATHS "${MPI_HOME}/bin" "$ENV{MPI_HOME}/bin" ${_MPI_PREFIX_PATH}) - interrogate_mpi_compiler(${lang} ${try_libs}) - mark_as_advanced(MPI_${lang}_COMPILER) - - # last ditch try -- if nothing works so far, just try running the regular compiler and - # see if we can create an MPI executable. - set(regular_compiler_worked 0) - if (NOT MPI_${lang}_LIBRARIES OR NOT MPI_${lang}_INCLUDE_PATH) - try_regular_compiler(${lang} regular_compiler_worked) - endif() - - if (regular_compiler_worked) - find_package_handle_standard_args(MPI_${lang} DEFAULT_MSG MPI_${lang}_COMPILER) - else() - find_package_handle_standard_args(MPI_${lang} DEFAULT_MSG MPI_${lang}_LIBRARIES MPI_${lang}_INCLUDE_PATH) - endif() - endif() -endforeach() - - -#============================================================================= -# More backward compatibility stuff -# -# Bare MPI sans ${lang} vars are set to CXX then C, depending on what was found. -# This mimics the behavior of the old language-oblivious FindMPI. -set(_MPI_OLD_VARS FOUND COMPILER INCLUDE_PATH COMPILE_FLAGS LINK_FLAGS LIBRARIES) -if (MPI_CXX_FOUND) - foreach (var ${_MPI_OLD_VARS}) - set(MPI_${var} ${MPI_CXX_${var}}) - endforeach() -elseif (MPI_C_FOUND) - foreach (var ${_MPI_OLD_VARS}) - set(MPI_${var} ${MPI_C_${var}}) - endforeach() -else() - # Note that we might still have found Fortran, but you'll need to use MPI_Fortran_FOUND - set(MPI_FOUND FALSE) -endif() - -# Chop MPI_LIBRARIES into the old-style MPI_LIBRARY and MPI_EXTRA_LIBRARY, and set them in cache. -if (MPI_LIBRARIES) - list(GET MPI_LIBRARIES 0 MPI_LIBRARY_WORK) - set(MPI_LIBRARY ${MPI_LIBRARY_WORK} CACHE FILEPATH "MPI library to link against" FORCE) -else() - set(MPI_LIBRARY "MPI_LIBRARY-NOTFOUND" CACHE FILEPATH "MPI library to link against" FORCE) -endif() - -list(LENGTH MPI_LIBRARIES MPI_NUMLIBS) -if (MPI_NUMLIBS GREATER 1) - set(MPI_EXTRA_LIBRARY_WORK ${MPI_LIBRARIES}) - list(REMOVE_AT MPI_EXTRA_LIBRARY_WORK 0) - set(MPI_EXTRA_LIBRARY ${MPI_EXTRA_LIBRARY_WORK} CACHE STRING "Extra MPI libraries to link against" FORCE) -else() - set(MPI_EXTRA_LIBRARY "MPI_EXTRA_LIBRARY-NOTFOUND" CACHE STRING "Extra MPI libraries to link against" FORCE) -endif() -#============================================================================= - -# unset these vars to cleanup namespace -unset(_MPI_OLD_VARS) -unset(_MPI_PREFIX_PATH) -unset(_MPI_BASE_DIR) -foreach (lang C CXX Fortran) - unset(_MPI_${lang}_COMPILER_NAMES) -endforeach() diff --git a/deploy/fab.py b/deploy/fab.py index c1512d540..e0d2dfa93 100644 --- a/deploy/fab.py +++ b/deploy/fab.py @@ -160,8 +160,7 @@ def configure_cmake(configurations, extras): options.update(extras) options.update(env.cmake_options) options.update({'CMAKE_INSTALL_PREFIX':env.install_path, - "HEMELB_DEPENDENCIES_INSTALL_PATH":env.install_path, - "HEMELB_SUBPROJECT_MAKE_JOBS":env.make_jobs}) + "HEMELB_DEPENDENCIES_INSTALL_PATH":env.install_path}) env.total_cmake_options = options env.cmake_flags = ' '.join(["-D%s=%s" % option for option in env.total_cmake_options.iteritems()]) diff --git a/deploy/machines.yml b/deploy/machines.yml index 54649e79b..27087f21a 100644 --- a/deploy/machines.yml +++ b/deploy/machines.yml @@ -93,18 +93,15 @@ archer: #the ARCHER supercomputer at EPCC CMAKE_CXX_COMPILER: "CC" CMAKE_C_COMPILER: "cc" CMAKE_CXX_FLAGS_RELEASE: "" - CPPUNIT_PATCH_LDL: OFF - CPPUNIT_PATCH_DYNAMIC: ON + MPI_CXX_COMPILER: "CC" + MPI_C_COMPILER: "cc" HEMELB_OPTIMISATION: "-O3" HEMELB_DEPENDENCIES_SET_RPATH: "OFF" HEMELB_USE_SSE3: "ON" - CTEMPLATE_PATCH_ALIGN: "ON" CTEMPLATE_USE_STATIC: "OFF" HEMELB_COMPUTE_ARCHITECTURE: "INTELSANDYBRIDGE" METIS_INCLUDE_DIR: "$CRAY_TPSL_PREFIX_DIR/include" - METIS_LIBRARY: "$CRAY_TPSL_PREFIX_DIR/lib" PARMETIS_INCLUDE_DIR: "$CRAY_TPSL_PREFIX_DIR/include" - PARMETIS_LIBRARY: "$CRAY_TPSL_PREFIX_DIR/lib" archer_dmapp: import: "archer" @@ -116,18 +113,15 @@ archer_dmapp: CMAKE_CXX_COMPILER: "CC" CMAKE_C_COMPILER: "cc" CMAKE_CXX_FLAGS_RELEASE: "" - CPPUNIT_PATCH_LDL: OFF - CPPUNIT_PATCH_DYNAMIC: ON + MPI_CXX_COMPILER: "CC" + MPI_C_COMPILER: "cc" HEMELB_OPTIMISATION: "-O3" HEMELB_DEPENDENCIES_SET_RPATH: "OFF" HEMELB_USE_SSE3: "ON" - CTEMPLATE_PATCH_ALIGN: "ON" CTEMPLATE_USE_STATIC: "OFF" HEMELB_COMPUTE_ARCHITECTURE: "INTELSANDYBRIDGE" METIS_INCLUDE_DIR: "$CRAY_TPSL_PREFIX_DIR/include" - METIS_LIBRARY: "$CRAY_TPSL_PREFIX_DIR/lib" PARMETIS_INCLUDE_DIR: "$CRAY_TPSL_PREFIX_DIR/include" - PARMETIS_LIBRARY: "$CRAY_TPSL_PREFIX_DIR/lib" run_prefix_commands: - export MPICH_MAX_THREAD_SAFETY=multiple - export MPICH_NEMESIS_ASYNC_PROGRESS=1 From ca9fe04e7a21d7b69aed2618c7dd2a75c776e815 Mon Sep 17 00:00:00 2001 From: Mayeul d'Avezac Date: Tue, 4 Apr 2017 08:14:16 +0100 Subject: [PATCH 94/99] MPI versions functions know about MPI and CXX flags --- Code/cmake/mpi.cmake | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Code/cmake/mpi.cmake b/Code/cmake/mpi.cmake index 102cd3cb2..f779af7b3 100644 --- a/Code/cmake/mpi.cmake +++ b/Code/cmake/mpi.cmake @@ -11,6 +11,9 @@ set(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} ${CMAKE_CXX_LINK_FLAGS}") include_directories(${MPI_INCLUDE_PATH}) function(TEST_MPI_VERSION_EQUAL ver output_var) + set(CMAKE_REQUIRED_FLAGS "${CXX11_FLAGS} ${CMAKE_CXX_COMPILE_FLAGS}") + set(CMAKE_REQUIRED_INCLUDES "${MPI_INCLUDE_PATH}") + set(CMAKE_REQUIRED_LIBRARIES "${MPI_CXX_LIBRARIES}") CHECK_CXX_SOURCE_COMPILES("#include int main(int argc, char* argv[]) { static_assert(MPI_VERSION == ${ver}, \"\"); @@ -19,6 +22,9 @@ int main(int argc, char* argv[]) { endfunction() function(TEST_MPI_SUBVERSION_EQUAL ver output_var) + set(CMAKE_REQUIRED_FLAGS "${CXX11_FLAGS} ${CMAKE_CXX_COMPILE_FLAGS}") + set(CMAKE_REQUIRED_INCLUDES "${MPI_INCLUDE_PATH}") + set(CMAKE_REQUIRED_LIBRARIES "${MPI_CXX_LIBRARIES}") CHECK_CXX_SOURCE_COMPILES("#include int main(int argc, char* argv[]) { static_assert(MPI_SUBVERSION == ${ver}, \"\"); From 6bff83c3042a5c11f0e9b7ccd89f12e8a0af7907 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 28 Apr 2017 14:49:14 +0100 Subject: [PATCH 95/99] revise mpi CMake in light of Mayeul's comments --- Code/cmake/mpi.cmake | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/Code/cmake/mpi.cmake b/Code/cmake/mpi.cmake index f779af7b3..c05799e86 100644 --- a/Code/cmake/mpi.cmake +++ b/Code/cmake/mpi.cmake @@ -55,21 +55,6 @@ function(GET_MPI_SUBVERSION output_var) message(FATAL_ERROR "Could not determine MPI_SUBVERSION") endfunction() -# CMake doens't seem to properly pass through the C++11 flag -# This is probably wrong -set(CMAKE_REQUIRED_FLAGS ${CXX11_FLAGS}) -set(CMAKE_REQUIRED_QUIET 1) - -# Put this in here as this is the key feature needed by the test programs -CHECK_CXX_SOURCE_COMPILES("int main(int, char**) { -static_assert(true, \"must not fail\"); -}" - HAVE_STATIC_ASSERT) - -if(NOT HAVE_STATIC_ASSERT) - message(FATAL_ERROR "No static_assert!") -endif() - GET_MPI_VERSION(MPI_STANDARD_VERSION_MAJOR) GET_MPI_SUBVERSION(MPI_STANDARD_VERSION_MINOR) SET(MPI_STANDARD_VERSION "${MPI_STANDARD_VERSION_MAJOR}.${MPI_STANDARD_VERSION_MINOR}") From 27a3c6f6631fafcbd09a504b6ff6fdb88388ac79 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 28 Apr 2017 20:07:24 +0100 Subject: [PATCH 96/99] Py3: move from file() to open() --- Tools/hemeTools/cache.py | 8 ++++---- Tools/hemeTools/parsers/extraction/__init__.py | 2 +- Tools/hemeTools/parsers/geometry/simple.py | 2 +- Tools/hemeTools/parsers/snapshot/Cfx.py | 6 +++--- Tools/hemeTools/parsers/snapshot/__init__.py | 12 ++++++------ 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Tools/hemeTools/cache.py b/Tools/hemeTools/cache.py index 8a999a448..baf6f8aa1 100644 --- a/Tools/hemeTools/cache.py +++ b/Tools/hemeTools/cache.py @@ -35,11 +35,11 @@ def cacher(*args, **kwargs): if os.path.exists(cachefile): - ans = cPickle.load(file(cachefile)) + ans = cPickle.load(open(cachefile)) else: ans = f(*args, **kwargs) cPickle.dump(ans, - file(cachefile, 'wb'), + open(cachefile, 'wb'), protocol=2) pass return ans @@ -63,11 +63,11 @@ def cacher(infile): if os.path.exists(cachefile) and \ os.path.getmtime(cachefile) > os.path.getmtime(infile): # Is the cachefile newer than the input? - ans = cPickle.load(file(cachefile)) + ans = cPickle.load(open(cachefile)) else: ans = f(infile) cPickle.dump(ans, - file(cachefile, 'wb'), + open(cachefile, 'wb'), protocol=2) pass return ans diff --git a/Tools/hemeTools/parsers/extraction/__init__.py b/Tools/hemeTools/parsers/extraction/__init__.py index 9a544b10f..725683add 100644 --- a/Tools/hemeTools/parsers/extraction/__init__.py +++ b/Tools/hemeTools/parsers/extraction/__init__.py @@ -144,7 +144,7 @@ def __init__(self, filename): """ self.filename = filename - self._file = file(filename, 'rb') + self._file = open(filename, 'rb') self._ReadMainHeader() self._ReadFieldHeader() diff --git a/Tools/hemeTools/parsers/geometry/simple.py b/Tools/hemeTools/parsers/geometry/simple.py index c3eefb840..2ed906aaf 100644 --- a/Tools/hemeTools/parsers/geometry/simple.py +++ b/Tools/hemeTools/parsers/geometry/simple.py @@ -65,7 +65,7 @@ def __init__(self, filename): self.Domain.Origin = np.array(oStr[1:-1].split(','), dtype=float) assert self.Domain.Origin.shape == (3,) - self.File = file(self.GmyFileName) + self.File = open(self.GmyFileName) return def Load(self): diff --git a/Tools/hemeTools/parsers/snapshot/Cfx.py b/Tools/hemeTools/parsers/snapshot/Cfx.py index 7552f1cff..d55a0ebef 100644 --- a/Tools/hemeTools/parsers/snapshot/Cfx.py +++ b/Tools/hemeTools/parsers/snapshot/Cfx.py @@ -1,4 +1,4 @@ - +\ # This file is part of HemeLB and is Copyright (C) # the HemeLB team and/or their institutions, as detailed in the # file AUTHORS. This software is provided under the terms of the @@ -103,7 +103,7 @@ def parseHeader(filename, AllData=False): '''Parses header to get data type for record array''' fieldRegEx = re.compile('(.+?)\s*\[\s*(.+?)\s*\]') - f = file(filename) + f = open(filename) for i in range(findStart(filename, AllData=AllData)+2): header = f.readline() continue @@ -163,7 +163,7 @@ def findStart(filename, AllData=False): if AllData == True: return -1 - for i, line in enumerate(file(filename)): + for i, line in enumerate(open(filename)): if line.find('[Data]')>=0: return i continue diff --git a/Tools/hemeTools/parsers/snapshot/__init__.py b/Tools/hemeTools/parsers/snapshot/__init__.py index 05a3a08d4..19ad28059 100644 --- a/Tools/hemeTools/parsers/snapshot/__init__.py +++ b/Tools/hemeTools/parsers/snapshot/__init__.py @@ -26,7 +26,7 @@ def HemeLbSnapshot(filename): numbers and more metadata. """ - start = file(filename).read(8) + start = open(filename).read(8) reader = xdrlib.Unpacker(start) firstInt = reader.unpack_uint() @@ -171,7 +171,7 @@ def _readHeader(cls, filename): """ - f = file(filename) + f = open(filename) stable = int(f.readline()) voxel_size = float(f.readline()) bb_min = np.array([int(x) for x in f.readline().split()]) @@ -198,7 +198,7 @@ class XdrVoxelFormatOneSnapshot(object): @classmethod def _load(cls, filename, header): # Skip past the header, slurp data, create XDR object - f = file(filename) + f = open(filename) f.seek(cls._headerLengthBytes) reader = xdrlib.Unpacker(f.read()) @@ -236,7 +236,7 @@ def _readHeader(cls, filename): 5- total number of fluid voxels """ - reader = xdrlib.Unpacker(file(filename).read(cls._headerLengthBytes)) + reader = xdrlib.Unpacker(open(filename).read(cls._headerLengthBytes)) header = {} header['stable'] = reader.unpack_int() header['voxel_size'] = reader.unpack_double() @@ -264,7 +264,7 @@ class XdrSnapshotVersionTwo(BaseSnapshot, XdrVoxelFormatOneSnapshot): def _readHeader(cls, filename): """Read the header lines, according to description in Code/io/formats/snapshot.h """ - reader = xdrlib.Unpacker(file(filename).read(cls._headerLengthBytes)) + reader = xdrlib.Unpacker(open(filename).read(cls._headerLengthBytes)) header = {} assert reader.unpack_uint() == HemeLbMagicNumber assert reader.unpack_uint() == SnapshotMagicNumber @@ -299,7 +299,7 @@ def VersionedXdrSnapshot(filename): """Examine the file and dispatch to the appropriate constructor. """ # Need the two magic numbers and the version number, i.e. 12 bytes - reader = xdrlib.Unpacker(file(filename).read(12)) + reader = xdrlib.Unpacker(open(filename).read(12)) assert reader.unpack_uint() == HemeLbMagicNumber assert reader.unpack_uint() == SnapshotMagicNumber From 927db5d0522d244c4b22355943ea5d1e6cfee299 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Fri, 28 Apr 2017 20:33:33 +0100 Subject: [PATCH 97/99] Py3: replace a few __builtins__.xrange calls --- Tools/hemeTools/parsers/extraction/__init__.py | 8 ++++---- Tools/hemeTools/utils/utils.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Tools/hemeTools/parsers/extraction/__init__.py b/Tools/hemeTools/parsers/extraction/__init__.py index 725683add..41b2a9199 100644 --- a/Tools/hemeTools/parsers/extraction/__init__.py +++ b/Tools/hemeTools/parsers/extraction/__init__.py @@ -88,7 +88,7 @@ def ParseFieldHeader(self, decoder): self._fieldSpec = FieldSpec([('id', None, np.uint64, 1, None), ('position', None, np.float32, (3,), None)]) - for iField in xrange(self._fieldCount): + for iField in range(self._fieldCount): name = decoder.unpack_string() length = decoder.unpack_uint() self._fieldSpec.Append(name, length, '>f8', np.float64) @@ -117,7 +117,7 @@ def ParseFieldHeader(self, decoder): ('position', None, np.float32, (3,), None)]) self._dataOffset = [0] - for iField in xrange(self._fieldCount): + for iField in range(self._fieldCount): name = decoder.unpack_string() length = decoder.unpack_uint() self._dataOffset.append(decoder.unpack_double()) @@ -172,7 +172,7 @@ def _ReadMainHeader(self): assert version in self.HandledVersions, "Incorrect extraction format version number" self.voxelSizeMetres = decoder.unpack_double() - self.originMetres = np.array([decoder.unpack_double() for i in xrange(3)]) + self.originMetres = np.array([decoder.unpack_double() for i in range(3)]) self.siteCount = decoder.unpack_uhyper() self.fieldCount = decoder.unpack_uint() @@ -218,7 +218,7 @@ def _DetermineTimes(self): nTimes = bodysize / self._recordLength times = np.zeros(nTimes, dtype=int) - for iT in xrange(nTimes): + for iT in range(nTimes): pos = self._totalHeaderLength + iT * self._recordLength self._file.seek(pos) timeBuf = self._file.read(TimeStepDataLength) diff --git a/Tools/hemeTools/utils/utils.py b/Tools/hemeTools/utils/utils.py index 4d963b5ad..e53f9d379 100644 --- a/Tools/hemeTools/utils/utils.py +++ b/Tools/hemeTools/utils/utils.py @@ -5,7 +5,7 @@ # license in the file LICENSE. """Utility functions for HemeTools """ - +from six.moves import xrange import numpy as np def MatchCorresponding(first, second): From cea72efd533ed4c95a59ab0819a5196ad2cbfc1e Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 1 May 2017 09:48:41 +0100 Subject: [PATCH 98/99] Py3: deal with str/bytes/unicode issues between versions for xdr encoded ASCII strings --- .../hemeTools/parsers/extraction/__init__.py | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Tools/hemeTools/parsers/extraction/__init__.py b/Tools/hemeTools/parsers/extraction/__init__.py index 41b2a9199..d5bf9f2a0 100644 --- a/Tools/hemeTools/parsers/extraction/__init__.py +++ b/Tools/hemeTools/parsers/extraction/__init__.py @@ -32,7 +32,7 @@ class FieldSpec(object): def __init__(self, memspec): # name, XDR dtype, in-memory dtype, length, offset - self._filespec = [('grid', '>i4', np.uint32, (3,), 0)] + self._filespec = [(u'grid', '>i4', np.uint32, (3,), 0)] self._memspec = memspec return @@ -51,13 +51,13 @@ def Append(self, name, length, pyType, datatype): def GetMem(self): """Get the numpy datatype for the in-memory array. """ - return np.dtype([(name, memType, length) + return np.dtype([(str(name), memType, length) for name, xdrType, memType, length, offset in (self._memspec + self._filespec)]) def GetXdr(self): """Get the numpy datatype for the XDR file. """ - return np.dtype([(name, xdrType, length) + return np.dtype([(str(name), xdrType, length) for name, xdrType, memType, length, offset in self._filespec]) def GetRecordLength(self): @@ -71,6 +71,10 @@ def __iter__(self): return iter(self._filespec) pass +def unpack_string(decoder): + bstr = decoder.unpack_bytes() + return bstr.decode('utf-8') + class ExtractedPropertyV3Parser(object): def __init__(self, fieldCount, siteCount): self._fieldCount = fieldCount @@ -85,11 +89,11 @@ def parse(self, memoryMappedData): return result def ParseFieldHeader(self, decoder): - self._fieldSpec = FieldSpec([('id', None, np.uint64, 1, None), - ('position', None, np.float32, (3,), None)]) + self._fieldSpec = FieldSpec([(u'id', None, np.uint64, 1, None), + (u'position', None, np.float32, (3,), None)]) for iField in range(self._fieldCount): - name = decoder.unpack_string() + name = unpack_string(decoder) length = decoder.unpack_uint() self._fieldSpec.Append(name, length, '>f8', np.float64) continue @@ -113,12 +117,12 @@ def parse(self, memoryMappedData): return result def ParseFieldHeader(self, decoder): - self._fieldSpec = FieldSpec([('id', None, np.uint64, 1, None), - ('position', None, np.float32, (3,), None)]) + self._fieldSpec = FieldSpec([(u'id', None, np.uint64, 1, None), + (u'position', None, np.float32, (3,), None)]) self._dataOffset = [0] for iField in range(self._fieldCount): - name = decoder.unpack_string() + name = unpack_string(decoder) length = decoder.unpack_uint() self._dataOffset.append(decoder.unpack_double()) self._fieldSpec.Append(name, length, '>f4', np.float32) @@ -215,7 +219,7 @@ def _DetermineTimes(self): bodysize = filesize - self._totalHeaderLength assert bodysize % self._recordLength == 0, \ "Extraction file appears to have partial record(s), residual %s / %s , bodysize %s"%(bodysize % self._recordLength,self._recordLength,bodysize) - nTimes = bodysize / self._recordLength + nTimes = bodysize // self._recordLength times = np.zeros(nTimes, dtype=int) for iT in range(nTimes): From 080bf3fa51828da6518d2c898b07878478ea990e Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 1 May 2017 12:44:00 +0100 Subject: [PATCH 99/99] remove final traces of net namespace --- Code/net/BuildInfo.h.in | 22 -- Code/net/CMakeLists.txt | 13 -- Code/net/IteratedAction.cc | 68 ------ Code/net/IteratedAction.h | 34 --- Code/net/phased/Concern.cc | 6 - Code/net/phased/Concern.h | 21 -- Code/net/phased/StepManager.cc | 210 ------------------ Code/net/phased/StepManager.h | 175 --------------- Code/net/phased/steps.h | 36 --- .../NeighbouringDataManagerTests.h | 2 +- .../lbtests/IncompressibilityCheckerTests.h | 1 - 11 files changed, 1 insertion(+), 587 deletions(-) delete mode 100644 Code/net/BuildInfo.h.in delete mode 100644 Code/net/CMakeLists.txt delete mode 100644 Code/net/IteratedAction.cc delete mode 100644 Code/net/IteratedAction.h delete mode 100644 Code/net/phased/Concern.cc delete mode 100644 Code/net/phased/Concern.h delete mode 100644 Code/net/phased/StepManager.cc delete mode 100644 Code/net/phased/StepManager.h delete mode 100644 Code/net/phased/steps.h diff --git a/Code/net/BuildInfo.h.in b/Code/net/BuildInfo.h.in deleted file mode 100644 index 28778d73b..000000000 --- a/Code/net/BuildInfo.h.in +++ /dev/null @@ -1,22 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_BUILDINFO_H_IN -#define HEMELB_NET_BUILDINFO_H_IN -namespace hemelb -{ - namespace net - { - typedef @HEMELB_POINTPOINT_IMPLEMENTATION@PointPoint PointPointImpl ; - #cmakedefine HEMELB_SEPARATE_CONCERNS - #ifdef HEMELB_SEPARATE_CONCERNS - static const bool separate_communications = true; - #else - static const bool separate_communications = false; - #endif - } -} -#endif // HEMELB_NET_BUILDINFO_H_IN diff --git a/Code/net/CMakeLists.txt b/Code/net/CMakeLists.txt deleted file mode 100644 index 0ff25081e..000000000 --- a/Code/net/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ - -# This file is part of HemeLB and is Copyright (C) -# the HemeLB team and/or their institutions, as detailed in the -# file AUTHORS. This software is provided under the terms of the -# license in the file LICENSE. -add_library(hemelb_net - IteratedAction.cc - CollectiveAction.cc - phased/StepManager.cc) -configure_file ( - "${PROJECT_SOURCE_DIR}/net/BuildInfo.h.in" - "${PROJECT_BINARY_DIR}/net/BuildInfo.h" - ) diff --git a/Code/net/IteratedAction.cc b/Code/net/IteratedAction.cc deleted file mode 100644 index 33204099f..000000000 --- a/Code/net/IteratedAction.cc +++ /dev/null @@ -1,68 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/IteratedAction.h" -#include "net/phased/StepManager.h" - -namespace hemelb -{ - namespace net - { - IteratedAction::~IteratedAction() - { - - } - - bool IteratedAction::CallAction(int action) - { - switch (static_cast(action)) - { - case phased::steps::BeginPhase: - RequestComms(); - return true; - case phased::steps::PreSend: - PreSend(); - return true; - case phased::steps::PreWait: - PreReceive(); - return true; - case phased::steps::EndPhase: - PostReceive(); - return true; - case phased::steps::EndAll: - EndIteration(); - return true; - default: - return false; - } - } - - void IteratedAction::RequestComms() - { - - } - - void IteratedAction::PreSend() - { - - } - - void IteratedAction::PreReceive() - { - - } - - void IteratedAction::PostReceive() - { - - } - - void IteratedAction::EndIteration() - { - - } - } -} diff --git a/Code/net/IteratedAction.h b/Code/net/IteratedAction.h deleted file mode 100644 index 44c1e137c..000000000 --- a/Code/net/IteratedAction.h +++ /dev/null @@ -1,34 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_ITERATEDACTION_H -#define HEMELB_NET_ITERATEDACTION_H -#include "net/phased/Concern.h" -namespace hemelb -{ - namespace net - { - class IteratedAction : public phased::Concern - { - public: - /*** - * When an iterated actor is called through the phased::StepManager mechanism, - * this method dispatches to the individual step methods - * @param action Enumeration indicating the step - * @return True if an action was successfully called for the step - */ - bool CallAction(int action); - virtual ~IteratedAction(); - virtual void RequestComms(); - virtual void PreSend(); - virtual void PreReceive(); - virtual void PostReceive(); - virtual void EndIteration(); - }; - } -} - -#endif /* HEMELB_NET_ITERATEDACTION_H */ diff --git a/Code/net/phased/Concern.cc b/Code/net/phased/Concern.cc deleted file mode 100644 index ac58e5a88..000000000 --- a/Code/net/phased/Concern.cc +++ /dev/null @@ -1,6 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - diff --git a/Code/net/phased/Concern.h b/Code/net/phased/Concern.h deleted file mode 100644 index 2ab9c8c96..000000000 --- a/Code/net/phased/Concern.h +++ /dev/null @@ -1,21 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASED_CONCERN_H -#define HEMELB_NET_PHASED_CONCERN_H -namespace hemelb{ - namespace net{ - namespace phased{ - class Concern{ - public: - Concern(){} - virtual ~Concern(){} - virtual bool CallAction(int action)=0; - }; - } - } -} -#endif //ONCE diff --git a/Code/net/phased/StepManager.cc b/Code/net/phased/StepManager.cc deleted file mode 100644 index 340b10728..000000000 --- a/Code/net/phased/StepManager.cc +++ /dev/null @@ -1,210 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/phased/StepManager.h" -#include - -namespace hemelb -{ - namespace net - { - namespace phased - { - - StepManager::StepManager(Phase phases, reporting::Timers *timers, bool separate_concerns) : - registry(phases), concerns(), timers(timers), separate_concerns(separate_concerns) - { - } - - void StepManager::Register(Phase phase, steps::Step step, Concern & concern, MethodLabel method) - { - if (step == steps::BeginAll || step == steps::EndAll) - { - phase = 0; // special actions are always recorded in the phase zero registry - } - registry[phase][step].push_back(Action(concern, method)); - if (std::find(concerns.begin(),concerns.end(),&concern)==concerns.end()){ - concerns.push_back(&concern); - } - } - - void StepManager::RegisterIteratedActorSteps(Concern &concern, Phase phase) - { - for (int step = steps::BeginPhase; step <= steps::EndAll; step++) - { - if (step == steps::Receive || step == steps::Send || step == steps::Wait) - { - continue; - } - // C++ makes it completely annoying to iterate over values in an enum - Register(phase, static_cast(step), concern, step); - } - } - - void StepManager::RegisterCommsSteps(Concern &concern, Phase phase) - { - Register(phase, steps::Send, concern, steps::Send); - Register(phase, steps::Receive, concern, steps::Receive); - Register(phase, steps::Wait, concern, steps::Wait); - } - - void StepManager::RegisterCommsForAllPhases(Concern &concern) - { - for (Phase phase = 0; phase < registry.size(); phase++) - { - RegisterCommsSteps(concern, phase); - } - } - - unsigned int StepManager::ConcernCount() const - { - return concerns.size(); - } - - unsigned int StepManager::ActionCount() const - { - unsigned int total = 0; - for (std::vector::const_iterator phase = registry.begin(); phase < registry.end(); phase++) - { - for (Registry::const_iterator step = phase->begin(); step != phase->end(); step++) - { - for (std::vector::const_iterator action = step->second.begin(); action != step->second.end(); - action++) - { - total++; - } - } - } - return total; - } - - void StepManager::CallActionsForPhase(Phase phase) - { - // It is assumed, that in the step enum, begin phase begins, and end phase ends, the steps which - // must be called for a given phase. - for (int step = steps::BeginPhase; step <= steps::EndPhase; step++) - { - CallActionsForStep(static_cast(step), phase); - } - } - - void StepManager::CallActionsForPhaseSeparatedConcerns(Phase phase) - { - for (std::vector::iterator concern = concerns.begin(); concern != concerns.end(); concern++) - { - for (int step = steps::BeginPhase; step <= steps::EndPhase; step++) - { - if (step == steps::Receive || step == steps::Send || step == steps::Wait) - { - // Call ALL comms actions for all concerns - // Because, these concerns are net::Net objects, that do the actual send/receive/wait MPI calls - /** - * E.g: - * if A is a concern, B is a concern, C is a concern - * A is an it actor, B is an it actor,C is a net::Net - * You want to go: A C A C A C B C B C B C - */ - CallActionsForStep(static_cast(step), phase); - } - else - { - // Call the actions only for THIS concern - CallActionsForStepForConcern(static_cast(step), *concern, phase); - } - } - } - } - - void StepManager::CallSpecialAction(steps::Step step) - { - // special actions are always recorded in the phase zero registry - CallActionsForStep(static_cast(step), 0); - } - - void StepManager::CallActionsSeparatedConcerns() - { - CallSpecialAction(steps::BeginAll); - for (Phase phase = 0; phase < registry.size(); phase++) - { - CallActionsForPhaseSeparatedConcerns(phase); - } - CallSpecialAction(steps::EndAll); - } - - void StepManager::CallActions() - { - if (separate_concerns) - { - CallActionsSeparatedConcerns(); - return; - } - CallSpecialAction(steps::BeginAll); - for (Phase phase = 0; phase < registry.size(); phase++) - { - CallActionsForPhase(phase); - } - CallSpecialAction(steps::EndAll); - } - - void StepManager::CallActionsForStep(steps::Step step, Phase phase) - { - StartTimer(step); - std::vector &actionsForStep = registry[phase][step]; - for (std::vector::iterator action = actionsForStep.begin(); action != actionsForStep.end(); action++) - { - action->Call(); - } - StopTimer(step); - } - - void StepManager::CallActionsForStepForConcern(steps::Step step, Concern * concern, Phase phase) - { - StartTimer(step); - std::vector &actionsForStep = registry[phase][step]; - for (std::vector::iterator action = actionsForStep.begin(); action != actionsForStep.end(); action++) - { - if (action->concern == concern) - { - action->Call(); - } - } - StopTimer(step); - } - - void StepManager::StartTimer(steps::Step step) - { - if (!timers) - { - return; - } - if (step == steps::Wait) - { - (*timers)[hemelb::reporting::Timers::mpiWait].Start(); - } - if (step == steps::Send) - { - (*timers)[hemelb::reporting::Timers::mpiSend].Start(); - } - } - - void StepManager::StopTimer(steps::Step step) - { - if (!timers) - { - return; - } - if (step == steps::Wait) - { - (*timers)[hemelb::reporting::Timers::mpiWait].Stop(); - } - if (step == steps::Send) - { - (*timers)[hemelb::reporting::Timers::mpiSend].Stop(); - } - } - } - } -} diff --git a/Code/net/phased/StepManager.h b/Code/net/phased/StepManager.h deleted file mode 100644 index fe4584361..000000000 --- a/Code/net/phased/StepManager.h +++ /dev/null @@ -1,175 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASED_STEPMANAGER_H -#define HEMELB_NET_PHASED_STEPMANAGER_H -#include -#include -#include -#include -#include "reporting/Timers.h" -#include "net/IteratedAction.h" -#include "net/phased/Concern.h" -#include "net/phased/steps.h" - -#include "log/Logger.h" -namespace hemelb -{ - namespace net - { - namespace phased - { - /*** - * The step manager provides ability to call, in sequence, methods on several - * interested objects (Called "concerns") for each of several Steps, - * in each of several Phases. - * This provides a mechanism to allow concerns to schedule there activities - * during and between each of the steps of an asynchronous communication pattern - * There may be several sequences of asynchronous communication during a single - * iteration of the SimulationMaster, by registering several phases, each with - * it's own pre/post send-receive steps, etc. - */ - class StepManager - { - public: - - typedef unsigned int Phase; - typedef int MethodLabel; - - /*** - * Class abstracting methods which can be called on concerns - * In principle, we would like to use function pointer classes to do this - * But to keep things simpler, we are assuming indirection through a 'CallAction' method - * on concerns, which handles dispatch to the appropriate method. - */ - class Action - { - public: - Concern * concern; - MethodLabel method; - std::string name; - Action(Concern &concern, MethodLabel method) : - concern(&concern), method(method) - { - } - Action(const Action & action) : - concern(action.concern), method(action.method) - { - } - bool Call() - { - return concern->CallAction(method); - } - }; - - typedef std::map > Registry; - - /*** - * Construct a step manager - * @param The number of phases, default 1. - * @param timers, Record the times for the steps to this timers object, if given - */ - StepManager(Phase phases = 1, reporting::Timers * timers = NULL, bool separate_concerns = false); - - /*** - * Register an action of a concern - * @param phase Phase where an action should be called - * @param step Step where an action should be called - * @param concern Concern on which the method should be called - * @param method label, indicating which method of the concern should be registered - */ - void Register(Phase phase, steps::Step step, Concern & concern, MethodLabel method); - - /*** - * Register a concern for all of the steps typically used by an action, - * syntactic sugar for the individual registrations - * @param concern Concern which should be called, typically an IteratedActor - * @param phase Phase for which this IteratedActor should be called. - */ - void RegisterIteratedActorSteps(Concern &concern, Phase phase = 0); - - /*** - * Register a concern for all of the steps typically used to send/receive communications - * syntactic sugar for the individual registrations (to Send, Receive, Wait) - * @param concern Concern which should be called, typically a net::Net - * @param phase Phase for which this concern should be used for comms - */ - void RegisterCommsSteps(Concern &concern, Phase phase = 0); - - /*** - * Register a concern for all comms steps in all phases, typically the main net::Net - * @param concern Concern to register, typically a net::Net - */ - void RegisterCommsForAllPhases(Concern &concern); - - /*** - * Call the actions, concern by concern, for the given phase - * @param phase - */ - void CallActionsForPhaseSeparatedConcerns(Phase phase = 0); - - /*** - * Call the actions for the given phase - * @param phase - */ - void CallActionsForPhase(Phase phase = 0); - - /*** - * Call all registered actions for a given phase in a given step - * @param step - * @param phase (Default the first phase) - */ - void CallActionsForStepForConcern(steps::Step step,Concern * concern, Phase phase = 0); - - /*** - * Call all registered actions for a given phase in a given step - * @param step - * @param phase (Default the first phase) - */ - void CallActionsForStep(steps::Step step, Phase phase = 0); - - /*** - * Call all actions for all phases - */ - void CallActions(); - - /*** - * Call actions in a different order: do all steps for each concerns separately - */ - void CallActionsSeparatedConcerns(); - - /*** - * Call actions which do not belong to phases: - * the special actions which occur before all phases and after all phases - * @param step - */ - void CallSpecialAction(steps::Step step); - - /*** - * Get the total number of registered concerns - * @return total number of registered concerns - */ - unsigned int ConcernCount() const; - - /*** - * Get the total number of registered actions - * @return total number of registered actions - */ - unsigned int ActionCount() const; - - private: - std::vector registry; // one registry for each phase - std::vector concerns; // can't be a set as must be order-stable - reporting::Timers *timers; - void StartTimer(steps::Step step); - void StopTimer(steps::Step step); - const bool separate_concerns; - - }; - } - } -} -#endif //ONCE diff --git a/Code/net/phased/steps.h b/Code/net/phased/steps.h deleted file mode 100644 index df6befe45..000000000 --- a/Code/net/phased/steps.h +++ /dev/null @@ -1,36 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASED_STEPS_H -#define HEMELB_NET_PHASED_STEPS_H - -namespace hemelb -{ - namespace net - { - namespace phased - { - namespace steps - { - enum Step - { - // Order significant here - // BeginPhase must begin and EndPhase must end, those steps which should be called for a given phase. - BeginAll = -1, // Called only before first phase - BeginPhase = 0, - Receive = 1, - PreSend = 2, - Send = 3, - PreWait = 4, - Wait = 5, - EndPhase = 6, - EndAll = 7, // Called only after final phase - }; - } - } - } -} -#endif diff --git a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h index 19c0aab72..66a45c96a 100644 --- a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h +++ b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h @@ -6,7 +6,7 @@ #ifndef HEMELB_UNITTESTS_GEOMETRY_NEIGHBOURING_NEIGHBOURINGDATAMANAGERTESTS_H #define HEMELB_UNITTESTS_GEOMETRY_NEIGHBOURING_NEIGHBOURINGDATAMANAGERTESTS_H -#include "net/phased/StepManager.h" + #include "geometry/neighbouring/NeighbouringDataManager.h" #include "unittests/helpers/MockCommsHelper.h" #include "comm/MpiEnvironment.h" diff --git a/Code/unittests/lbtests/IncompressibilityCheckerTests.h b/Code/unittests/lbtests/IncompressibilityCheckerTests.h index 7264c517b..7f28158ab 100644 --- a/Code/unittests/lbtests/IncompressibilityCheckerTests.h +++ b/Code/unittests/lbtests/IncompressibilityCheckerTests.h @@ -10,7 +10,6 @@ #include #include "lb/IncompressibilityChecker.h" -#include "net/phased/steps.h" #include "timestep/TimeStepManager.h" #include "unittests/FourCubeLatticeData.h"