From f99030b8e5984971aecf1b4c2da714b5283de182 Mon Sep 17 00:00:00 2001 From: Rupert Nash Date: Mon, 5 Feb 2024 15:01:28 +0000 Subject: [PATCH] Partially de-template StabilityTester and make master non-templated --- Code/CMakeLists.txt | 2 +- ...ationMaster.impl.h => SimulationMaster.cc} | 64 ++---- Code/SimulationMaster.h | 17 +- Code/configuration/SimBuilder.h | 25 +-- Code/lb/StabilityTester.cc | 113 +++++++++- Code/lb/StabilityTester.h | 197 ++++++------------ Code/main.cc | 2 +- Code/multiscale/MultiscaleSimulationMaster.h | 4 +- Code/tests/SimulationMasterTests.cc | 8 +- 9 files changed, 211 insertions(+), 221 deletions(-) rename Code/{SimulationMaster.impl.h => SimulationMaster.cc} (83%) diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 2d95afd53..8d00ecac8 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -123,7 +123,7 @@ if (HEMELB_USE_DEBUGGER) set(CMAKE_BUILD_TYPE Debug) endif() -add_executable(${HEMELB_EXECUTABLE} main.cc) +add_executable(${HEMELB_EXECUTABLE} SimulationMaster.cc main.cc) codesign(${HEMELB_EXECUTABLE}) include_directories(${PROJECT_SOURCE_DIR}) diff --git a/Code/SimulationMaster.impl.h b/Code/SimulationMaster.cc similarity index 83% rename from Code/SimulationMaster.impl.h rename to Code/SimulationMaster.cc index 00507ae35..d2bc383ee 100644 --- a/Code/SimulationMaster.impl.h +++ b/Code/SimulationMaster.cc @@ -2,8 +2,6 @@ // the HemeLB team and/or their institutions, as detailed in the // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_SIMULATIONMASTER_IMPL_H -#define HEMELB_SIMULATIONMASTER_IMPL_H #include "SimulationMaster.h" @@ -39,8 +37,7 @@ namespace hemelb * Initialises member variables including the network topology * object. */ - template - SimulationMaster::SimulationMaster(configuration::CommandLine & options, + SimulationMaster::SimulationMaster(configuration::CommandLine & options, const net::IOCommunicator& ioComm) : build_info(), ioComms(ioComm.Duplicate()), communicationNet(ioComms) { @@ -56,32 +53,16 @@ namespace hemelb // Use it to initialise self auto builder = configuration::SimBuilder(simConfig); log::Logger::Log("Beginning Initialisation."); - builder(*this); + builder.build<>(*this); } - /** - * Destructor for the SimulationMaster class. - * - * Deallocates dynamically allocated memory to contained classes. - */ - template - SimulationMaster::~SimulationMaster() = default; - - /** - * Returns true if the current processor is the dedicated I/O - * processor. - */ - template - bool SimulationMaster::IsCurrentProcTheIOProc() - { - return ioComms.OnIORank(); - } + /// Destructor for the SimulationMaster class. + SimulationMaster::~SimulationMaster() = default; /** * Returns the number of processors involved in the simulation. */ - template - int SimulationMaster::GetProcessorCount() + int SimulationMaster::GetProcessorCount() { return ioComms.Size(); } @@ -90,20 +71,17 @@ namespace hemelb * Initialises various elements of the simulation if necessary: * domain decomposition, LBM and visualisation. */ - template - void SimulationMaster::Initialise() + void SimulationMaster::Initialise() { } - template - void SimulationMaster::HandleActors() + void SimulationMaster::HandleActors() { stepManager->CallActions(); } - template - void SimulationMaster::OnUnstableSimulation() + void SimulationMaster::OnUnstableSimulation() { LogStabilityReport(); log::Logger::Log("Aborting: time step length: %f\n", @@ -115,8 +93,7 @@ namespace hemelb /** * Begin the simulation. */ - template - void SimulationMaster::RunSimulation() + void SimulationMaster::RunSimulation() { log::Logger::Log("Beginning to run simulation."); timings.simulation().Start(); @@ -145,8 +122,7 @@ namespace hemelb Finalise(); } - template - void SimulationMaster::Finalise() + void SimulationMaster::Finalise() { timings.total().Stop(); timings.Reduce(ioComms); @@ -164,8 +140,7 @@ namespace hemelb log::Logger::Log("Finish running simulation."); } - template - void SimulationMaster::DoTimeStep() + void SimulationMaster::DoTimeStep() { log::Logger::Log("Current LB time: %e", simulationState->GetTime()); @@ -218,8 +193,7 @@ namespace hemelb simulationState->Increment(); } - template - void SimulationMaster::RecalculatePropertyRequirements() + void SimulationMaster::RecalculatePropertyRequirements() { // Get the property cache & reset its list of properties to get. lb::MacroscopicPropertyCache& propertyCache = latticeBoltzmannModel->GetPropertyCache(); @@ -242,8 +216,7 @@ namespace hemelb /** * Called on error to abort the simulation and pull-down the MPI environment. */ - template - void SimulationMaster::Abort() + void SimulationMaster::Abort() { // This gives us something to work from when we have an error - we get the rank // that calls abort, and we get a stack-trace from the exception having been thrown. @@ -253,8 +226,7 @@ namespace hemelb exit(1); } - template - void SimulationMaster::LogStabilityReport() + void SimulationMaster::LogStabilityReport() { if (incompressibilityChecker && incompressibilityChecker->AreDensitiesAvailable()) { @@ -274,14 +246,12 @@ namespace hemelb } } - template - const util::UnitConverter& SimulationMaster::GetUnitConverter() const + const util::UnitConverter& SimulationMaster::GetUnitConverter() const { return *unitConverter; } - template - configuration::SimConfig SimulationMaster::ToConfig() const { + configuration::SimConfig SimulationMaster::ToConfig() const { using namespace configuration; SimConfig ans = simConfig; if (simConfig.HasColloidSection()) @@ -289,5 +259,3 @@ namespace hemelb return ans; } } - -#endif diff --git a/Code/SimulationMaster.h b/Code/SimulationMaster.h index cd0dc2890..fb3182225 100644 --- a/Code/SimulationMaster.h +++ b/Code/SimulationMaster.h @@ -8,7 +8,7 @@ #include #include "lb/Lattices.h" -#include "lb/lb.hpp" +#include "lb/lb.h" #include "lb/StabilityTester.h" #include "net/net.h" #include "lb/EntropyTester.h" @@ -32,13 +32,10 @@ namespace hemelb namespace extraction { class PropertyActor; } namespace io { class Checkpointer; } - template> class SimulationMaster { public: friend class configuration::SimBuilder; - using Traits = TRAITS; - using LatticeType = typename Traits::Lattice; protected: reporting::BuildInfo build_info; @@ -53,7 +50,7 @@ namespace hemelb /* The next quantities are protected because they are used by MultiscaleSimulationMaster */ std::shared_ptr domainData; std::shared_ptr fieldData; - std::shared_ptr> latticeBoltzmannModel; + std::shared_ptr latticeBoltzmannModel; std::shared_ptr neighbouringDataManager; std::shared_ptr fileManager; @@ -62,8 +59,8 @@ namespace hemelb std::shared_ptr simulationState; /** Struct containing the configuration of various checkers/testers */ - std::shared_ptr> stabilityTester; - std::shared_ptr> entropyTester; + std::shared_ptr stabilityTester; + // std::shared_ptr> entropyTester; /** Actor in charge of checking the maximum density difference across the domain */ using ICC = lb::IncompressibilityChecker>; std::shared_ptr incompressibilityChecker; @@ -87,7 +84,10 @@ namespace hemelb void Abort(); - bool IsCurrentProcTheIOProc(); + /// Returns true if the current processor is the dedicated I/O processor. + inline bool IsCurrentProcTheIOProc() { + return ioComms.OnIORank(); + } int GetProcessorCount(); @@ -143,5 +143,4 @@ namespace hemelb }; } -#include "SimulationMaster.impl.h" #endif /* HEMELB_SIMULATIONMASTER_H */ diff --git a/Code/configuration/SimBuilder.h b/Code/configuration/SimBuilder.h index a5da12e59..bb93a2075 100644 --- a/Code/configuration/SimBuilder.h +++ b/Code/configuration/SimBuilder.h @@ -6,12 +6,14 @@ #ifndef HEMELB_CONFIGURATION_SIMBUILDER_H #define HEMELB_CONFIGURATION_SIMBUILDER_H +#include "SimulationMaster.h" #include "configuration/SimConfig.h" #include "extraction/LbDataSourceIterator.h" #include "extraction/PropertyActor.h" #include "io/Checkpointer.h" #include "geometry/GmyReadResult.h" #include "geometry/neighbouring/NeighbouringDataManager.h" +#include "lb/lb.hpp" #include "lb/InitialCondition.h" #include "lb/StabilityTester.h" #include "lb/IncompressibilityChecker.hpp" @@ -64,8 +66,8 @@ namespace hemelb::configuration { } // Fully build the T = SimulationMaster from the configuration. - template - void operator()(T & control) const; + template > + void build(SimulationMaster& control) const; // The below could probably be protected/private, but handy for testing. [[nodiscard]] std::shared_ptr BuildSimulationState() const; @@ -89,8 +91,8 @@ namespace hemelb::configuration { [[nodiscard]] std::shared_ptr BuildFlowExtension(FlowExtensionConfig const& conf) const; [[nodiscard]] std::shared_ptr BuildColloidController() const; - template - [[nodiscard]] std::shared_ptr BuildCellController(T const& control, reporting::Timers& timings) const; + template + [[nodiscard]] std::shared_ptr BuildCellController(SimulationMaster const& control, reporting::Timers& timings) const; [[nodiscard]] lb::InitialCondition BuildInitialCondition() const; [[nodiscard]] std::shared_ptr BuildPropertyExtraction( std::filesystem::path const& xtr_path, @@ -115,10 +117,9 @@ namespace hemelb::configuration { }; - template - void SimBuilder::operator()(T& control) const { - using traitsType = typename T::Traits; - using LatticeType = typename T::LatticeType; + template + void SimBuilder::build(SimulationMaster& control) const { + using LatticeType = typename traitsType::Lattice; auto& timings = control.timings; auto& ioComms = control.ioComms; @@ -198,7 +199,7 @@ namespace hemelb::configuration { ); maybe_register_actor(control.outletValues, 1); - maybe_register_actor(control.cellController = BuildCellController(control, timings), 1); + maybe_register_actor(control.cellController = BuildCellController(control, timings), 1); // Copy cos about to scale to lattice units. auto mon_conf = config.GetMonitoringConfiguration(); @@ -207,7 +208,7 @@ namespace hemelb::configuration { } // Always track stability - control.stabilityTester = std::make_shared>( + control.stabilityTester = std::make_shared>( control.fieldData, &control.communicationNet, &*control.simulationState, @@ -294,8 +295,8 @@ namespace hemelb::configuration { } #endif - template - [[nodiscard]] std::shared_ptr SimBuilder::BuildCellController(T const& control, reporting::Timers& timings) const { + template + [[nodiscard]] std::shared_ptr SimBuilder::BuildCellController(SimulationMaster const& control, reporting::Timers& timings) const { if (config.HasRBCSection()) { #ifdef HEMELB_BUILD_RBC using traitsType = typename T::Traits; diff --git a/Code/lb/StabilityTester.cc b/Code/lb/StabilityTester.cc index d0680ba43..b0aed4ce7 100644 --- a/Code/lb/StabilityTester.cc +++ b/Code/lb/StabilityTester.cc @@ -4,12 +4,115 @@ // license in the file LICENSE. #include "lb/StabilityTester.h" -#include "lb/LbmParameters.h" -namespace hemelb +namespace hemelb::lb { - namespace lb - { + StabilityTester::StabilityTester(std::shared_ptr iLatDat, net::Net* net, + SimulationState* simState, reporting::Timers& timings, + const hemelb::configuration::MonitoringConfig& testerConfig) : + net::PhasedBroadcastRegular<>(net, simState, SPREADFACTOR), mLatDat(std::move(iLatDat)), + mSimState(simState), timings(timings), testerConfig(testerConfig) + { + Reset(); + } - } + bool StabilityTester::ShouldTerminateWhenConverged() const { + return testerConfig.convergenceTerminate; + } + /** + * Override the reset method in the base class, to reset the stability variables. + */ + void StabilityTester::Reset() + { + mUpwardsStability = UndefinedStability; + mDownwardsStability = UndefinedStability; + + mSimState->SetStability(UndefinedStability); + + std::fill(mChildrensStability.begin(), mChildrensStability.end(), UndefinedStability); + } + + /** + * Override the methods from the base class to propagate data from the root, and + * to send data about this node and its childrens' stabilities up towards the root. + */ + void StabilityTester::ProgressFromChildren(unsigned long splayNumber) + { + ReceiveFromChildren(mChildrensStability.data(), 1); + } + + void StabilityTester::ProgressFromParent(unsigned long splayNumber) + { + ReceiveFromParent(&mDownwardsStability, 1); + } + + void StabilityTester::ProgressToChildren(unsigned long splayNumber) + { + SendToChildren(&mDownwardsStability, 1); + } + + void StabilityTester::ProgressToParent(unsigned long splayNumber) + { + SendToParent(&mUpwardsStability, 1); + } + + /// Take the combined stability information (an int, with a value of hemelb::lb::Unstable + /// if any child node is unstable) and start passing it back down the tree. + void StabilityTester::TopNodeAction() + { + mDownwardsStability = mUpwardsStability; + } + + void StabilityTester::PostReceiveFromChildren(unsigned long splayNumber) + { + timings.monitoring().Start(); + + // No need to test children's stability if this node is already unstable. + if (mUpwardsStability != Unstable) + { + if (std::any_of( + mChildrensStability.begin(), mChildrensStability.end(), + [](int _) { + return _ == Unstable; + } + )) { + mUpwardsStability = Unstable; + } + + // If the simulation wasn't found to be unstable and we need to check for convergence, do it now. + if ( (mUpwardsStability != Unstable) && testerConfig.doConvergenceCheck) + { + // mChildrensStability will contain UndefinedStability for non-existent children + bool anyConverged = std::any_of( + mChildrensStability.begin(), mChildrensStability.end(), + [](int _) { return _ == StableAndConverged; } + ); + bool anyStableNotConverged = std::any_of( + mChildrensStability.begin(), mChildrensStability.end(), + [](int _) { return _ == Stable; } + ); + + // With the current configuration the root node of the tree won't own any fluid sites. Its + // state only depends on children nodes not on local state. + if (anyConverged + && (mUpwardsStability == StableAndConverged || GetParent() == NOPARENT)) + { + mUpwardsStability = StableAndConverged; + } + + if (anyStableNotConverged) + { + mUpwardsStability = Stable; + } + } + } + + timings.monitoring().Stop(); + } + + /// Apply the stability value sent by the root node to the simulation logic. + void StabilityTester::Effect() + { + mSimState->SetStability((Stability) mDownwardsStability); + } } diff --git a/Code/lb/StabilityTester.h b/Code/lb/StabilityTester.h index 989667ec8..5132c2a4b 100644 --- a/Code/lb/StabilityTester.h +++ b/Code/lb/StabilityTester.h @@ -26,61 +26,82 @@ namespace hemelb::lb * the tree to compose the local stability for all nodes to discover whether the simulation as * a whole is stable. */ - template - class StabilityTester : public net::PhasedBroadcastRegular<> - { - public: - StabilityTester(std::shared_ptr iLatDat, net::Net* net, - SimulationState* simState, reporting::Timers& timings, - const hemelb::configuration::MonitoringConfig& testerConfig) : - net::PhasedBroadcastRegular<>(net, simState, SPREADFACTOR), mLatDat(std::move(iLatDat)), - mSimState(simState), timings(timings), testerConfig(testerConfig) - { - Reset(); - } + class StabilityTester: public net::PhasedBroadcastRegular<> { + protected: + /** + * Slightly arbitrary spread factor for the tree. + */ + static constexpr unsigned SPREADFACTOR = 10; - bool ShouldTerminateWhenConverged() const { - return testerConfig.convergenceTerminate; - } + std::shared_ptr mLatDat; /** - * Override the reset method in the base class, to reset the stability variables. + * Stability value of this node and its children to propagate upwards. */ - void Reset() - { - mUpwardsStability = UndefinedStability; - mDownwardsStability = UndefinedStability; + int mUpwardsStability; + /** + * Stability value as understood by the root node, to pass downwards. + */ + int mDownwardsStability; + /** + * Array for storing the passed-up stability values from child nodes. + */ + std::array mChildrensStability; + /** + * Pointer to the simulation state used in the rest of the simulation. + */ + lb::SimulationState* mSimState; - mSimState->SetStability(UndefinedStability); + /** Timing object. */ + reporting::Timers& timings; - std::fill(mChildrensStability.begin(), mChildrensStability.end(), UndefinedStability); - } + /** Object containing the user-provided configuration for this class */ + hemelb::configuration::MonitoringConfig testerConfig; + + public: + StabilityTester(std::shared_ptr iLatDat, net::Net* net, + SimulationState* simState, reporting::Timers& timings, + const hemelb::configuration::MonitoringConfig& testerConfig); + bool ShouldTerminateWhenConverged() const; + /** + * Override the reset method in the base class, to reset the stability variables. + */ + void Reset(); protected: /** * Override the methods from the base class to propagate data from the root, and * to send data about this node and its childrens' stabilities up towards the root. */ - void ProgressFromChildren(unsigned long splayNumber) override - { - ReceiveFromChildren(mChildrensStability.data(), 1); - } + void ProgressFromChildren(unsigned long splayNumber) override; - void ProgressFromParent(unsigned long splayNumber) override - { - ReceiveFromParent(&mDownwardsStability, 1); - } + void ProgressFromParent(unsigned long splayNumber) override; - void ProgressToChildren(unsigned long splayNumber) override - { - SendToChildren(&mDownwardsStability, 1); - } + void ProgressToChildren(unsigned long splayNumber) override; - void ProgressToParent(unsigned long splayNumber) override - { - SendToParent(&mUpwardsStability, 1); - } + void ProgressToParent(unsigned long splayNumber) override; + + // PostSendToParent in Impl below + + /// Take the combined stability information (an int, with a value of hemelb::lb::Unstable + /// if any child node is unstable) and start passing it back down the tree. + void TopNodeAction() override; + /// Override the method from the base class to use the data from child nodes. + void PostReceiveFromChildren(unsigned long splayNumber) override; + /** + * Apply the stability value sent by the root node to the simulation logic. + */ + void Effect() override; + }; + + template + class StabilityTesterImpl : public StabilityTester + { + public: + using StabilityTester::StabilityTester; + + protected: /** * The algorithm that checks distribution function convergence must be run in this * method rather than in ProgressToParent to make sure that the current timestep has @@ -180,104 +201,6 @@ namespace hemelb::lb throw Exception() << "Convergence check based on requested variable currently not available"; } } - - /** - * Take the combined stability information (an int, with a value of hemelb::lb::Unstable - * if any child node is unstable) and start passing it back down the tree. - */ - void TopNodeAction() override - { - mDownwardsStability = mUpwardsStability; - } - - /** - * Override the method from the base class to use the data from child nodes. - */ - void PostReceiveFromChildren(unsigned long splayNumber) override - { - timings.monitoring().Start(); - - // No need to test children's stability if this node is already unstable. - if (mUpwardsStability != Unstable) - { - if (std::any_of( - mChildrensStability.begin(), mChildrensStability.end(), - [](int _) { - return _ == Unstable; - } - )) { - mUpwardsStability = Unstable; - } - - // If the simulation wasn't found to be unstable and we need to check for convergence, do it now. - if ( (mUpwardsStability != Unstable) && testerConfig.doConvergenceCheck) - { - // mChildrensStability will contain UndefinedStability for non-existent children - bool anyConverged = std::any_of( - mChildrensStability.begin(), mChildrensStability.end(), - [](int _) { return _ == StableAndConverged; } - ); - bool anyStableNotConverged = std::any_of( - mChildrensStability.begin(), mChildrensStability.end(), - [](int _) { return _ == Stable; } - ); - - // With the current configuration the root node of the tree won't own any fluid sites. Its - // state only depends on children nodes not on local state. - if (anyConverged - && (mUpwardsStability == StableAndConverged || GetParent() == NOPARENT)) - { - mUpwardsStability = StableAndConverged; - } - - if (anyStableNotConverged) - { - mUpwardsStability = Stable; - } - } - } - - timings.monitoring().Stop(); - } - - /** - * Apply the stability value sent by the root node to the simulation logic. - */ - void Effect() override - { - mSimState->SetStability((Stability) mDownwardsStability); - } - - private: - /** - * Slightly arbitrary spread factor for the tree. - */ - static constexpr unsigned SPREADFACTOR = 10; - - std::shared_ptr mLatDat; - - /** - * Stability value of this node and its children to propagate upwards. - */ - int mUpwardsStability; - /** - * Stability value as understood by the root node, to pass downwards. - */ - int mDownwardsStability; - /** - * Array for storing the passed-up stability values from child nodes. - */ - std::array mChildrensStability; - /** - * Pointer to the simulation state used in the rest of the simulation. - */ - lb::SimulationState* mSimState; - - /** Timing object. */ - reporting::Timers& timings; - - /** Object containing the user-provided configuration for this class */ - hemelb::configuration::MonitoringConfig testerConfig; }; } diff --git a/Code/main.cc b/Code/main.cc index 7368b54c6..ba5286406 100644 --- a/Code/main.cc +++ b/Code/main.cc @@ -40,7 +40,7 @@ int main(int argc, char *argv[]) debug::Init(options.GetDebug(), argv[0], commWorld); // Prepare main simulation object... - SimulationMaster<> master(options, hemelbCommunicator); + SimulationMaster master(options, hemelbCommunicator); // ..and run it. master.RunSimulation(); diff --git a/Code/multiscale/MultiscaleSimulationMaster.h b/Code/multiscale/MultiscaleSimulationMaster.h index aef65761b..30394fe3f 100644 --- a/Code/multiscale/MultiscaleSimulationMaster.h +++ b/Code/multiscale/MultiscaleSimulationMaster.h @@ -19,13 +19,13 @@ namespace hemelb::multiscale * so the main code can be read without thinking about multiscale. */ template - class MultiscaleSimulationMaster : public SimulationMaster<> + class MultiscaleSimulationMaster : public SimulationMaster { public: MultiscaleSimulationMaster(configuration::CommandLine &options, const net::IOCommunicator& ioComm, Intercommunicator & aintercomms) : - SimulationMaster<>(options, ioComm), intercomms(aintercomms), + SimulationMaster(options, ioComm), intercomms(aintercomms), multiscaleIoletType("inoutlet") { // We only have one shared object type so far, an iolet. diff --git a/Code/tests/SimulationMasterTests.cc b/Code/tests/SimulationMasterTests.cc index 11e5972d5..4889c95fb 100644 --- a/Code/tests/SimulationMasterTests.cc +++ b/Code/tests/SimulationMasterTests.cc @@ -12,10 +12,8 @@ #include "tests/helpers/LaddFail.h" namespace fs = std::filesystem; -namespace hemelb +namespace hemelb::tests { - namespace tests - { TEST_CASE_METHOD(helpers::FolderTestFixture, "SimulationMaster") { const int argc = 3; @@ -29,7 +27,7 @@ namespace hemelb CopyResourceToTempdir("four_cube.gmy"); auto options = std::make_unique(argc, argv); - auto master = std::make_unique>(*options, Comms()); + auto master = std::make_unique(*options, Comms()); SECTION("Running a simulation creates outputs") { // TODO: This test is fatal if run with LADDIOLET. See ticket #605. @@ -47,6 +45,4 @@ namespace hemelb } } } - - } }