diff --git a/CMakeLists.txt b/CMakeLists.txt index d3eaf61d0..fc14273b3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,13 +20,9 @@ set(HEMELB_READING_GROUP_SIZE 5 CACHE INTEGER "Number of cores to use to read geometry file.") set(HEMELB_LOG_LEVEL Info CACHE STRING "Log level, choose 'Critical', 'Error', 'Warning', 'Info', 'Debug' or 'Trace'" ) -set(HEMELB_STEERING_LIB basic - CACHE STRING "Steering library, choose 'basic' or 'none'" ) option(HEMELB_USE_MULTIMACHINE "Use multi-level parallelism support" OFF) option(HEMELB_BUILD_UNITTESTS "Build the unit-tests" ON) -option(HEMELB_USE_STREAKLINES "Calculate streakline images" OFF) option(HEMELB_USE_ALL_WARNINGS_GNU "Show all compiler warnings on development builds (gnu-style-compilers)" ON) -option(HEMELB_STATIC_ASSERT "Use simple compile-time assertions" ON) set(HEMELB_OPTIMISATION "-O4" CACHE STRING "Optimisation level (can be blank or -O1 to -O4)") option(HEMELB_BUILD_MULTISCALE "Build HemeLB Multiscale functionality" OFF) set(HEMELB_LATTICE "D3Q15" @@ -35,10 +31,8 @@ set(HEMELB_KERNEL "LBGK" CACHE STRING "Select the kernel to use (LBGK,EntropicAnsumali,EntropicChik,MRT,NNCY,NNC,NNTPL)") set(HEMELB_WALL_BOUNDARY "SIMPLEBOUNCEBACK" CACHE STRING "Select the boundary conditions to be used at the walls (FINTERPOLATION,GZS,SIMPLEBOUNCEBACK,JUNKYANG)") -set(HEMELB_STEERING_HOST "CCS" CACHE STRING "Use a default host suffix for steering? (CCS, NGS2Leeds, NGS2Manchester, LONI, NCSA or blank)") option(HEMELB_DEPENDENCIES_SET_RPATH "Set runtime RPATH" ON) set(HEMELB_SUBPROJECT_MAKE_JOBS 1 CACHE INTEGER "Number of jobs to use for subproject build steps") -option(HEMELB_WAIT_ON_CONNECT "Wait for steering client" OFF) option(HEMELB_IMAGES_TO_NULL "Write images to null" OFF) option(HEMELB_USE_SSE3 "Use SSE3 intrinsics" OFF) set(HEMELB_COMPUTE_ARCHITECTURE "AMDBULLDOZER" @@ -62,10 +56,8 @@ ExternalProject_Add( -DHEMELB_USE_DEBUGGER=${HEMELB_USE_DEBUGGER} -DHEMELB_VALIDATE_GEOMETRY=${HEMELB_VALIDATE_GEOMETRY} -DHEMELB_LOG_LEVEL=${HEMELB_LOG_LEVEL} - -DHEMELB_STEERING_LIB=${HEMELB_STEERING_LIB} -DHEMELB_USE_MULTIMACHINE=${HEMELB_USE_MULTIMACHINE} -DHEMELB_BUILD_UNITTESTS=${HEMELB_BUILD_UNITTESTS} - -DHEMELB_USE_STREAKLINES=${HEMELB_USE_STREAKLINES} -DHEMELB_OPTIMISATION=${HEMELB_OPTIMISATION} -DHEMELB_READING_GROUP_SIZE=${HEMELB_READING_GROUP_SIZE} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} @@ -75,11 +67,9 @@ ExternalProject_Add( -DMPI_C_NO_INTERROGATE=${MPI_C_NO_INTERROGATE} -DMPI_CXX_NO_INTERROGATE=${MPI_CXX_NO_INTERROGATE} -DHEMELB_USE_ALL_WARNINGS_GNU=${HEMELB_USE_ALL_WARNINGS_GNU} - -DHEMELB_STEERING_HOST=${HEMELB_STEERING_HOST} -DCTEMPLATE_USE_STATIC=${CTEMPLATE_USE_STATIC} -DCPPUNIT_USE_STATIC=${CPPUNIT_USE_STATIC} -DHEMELB_DEPENDENCIES_SET_RPATH=${HEMELB_DEPENDENCIES_SET_RPATH} - -DHEMELB_STATIC_ASSERT=${HEMELB_STATIC_ASSERT} -DHEMELB_LATTICE=${HEMELB_LATTICE} -DHEMELB_KERNEL=${HEMELB_KERNEL} -DHEMELB_WALL_BOUNDARY=${HEMELB_WALL_BOUNDARY} diff --git a/Code/CMakeLists.txt b/Code/CMakeLists.txt index 35a6cc520..a88f3d2be 100644 --- a/Code/CMakeLists.txt +++ b/Code/CMakeLists.txt @@ -7,9 +7,6 @@ cmake_minimum_required (VERSION 3.2) project(HemeLB) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED ON) - #---- OPTION switches accepted by the build ------- option(HEMELB_BUILD_DEBUGGER "Build the built in debugger" ON) @@ -21,10 +18,7 @@ option(HEMELB_BUILD_TESTS_ALL "Build all the tests" ON) option(HEMELB_BUILD_TESTS_UNIT "Build the unit-tests (HEMELB_BUILD_TESTS_ALL takes precedence)" ON) option(HEMELB_BUILD_TESTS_FUNCTIONAL "Build the functional tests (HEMELB_BUILD_TESTS_ALL takes precedence)" ON) option(HEMELB_USE_ALL_WARNINGS_GNU "Show all compiler warnings on development builds (gnu-style-compilers)" ON) -option(HEMELB_USE_STREAKLINES "Calculate streakline images" OFF) option(HEMELB_DEPENDENCIES_SET_RPATH "Set runtime RPATH" ON) -option(HEMELB_STATIC_ASSERT "Use simple compile-time assertions" ON) -option(HEMELB_WAIT_ON_CONNECT "Wait for steering client" OFF) option(HEMELB_BUILD_MULTISCALE "Build HemeLB Multiscale functionality" OFF) option(HEMELB_IMAGES_TO_NULL "Write images to null" OFF) option(HEMELB_USE_SSE3 "Use SSE3 intrinsics" OFF) @@ -36,8 +30,6 @@ set(HEMELB_READING_GROUP_SIZE 5 CACHE INTEGER "Number of cores to use to read geometry file.") set(HEMELB_LOG_LEVEL Info CACHE STRING "Log level, choose 'Critical', 'Error', 'Warning', 'Info', 'Debug' or 'Trace'" ) -set(HEMELB_STEERING_LIB basic - CACHE STRING "Steering library, choose 'basic' or 'none'" ) set(HEMELB_DEPENDENCIES_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../dependencies" CACHE FILEPATH "Path to find dependency find modules") set(HEMELB_DEPENDENCIES_INSTALL_PATH ${HEMELB_DEPENDENCIES_PATH} @@ -61,10 +53,6 @@ set(HEMELB_WALL_OUTLET_BOUNDARY "NASHZEROTHORDERPRESSURESBB" CACHE STRING "Select the boundary conditions to be used at corners between walls and outlets (NASHZEROTHORDERPRESSURESBB,NASHZEROTHORDERPRESSUREBFL,LADDIOLETSBB,LADDIOLETBFL)") set(HEMELB_POINTPOINT_IMPLEMENTATION Coalesce CACHE STRING "Point to point comms implementation, choose 'Coalesce', 'Separated', or 'Immediate'" ) -set(HEMELB_GATHERS_IMPLEMENTATION Separated - CACHE STRING "Gather comms implementation, choose 'Separated', or 'ViaPointPoint'" ) -set(HEMELB_ALLTOALL_IMPLEMENTATION Separated - CACHE STRING "Alltoall comms implementation, choose 'Separated', or 'ViaPointPoint'" ) option(HEMELB_SEPARATE_CONCERNS "Communicate for each concern separately" OFF) # Add warnings flags to development build types @@ -94,18 +82,10 @@ if(HEMELB_VALIDATE_GEOMETRY) add_definitions(-DHEMELB_VALIDATE_GEOMETRY) endif() -if (NOT HEMELB_USE_STREAKLINES) - add_definitions(-DNO_STREAKLINES) -endif() - if (HEMELB_WAIT_ON_CONNECT) add_definitions(-DHEMELB_WAIT_ON_CONNECT) endif() -if (NOT HEMELB_STATIC_ASSERT) - add_definitions(-DHEMELB_NO_STATIC_ASSERT) -endif() - if (HEMELB_IMAGES_TO_NULL) add_definitions(-DHEMELB_IMAGES_TO_NULL) endif() @@ -121,6 +101,7 @@ if (HEMELB_USE_VELOCITY_WEIGHTS_FILE) add_definitions(-DHEMELB_USE_VELOCITY_WEIGHTS_FILE) endif() + list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake" "${HEMELB_DEPENDENCIES_PATH}/Modules/") list(APPEND CMAKE_INCLUDE_PATH ${HEMELB_DEPENDENCIES_INSTALL_PATH}/include) list(APPEND CMAKE_LIBRARY_PATH ${HEMELB_DEPENDENCIES_INSTALL_PATH}/lib) @@ -131,6 +112,7 @@ if(HEMELB_DEPENDENCIES_SET_RPATH) SET(CMAKE_INSTALL_RPATH "${HEMELB_DEPENDENCIES_INSTALL_PATH}/lib") endif() +include(cxx11) include(build_environment) include(platform_checks) @@ -158,6 +140,10 @@ if(NOT HAVE_XDRUINTXX_T) endif() include(mpi) +if(MPI_STANDARD_VERSION VERSION_LESS 3) + message(FATAL_ERROR "Must have MPI 3.0 or greater") +endif() + include(dependencies) #-------------Resources ----------------------- @@ -190,13 +176,12 @@ add_executable(${HEMELB_EXECUTABLE} main.cc ${root_sources}) include_directories(${PROJECT_SOURCE_DIR}) set(package_subdirs configuration + comm extraction reporting - steering - vis lb geometry - net + timestep debug util io @@ -236,14 +221,13 @@ if (HEMELB_BUILD_MULTISCALE) add_executable(multiscale_hemelb mainMultiscale.cc ${root_sources}) include_directories(${PROJECT_SOURCE_DIR}) set(package_subdirs + comm configuration extraction reporting - steering - vis lb geometry - net + timestep debug util io diff --git a/Code/SimulationMaster.cc b/Code/SimulationMaster.cc index d617fe087..3f1a3cf25 100644 --- a/Code/SimulationMaster.cc +++ b/Code/SimulationMaster.cc @@ -17,10 +17,9 @@ #include "lb/HFunction.h" #include "io/xml/XmlAbstractionLayer.h" #include "colloids/ColloidController.h" -#include "net/BuildInfo.h" -#include "net/IOCommunicator.h" #include "colloids/BodyForces.h" #include "colloids/BoundaryConditions.h" +#include "comm/MpiEnvironment.h" #include #include @@ -32,8 +31,8 @@ * Initialises member variables including the network topology * object. */ -SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, const hemelb::net::IOCommunicator& ioComm) : - ioComms(ioComm), timings(ioComm), build_info(), communicationNet(ioComm) +SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, hemelb::comm::Communicator::ConstPtr ioComm) : + ioComms(ioComm), timings(ioComm), build_info(), asyncCommQ(hemelb::comm::Async::New(ioComm)) { timings[hemelb::reporting::Timers::total].Start(); @@ -41,16 +40,12 @@ SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, colloidController = NULL; latticeBoltzmannModel = NULL; - steeringCpt = NULL; propertyDataSource = NULL; - visualisationControl = NULL; propertyExtractor = NULL; simulationState = NULL; stepManager = NULL; - netConcern = NULL; + asyncCommsManager = NULL; neighbouringDataManager = NULL; - imagesPerSimulation = options.NumberOfImages(); - steeringSessionId = options.GetSteeringSessionId(); fileManager = new hemelb::io::PathManager(options, IsCurrentProcTheIOProc(), GetProcessorCount()); simConfig = hemelb::configuration::SimConfig::New(fileManager->GetInputFile()); @@ -82,18 +77,11 @@ SimulationMaster::SimulationMaster(hemelb::configuration::CommandLine & options, SimulationMaster::~SimulationMaster() { - if (ioComms.OnIORank()) - { - delete imageSendCpt; - } delete latticeData; delete colloidController; delete latticeBoltzmannModel; delete inletValues; delete outletValues; - delete network; - delete steeringCpt; - delete visualisationControl; delete propertyExtractor; delete propertyDataSource; delete stabilityTester; @@ -109,7 +97,7 @@ SimulationMaster::~SimulationMaster() delete reporter; } delete stepManager; - delete netConcern; + delete asyncCommsManager; } /** @@ -118,7 +106,7 @@ SimulationMaster::~SimulationMaster() */ bool SimulationMaster::IsCurrentProcTheIOProc() { - return ioComms.OnIORank(); + return ioComms->OnIORank(); } /** @@ -126,12 +114,12 @@ bool SimulationMaster::IsCurrentProcTheIOProc() */ int SimulationMaster::GetProcessorCount() { - return ioComms.Size(); + return ioComms->Size(); } /** - * Initialises various elements of the simulation if necessary - steering, - * domain decomposition, LBM and visualisation. + * Initialises various elements of the simulation if necessary - + * domain decomposition, and LBM. */ void SimulationMaster::Initialise() { @@ -147,8 +135,7 @@ void SimulationMaster::Initialise() // Use a reader to read in the file. hemelb::log::Logger::Log("Loading file and decomposing geometry."); - hemelb::geometry::GeometryReader reader(hemelb::steering::SteeringComponent::RequiresSeparateSteeringCore(), - latticeType::GetLatticeInfo(), + hemelb::geometry::GeometryReader reader(latticeType::GetLatticeInfo(), timings, ioComms); hemelb::geometry::Geometry readGeometryData = reader.LoadAndDecompose(simConfig->GetDataFilePath()); @@ -161,10 +148,10 @@ void SimulationMaster::Initialise() neighbouringDataManager = new hemelb::geometry::neighbouring::NeighbouringDataManager(*latticeData, latticeData->GetNeighbouringData(), - communicationNet); + asyncCommQ); hemelb::log::Logger::Log("Initialising LBM."); latticeBoltzmannModel = new hemelb::lb::LBM(simConfig, - &communicationNet, + asyncCommQ, latticeData, simulationState, timings, @@ -199,28 +186,19 @@ void SimulationMaster::Initialise() } timings[hemelb::reporting::Timers::colloidInitialisation].Stop(); - // Initialise and begin the steering. - if (ioComms.OnIORank()) - { - network = new hemelb::steering::Network(steeringSessionId, timings); - } - else - { - network = NULL; - } stabilityTester = new hemelb::lb::StabilityTester(latticeData, - &communicationNet, + ioComms, simulationState, timings, - monitoringConfig); + monitoringConfig->doConvergenceCheck, + monitoringConfig->convergenceRelativeTolerance); entropyTester = NULL; if (monitoringConfig->doIncompressibilityCheck) { - incompressibilityChecker = new hemelb::lb::IncompressibilityChecker< - hemelb::net::PhasedBroadcastRegular<> >(latticeData, - &communicationNet, + incompressibilityChecker = new hemelb::lb::IncompressibilityChecker(latticeData, + ioComms, simulationState, latticeBoltzmannModel->GetPropertyCache(), timings); @@ -230,62 +208,28 @@ void SimulationMaster::Initialise() incompressibilityChecker = NULL; } - hemelb::log::Logger::Log("Initialising visualisation controller."); - visualisationControl = - new hemelb::vis::Control(latticeBoltzmannModel->GetLbmParams()->StressType, - &communicationNet, - simulationState, - latticeBoltzmannModel->GetPropertyCache(), - latticeData, - timings[hemelb::reporting::Timers::visualisation]); - - if (ioComms.OnIORank()) - { - imageSendCpt = new hemelb::steering::ImageSendComponent(simulationState, - visualisationControl, - latticeBoltzmannModel->GetLbmParams(), - network, - latticeBoltzmannModel->InletCount()); - - } - else - { - imageSendCpt = NULL; - } - inletValues = new hemelb::lb::iolets::BoundaryValues(hemelb::geometry::INLET_TYPE, latticeData, simConfig->GetInlets(), simulationState, - ioComms, + asyncCommQ, *unitConverter); outletValues = new hemelb::lb::iolets::BoundaryValues(hemelb::geometry::OUTLET_TYPE, latticeData, simConfig->GetOutlets(), simulationState, - ioComms, + asyncCommQ, *unitConverter); - latticeBoltzmannModel->Initialise(visualisationControl, inletValues, outletValues, unitConverter); + latticeBoltzmannModel->Initialise(inletValues, outletValues, unitConverter); neighbouringDataManager->ShareNeeds(); neighbouringDataManager->TransferNonFieldDependentInformation(); - steeringCpt = new hemelb::steering::SteeringComponent(network, - visualisationControl, - imageSendCpt, - &communicationNet, - simulationState, - simConfig, - unitConverter); - - // Read in the visualisation parameters. - latticeBoltzmannModel->ReadVisParameters(); - propertyDataSource = new hemelb::extraction::LbDataSourceIterator(latticeBoltzmannModel->GetPropertyCache(), *latticeData, - ioComms.Rank(), + ioComms->Rank(), *unitConverter); if (simConfig->PropertyOutputCount() > 0) @@ -303,43 +247,37 @@ void SimulationMaster::Initialise() timings, ioComms); } - imagesPeriod = OutputPeriod(imagesPerSimulation); + const int nPhase = 2; + stepManager = new hemelb::timestep::TimeStepManager(nPhase); + asyncCommsManager = new hemelb::comm::AsyncConcern(asyncCommQ); + for (auto i=0; iAddToPhase(i, asyncCommsManager); - stepManager = new hemelb::net::phased::StepManager(2, - &timings, - hemelb::net::separate_communications); - netConcern = new hemelb::net::phased::NetConcern(communicationNet); - stepManager->RegisterIteratedActorSteps(*neighbouringDataManager, 0); + stepManager->AddToPhase(0, neighbouringDataManager); if (colloidController != NULL) { - stepManager->RegisterIteratedActorSteps(*colloidController, 1); + stepManager->AddToPhase(1, colloidController); } - stepManager->RegisterIteratedActorSteps(*latticeBoltzmannModel, 1); + stepManager->AddToPhase(1, latticeBoltzmannModel); - stepManager->RegisterIteratedActorSteps(*inletValues, 1); - stepManager->RegisterIteratedActorSteps(*outletValues, 1); - stepManager->RegisterIteratedActorSteps(*steeringCpt, 1); - stepManager->RegisterIteratedActorSteps(*stabilityTester, 1); + stepManager->AddToPhase(1, inletValues); + stepManager->AddToPhase(1, outletValues); + stepManager->AddToPhase(1, stabilityTester); if (entropyTester != NULL) { - stepManager->RegisterIteratedActorSteps(*entropyTester, 1); + stepManager->AddToPhase(1, entropyTester); } if (monitoringConfig->doIncompressibilityCheck) { - stepManager->RegisterIteratedActorSteps(*incompressibilityChecker, 1); + stepManager->AddToPhase(1, incompressibilityChecker); } - stepManager->RegisterIteratedActorSteps(*visualisationControl, 1); + if (propertyExtractor != NULL) { - stepManager->RegisterIteratedActorSteps(*propertyExtractor, 1); + stepManager->AddToPhase(1, propertyExtractor); } - if (ioComms.OnIORank()) - { - stepManager->RegisterIteratedActorSteps(*network, 1); - } - stepManager->RegisterCommsForAllPhases(*netConcern); } unsigned int SimulationMaster::OutputPeriod(unsigned int frequency) @@ -354,7 +292,7 @@ unsigned int SimulationMaster::OutputPeriod(unsigned int frequency) void SimulationMaster::HandleActors() { - stepManager->CallActions(); + stepManager->DoStep(); } void SimulationMaster::OnUnstableSimulation() @@ -366,77 +304,6 @@ void SimulationMaster::OnUnstableSimulation() Abort(); } -void SimulationMaster::WriteLocalImages() -{ - /** - * this map iteration iterates over all those image generation requests completing this step. - * The map key (it->first) is the completion time step number. - * The map value (it->second) is the initiation time step number. - */ - for (MapType::const_iterator it = writtenImagesCompleted.find(simulationState->GetTimeStep()); - it != writtenImagesCompleted.end() && it->first == simulationState->GetTimeStep(); ++it) - { - - if (ioComms.OnIORank()) - { - reporter->Image(); - hemelb::io::writers::Writer * writer = fileManager->XdrImageWriter(1 - + ( (it->second - 1) % simulationState->GetTimeStep())); - - const hemelb::vis::PixelSet* result = - visualisationControl->GetResult(it->second); - - visualisationControl->WriteImage(writer, - *result, - visualisationControl->domainStats, - visualisationControl->visSettings); - - delete writer; - } - } - - writtenImagesCompleted.erase(simulationState->GetTimeStep()); -} - -void SimulationMaster::GenerateNetworkImages() -{ - for (std::multimap::const_iterator it = - networkImagesCompleted.find(simulationState->GetTimeStep()); - it != networkImagesCompleted.end() && it->first == simulationState->GetTimeStep(); ++it) - { - if (ioComms.OnIORank()) - { - - const hemelb::vis::PixelSet* result = - visualisationControl->GetResult(it->second); - - if (steeringCpt->updatedMouseCoords) - { - float density, stress; - - if (visualisationControl->MouseIsOverPixel(result, &density, &stress)) - { - double mousePressure = 0.0, mouseStress = 0.0; - latticeBoltzmannModel->CalculateMouseFlowField(density, - stress, - mousePressure, - mouseStress, - visualisationControl->domainStats.density_threshold_min, - visualisationControl->domainStats.density_threshold_minmax_inv, - visualisationControl->domainStats.stress_threshold_max_inv); - - visualisationControl->SetMouseParams(mousePressure, mouseStress); - } - steeringCpt->updatedMouseCoords = false; - } - - imageSendCpt->DoWork(result); - - } - } - - networkImagesCompleted.erase(simulationState->GetTimeStep()); -} /** * Begin the simulation. @@ -446,12 +313,21 @@ void SimulationMaster::RunSimulation() hemelb::log::Logger::Log("Beginning to run simulation."); timings[hemelb::reporting::Timers::simulation].Start(); - while (simulationState->GetTimeStep() <= simulationState->GetTotalTimeSteps()) + int LastTS = simulationState->GetTotalTimeSteps(); + while (simulationState->GetTimeStep() <= LastTS) { + // We need to keep the master rank in sync with the workers + // Here, each rank notifies that it is beginning a time step + auto syncReq = ioComms->Ibarrier(); + DoTimeStep(); + + syncReq->Wait(); + // Now all ranks have at least started this time step + if (simulationState->IsTerminating()) { - break; + LastTS = std::min(simulationState->GetTimeStep(), simulationState->GetTotalTimeSteps()); } } @@ -468,64 +344,28 @@ void SimulationMaster::Finalise() reporter->FillDictionary(); reporter->Write(); } - // DTMP: Logging output on communication as debug output for now. - hemelb::log::Logger::Log("sync points: %lld, bytes sent: %lld", - communicationNet.SyncPointsCounted, - communicationNet.BytesSent); - + hemelb::log::Logger::Log("Finish running simulation."); } void SimulationMaster::DoTimeStep() { - bool writeImage = ( (simulationState->GetTimeStep() % imagesPeriod) == 0) ? - true : - false; - - // Make sure we're rendering if we're writing this iteration. - if (writeImage) - { - /*** - * writtenImagesCompleted and networkImagesCompleted are multimaps. - * The keys are the iterations on which production of an image will complete, and should be written or sent over the network. - * The values are the iterations on which the image creation began. - */ - writtenImagesCompleted.insert(std::pair(visualisationControl->Start(), - simulationState->GetTimeStep())); - } - - if (simulationState->IsRendering()) - { - // Here, Start() actually triggers the render. - networkImagesCompleted.insert(std::pair(visualisationControl->Start(), - simulationState->GetTimeStep())); - hemelb::log::Logger::Log("%d images currently being composited for the steering client", - networkImagesCompleted.size()); - simulationState->SetIsRendering(false); - } - - /* In the following two if blocks we do the core magic to ensure we only Render - when (1) we are not sending a frame or (2) we need to output to disk */ - - /* TODO for debugging purposes we want to ensure we capture the variables in a single - instant of time since variables might be altered by the thread half way through? - This is to be done. */ - - bool renderForNetworkStream = false; - if (ioComms.OnIORank() - && !steeringCpt->readyForNextImage) - { - renderForNetworkStream = imageSendCpt->ShouldRenderNewNetworkImage(); - steeringCpt->readyForNextImage = renderForNetworkStream; - } - + // If the simulation is finishing, all running collective actions + // must wait this time step + if (simulationState->IsTerminating()) + { + if (stabilityTester) + stabilityTester->MustFinishThisTimeStep(); + if (entropyTester) + entropyTester->MustFinishThisTimeStep(); + if (incompressibilityChecker) + incompressibilityChecker->MustFinishThisTimeStep(); + } + if (simulationState->GetTimeStep() % 100 == 0) { - hemelb::log::Logger::Log("time step %i render_network_stream %i write_image_to_disk %i rendering %i", - simulationState->GetTimeStep(), - renderForNetworkStream, - writeImage, - simulationState->IsRendering()); + hemelb::log::Logger::Log("time step %i", + simulationState->GetTimeStep()); LogStabilityReport(); } @@ -539,7 +379,7 @@ void SimulationMaster::DoTimeStep() } // If the user requested to terminate converged steady flow simulations, mark - // simulation to be finished at the end of the current timestep. + // simulation to be finished ASAP. if ( (simulationState->GetStability() == hemelb::lb::StableAndConverged) && monitoringConfig->convergenceTerminate) { @@ -550,22 +390,6 @@ void SimulationMaster::DoTimeStep() if ( (simulationState->GetTimeStep() % 500 == 0) && colloidController != NULL) colloidController->OutputInformation(simulationState->GetTimeStep()); -#ifndef NO_STREAKLINES - visualisationControl->ProgressStreaklines(simulationState->GetTimeStep(), - simulationState->GetTotalTimeSteps()); -#endif - - if (writtenImagesCompleted.count(simulationState->GetTimeStep()) > 0) - { - WriteLocalImages(); - - } - - if (networkImagesCompleted.count(simulationState->GetTimeStep()) > 0) - { - GenerateNetworkImages(); - } - if (simulationState->GetTimeStep() % FORCE_FLUSH_PERIOD == 0 && IsCurrentProcTheIOProc()) { fflush(NULL); @@ -580,22 +404,6 @@ void SimulationMaster::RecalculatePropertyRequirements() propertyCache.ResetRequirements(); - // Check whether we're rendering images on this iteration. - if (visualisationControl->IsRendering()) - { - propertyCache.densityCache.SetRefreshFlag(); - propertyCache.velocityCache.SetRefreshFlag(); - - if (simConfig->GetStressType() == hemelb::lb::ShearStress) - { - propertyCache.wallShearStressMagnitudeCache.SetRefreshFlag(); - } - else if (simConfig->GetStressType() == hemelb::lb::VonMises) - { - propertyCache.vonMisesStressCache.SetRefreshFlag(); - } - } - if (monitoringConfig->doIncompressibilityCheck) { propertyCache.densityCache.SetRefreshFlag(); @@ -607,11 +415,6 @@ void SimulationMaster::RecalculatePropertyRequirements() { propertyExtractor->SetRequiredProperties(propertyCache); } - - // If using streaklines, the velocity will be needed. -#ifndef NO_STREAKLINES - propertyCache.velocityCache.SetRefreshFlag(); -#endif } /** @@ -622,15 +425,14 @@ void SimulationMaster::Abort() // This gives us something to work from when we have an error - we get the rank // that calls abort, and we get a stack-trace from the exception having been thrown. hemelb::log::Logger::Log("Aborting"); - hemelb::net::MpiEnvironment::Abort(1); + hemelb::comm::MpiEnvironment::Abort(1); exit(1); } void SimulationMaster::LogStabilityReport() { - if (monitoringConfig->doIncompressibilityCheck - && incompressibilityChecker->AreDensitiesAvailable()) + if (monitoringConfig->doIncompressibilityCheck) { hemelb::log::Logger::Log("time step %i, tau %.6f, max_relative_press_diff %.3f, Ma %.3f, max_vel_phys %e", simulationState->GetTimeStep(), diff --git a/Code/SimulationMaster.h b/Code/SimulationMaster.h index 50446f4df..3e5bcbe80 100644 --- a/Code/SimulationMaster.h +++ b/Code/SimulationMaster.h @@ -10,9 +10,6 @@ #include "extraction/PropertyActor.h" #include "lb/lb.hpp" #include "lb/StabilityTester.h" -#include "net/net.h" -#include "steering/ImageSendComponent.h" -#include "steering/SteeringComponent.h" #include "lb/EntropyTester.h" #include "lb/iolets/BoundaryValues.h" #include "util/UnitConverter.h" @@ -21,16 +18,16 @@ #include "reporting/Reporter.h" #include "reporting/Timers.h" #include "reporting/BuildInfo.h" -#include "lb/IncompressibilityChecker.hpp" +#include "lb/IncompressibilityChecker.h" #include "colloids/ColloidController.h" -#include "net/phased/StepManager.h" -#include "net/phased/NetConcern.h" +#include "timestep/TimeStepManager.h" +#include "comm/AsyncConcern.h" #include "geometry/neighbouring/NeighbouringDataManager.h" class SimulationMaster { public: - SimulationMaster(hemelb::configuration::CommandLine &options, const hemelb::net::IOCommunicator& ioComms); + SimulationMaster(hemelb::configuration::CommandLine &options, hemelb::comm::Communicator::ConstPtr ioComms); virtual ~SimulationMaster(); void Abort(); @@ -56,7 +53,7 @@ class SimulationMaster hemelb::geometry::LatticeData* latticeData; hemelb::lb::LBM* latticeBoltzmannModel; hemelb::geometry::neighbouring::NeighbouringDataManager *neighbouringDataManager; - const hemelb::net::IOCommunicator& ioComms; + hemelb::comm::Communicator::ConstPtr ioComms; private: void Initialise(); @@ -64,8 +61,6 @@ class SimulationMaster unsigned int OutputPeriod(unsigned int frequency); void HandleActors(); void OnUnstableSimulation(); - void WriteLocalImages(); - void GenerateNetworkImages(); /** * Updates the property caches record of which properties need to be calculated * and cached on this iteration. @@ -84,13 +79,6 @@ class SimulationMaster hemelb::reporting::BuildInfo build_info; typedef std::multimap MapType; - MapType writtenImagesCompleted; - MapType networkImagesCompleted; - - hemelb::steering::Network* network; - hemelb::steering::ImageSendComponent *imageSendCpt; - hemelb::steering::SteeringComponent* steeringCpt; - hemelb::lb::SimulationState* simulationState; /** Struct containing the configuration of various checkers/testers */ @@ -98,23 +86,18 @@ class SimulationMaster hemelb::lb::StabilityTester* stabilityTester; hemelb::lb::EntropyTester* entropyTester; /** Actor in charge of checking the maximum density difference across the domain */ - hemelb::lb::IncompressibilityChecker >* incompressibilityChecker; + hemelb::lb::IncompressibilityChecker* incompressibilityChecker; hemelb::colloids::ColloidController* colloidController; - hemelb::net::Net communicationNet; - + hemelb::comm::Async::Ptr asyncCommQ; const hemelb::util::UnitConverter* unitConverter; - hemelb::vis::Control* visualisationControl; hemelb::extraction::IterableDataSource* propertyDataSource; hemelb::extraction::PropertyActor* propertyExtractor; - hemelb::net::phased::StepManager* stepManager; - hemelb::net::phased::NetConcern* netConcern; + hemelb::timestep::TimeStepManager* stepManager; + hemelb::comm::AsyncConcern* asyncCommsManager; - unsigned int imagesPerSimulation; - int steeringSessionId; - unsigned int imagesPeriod; static const hemelb::LatticeTimeStep FORCE_FLUSH_PERIOD=1000; }; diff --git a/Code/cmake/build_environment.cmake b/Code/cmake/build_environment.cmake index 58df48676..974bb0e7a 100644 --- a/Code/cmake/build_environment.cmake +++ b/Code/cmake/build_environment.cmake @@ -5,7 +5,7 @@ # license in the file LICENSE. #------Capture build environment ------------- find_package(Git REQUIRED) -execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse HEAD +execute_process(COMMAND ${GIT_EXECUTABLE} --git-dir ${CMAKE_SOURCE_DIR}/../.git rev-parse HEAD RESULT_VARIABLE rev_ok OUTPUT_VARIABLE HEMELB_REVISION_NUMBER OUTPUT_STRIP_TRAILING_WHITESPACE) if (NOT rev_ok EQUAL 0) message("Could not get revision number from Git, looking for revision_info.txt") diff --git a/Code/cmake/cxx11.cmake b/Code/cmake/cxx11.cmake new file mode 100644 index 000000000..d063ae308 --- /dev/null +++ b/Code/cmake/cxx11.cmake @@ -0,0 +1,8 @@ +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +set(CXX11_FLAGS -std=c++11) + +if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX11_FLAGS}") +endif() diff --git a/Code/cmake/intel_cpp11.cmake b/Code/cmake/intel_cpp11.cmake deleted file mode 100644 index d5c97939e..000000000 --- a/Code/cmake/intel_cpp11.cmake +++ /dev/null @@ -1,3 +0,0 @@ -if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel") - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") -endif() diff --git a/Code/cmake/mpi.cmake b/Code/cmake/mpi.cmake index e11b4f093..c05799e86 100644 --- a/Code/cmake/mpi.cmake +++ b/Code/cmake/mpi.cmake @@ -10,20 +10,55 @@ set(CMAKE_CXX_COMPILE_FLAGS "${CMAKE_CXX_COMPILE_FLAGS} ${MPI_COMPILE_FLAGS}") set(CMAKE_CXX_LINK_FLAGS "${CMAKE_CXX_LINK_FLAGS} ${CMAKE_CXX_LINK_FLAGS}") include_directories(${MPI_INCLUDE_PATH}) -# Figure out if this MPI implementation has a const-correct API (supports MPI 3) -set(CMAKE_REQUIRED_FLAGS -Werror) -set(CMAKE_REQUIRED_DEFINITIONS ${MPI_COMPILE_FLAGS}) -set(CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH}) -set(CMAKE_REQUIRED_LIBRARIES ${MPI_LIBRARIES}) -CHECK_CXX_SOURCE_COMPILES("#include +function(TEST_MPI_VERSION_EQUAL ver output_var) + set(CMAKE_REQUIRED_FLAGS "${CXX11_FLAGS} ${CMAKE_CXX_COMPILE_FLAGS}") + set(CMAKE_REQUIRED_INCLUDES "${MPI_INCLUDE_PATH}") + set(CMAKE_REQUIRED_LIBRARIES "${MPI_CXX_LIBRARIES}") + CHECK_CXX_SOURCE_COMPILES("#include int main(int argc, char* argv[]) { - const int send = 0; - int recv; - MPI_Request req; - MPI_Init(&argc, &argv); - MPI_Irecv(&recv, 1, MPI_INT, 0, 0, MPI_COMM_WORLD, &req); - MPI_Send(&send, 1, MPI_INT, 0, 0, MPI_COMM_WORLD); - MPI_Wait(&req, MPI_STATUS_IGNORE); - MPI_Finalize(); -}" HAVE_CONSTCORRECTMPI) + static_assert(MPI_VERSION == ${ver}, \"\"); +}" HAVE_MPI_STANDARD_VERSION_${ver}) + SET(${output_var} ${HAVE_MPI_STANDARD_VERSION_${ver}} PARENT_SCOPE) +endfunction() +function(TEST_MPI_SUBVERSION_EQUAL ver output_var) + set(CMAKE_REQUIRED_FLAGS "${CXX11_FLAGS} ${CMAKE_CXX_COMPILE_FLAGS}") + set(CMAKE_REQUIRED_INCLUDES "${MPI_INCLUDE_PATH}") + set(CMAKE_REQUIRED_LIBRARIES "${MPI_CXX_LIBRARIES}") + CHECK_CXX_SOURCE_COMPILES("#include +int main(int argc, char* argv[]) { + static_assert(MPI_SUBVERSION == ${ver}, \"\"); +}" HAVE_MPI_STANDARD_SUBVERSION_${ver}) + SET(${output_var} ${HAVE_MPI_STANDARD_SUBVERSION_${ver}} PARENT_SCOPE) +endfunction() + +function(GET_MPI_VERSION output_var) + # Future proof as MPI 4.0 will appear eventually + foreach(version RANGE 1 4) + TEST_MPI_VERSION_EQUAL(${version} tmp) + if(${tmp}) + SET(${output_var} ${version} PARENT_SCOPE) + return() + endif() + endforeach() + message(FATAL_ERROR "Could not determine MPI_VERSION") +endfunction() + +function(GET_MPI_SUBVERSION output_var) + foreach(version RANGE 9) + TEST_MPI_SUBVERSION_EQUAL(${version} tmp) + if(${tmp}) + SET(${output_var} ${version} PARENT_SCOPE) + return() + endif() + endforeach() + message(FATAL_ERROR "Could not determine MPI_SUBVERSION") +endfunction() + +GET_MPI_VERSION(MPI_STANDARD_VERSION_MAJOR) +GET_MPI_SUBVERSION(MPI_STANDARD_VERSION_MINOR) +SET(MPI_STANDARD_VERSION "${MPI_STANDARD_VERSION_MAJOR}.${MPI_STANDARD_VERSION_MINOR}") +message(STATUS "MPI library claims to implement standard version ${MPI_STANDARD_VERSION}") + +unset(CMAKE_REQUIRED_FLAGS) +unset(CMAKE_REQUIRED_QUIET) diff --git a/Code/cmake/platform_checks.cmake b/Code/cmake/platform_checks.cmake index d338bb3d3..00c0ca2d3 100644 --- a/Code/cmake/platform_checks.cmake +++ b/Code/cmake/platform_checks.cmake @@ -5,7 +5,6 @@ # license in the file LICENSE. include(gnu_bug) include(mountain_lion_scandir) -include(intel_cpp11) include(CheckCXXSourceCompiles) CHECK_CXX_SOURCE_COMPILES("#include \n int main(int c,char** v){ return isnan(1.0); }" HAVE_ISNAN) diff --git a/Code/colloids/ColloidController.cc b/Code/colloids/ColloidController.cc index a2825de7e..f1c975894 100644 --- a/Code/colloids/ColloidController.cc +++ b/Code/colloids/ColloidController.cc @@ -34,7 +34,7 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, const std::string& outputPath, - const net::IOCommunicator& ioComms_, + comm::Communicator::ConstPtr ioComms_, reporting::Timers& timers) : ioComms(ioComms_), simulationState(simulationState), timers(timers) { @@ -50,10 +50,10 @@ namespace hemelb // determine information about neighbour sites and processors for all local fluid sites InitialiseNeighbourList(latDatLBM, gmyResult, neighbourhood); - bool allGood = ioComms.OnIORank() || (neighbourProcessors.size() > 0); + bool allGood = ioComms->OnIORank() || (neighbourProcessors.size() > 0); log::Logger::Log( "[Rank %i]: ColloidController - neighbourhood %i, neighbours %i, allGood %i\n", - ioComms.Rank(), neighbourhood.size(), neighbourProcessors.size(), allGood); + ioComms->Rank(), neighbourhood.size(), neighbourProcessors.size(), allGood); io::xml::Element particlesElem = xml.GetRoot().GetChildOrThrow("colloids").GetChildOrThrow("particles"); particleSet = new ParticleSet(latDatLBM, particlesElem, propertyCache, @@ -108,7 +108,7 @@ namespace hemelb // if site is local site_t siteId = siteTraverser.GetCurrentIndex(); - if (gmyResult.Blocks[blockId].Sites[siteId].targetProcessor != this->ioComms.Rank()) + if (gmyResult.Blocks[blockId].Sites[siteId].targetProcessor != this->ioComms->Rank()) { log::Logger::Log( "ColloidController: site with id %i and coords (%i,%i,%i) has proc %i (non-local).\n", @@ -143,7 +143,7 @@ namespace hemelb &neighbourBlockId, &neighbourSiteId, &neighbourRank); // if neighbour is remote - if (!isValid || neighbourRank == this->ioComms.Rank()) + if (!isValid || neighbourRank == this->ioComms->Rank()) continue; // if new neighbourRank @@ -159,7 +159,7 @@ namespace hemelb // debug message so this neighbour list can be compared to the LatticeData one log::Logger::Log( "ColloidController: added %i as neighbour for %i because site %i in block %i is neighbour to site %i in block %i in direction (%i,%i,%i)\n", - (int)neighbourRank, (int)(this->ioComms.Rank()), + (int)neighbourRank, (int)(this->ioComms->Rank()), (int)neighbourSiteId, (int)neighbourBlockId, (int)siteId, (int)blockId, (*itDirectionVector).x, (*itDirectionVector).y, (*itDirectionVector).z); diff --git a/Code/colloids/ColloidController.h b/Code/colloids/ColloidController.h index 1cd3b9d2a..92b595457 100644 --- a/Code/colloids/ColloidController.h +++ b/Code/colloids/ColloidController.h @@ -8,8 +8,6 @@ #define HEMELB_COLLOIDS_COLLOIDCONTROLLER_H #include -#include "net/net.h" -#include "net/IteratedAction.h" #include "geometry/LatticeData.h" #include "geometry/Geometry.h" #include "io/xml/XmlAbstractionLayer.h" @@ -17,13 +15,14 @@ #include "colloids/ParticleSet.h" #include "util/Vector3D.h" #include "units.h" +#include "timestep/Actor.h" namespace hemelb { namespace colloids { /** provides the control interface between colloid simulation and the rest of the system */ - class ColloidController : public net::IteratedAction + class ColloidController : public timestep::Actor { public: /** constructor - currently only initialises the neighbour list */ @@ -34,23 +33,40 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, const std::string& outputPath, - const net::IOCommunicator& ioComms_, + comm::Communicator::ConstPtr ioComms_, reporting::Timers& timers); /** destructor - releases resources allocated by this class */ ~ColloidController(); - /** overloaded from IteratedAction */ void RequestComms(); - /** overloaded from IteratedAction */ void EndIteration(); const void OutputInformation(const LatticeTimeStep timestep) const; - + inline virtual void BeginAll() { + } + inline virtual void Begin() { + } + inline virtual void Receive() { + } + inline virtual void PreSend() { + } + inline virtual void Send() { + } + inline virtual void PreWait() { + } + + inline virtual void Wait() { + } + + inline virtual void End() { + } + virtual void EndAll() { + } private: /** Main code communicator */ - const net::IOCommunicator& ioComms; + comm::Communicator::ConstPtr ioComms; const lb::SimulationState& simulationState; diff --git a/Code/colloids/Particle.h b/Code/colloids/Particle.h index db460d781..1e9468679 100644 --- a/Code/colloids/Particle.h +++ b/Code/colloids/Particle.h @@ -7,7 +7,7 @@ #ifndef HEMELB_COLLOIDS_PARTICLE_H #define HEMELB_COLLOIDS_PARTICLE_H -#include "net/mpi.h" +#include "comm/MpiDataType.h" #include "colloids/PersistedParticle.h" #include "geometry/LatticeData.h" #include "io/xml/XmlAbstractionLayer.h" @@ -27,7 +27,7 @@ namespace hemelb * all persisted properties, i.e. those that are read in from a config file, * are inherited from the PersistedParticle class (which handles the I/O) */ - class Particle : PersistedParticle + class Particle : public PersistedParticle { public: /** constructor - gets initial values from an xml configuration file */ @@ -195,7 +195,7 @@ namespace hemelb friend struct ParticleSorter; }; } - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); diff --git a/Code/colloids/ParticleMpiDatatypes.cc b/Code/colloids/ParticleMpiDatatypes.cc index a3610a00a..02f08af2d 100644 --- a/Code/colloids/ParticleMpiDatatypes.cc +++ b/Code/colloids/ParticleMpiDatatypes.cc @@ -9,7 +9,7 @@ namespace hemelb { - namespace net + namespace comm { // boiler-plate template specialisation for colloids Particle object template<> @@ -138,7 +138,7 @@ namespace hemelb MPI_DOUBLE, MPI_UNSIGNED_LONG, MPI_UNSIGNED_LONG, - net::MpiDataType > () }; + comm::MpiDataType > () }; // create a first draft of the MPI datatype for a Particle // the lower bound and displacements of fields are correct @@ -205,7 +205,7 @@ namespace hemelb displacementOfEachFieldBlock[1] -= baseAddress; // the built-in MPI datatype of each field must match the C++ type - MPI_Datatype datatypeOfEachFieldBlock[] = { MPI_UNSIGNED_LONG, net::MpiDataType > () }; // create a first draft of the MPI datatype for a Particle diff --git a/Code/colloids/ParticleSet.cc b/Code/colloids/ParticleSet.cc index 35cf8b2bf..cc9700e35 100644 --- a/Code/colloids/ParticleSet.cc +++ b/Code/colloids/ParticleSet.cc @@ -12,6 +12,7 @@ #include "io/writers/xdr/XdrMemWriter.h" #include "io/formats/formats.h" #include "io/formats/colloids.h" +#include "comm/Async.h" namespace hemelb { @@ -43,16 +44,16 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, std::vector& neighbourProcessors, - const net::IOCommunicator& ioComms_, + comm::Communicator::ConstPtr ioComms_, const std::string& outputPath) : - ioComms(ioComms_), localRank(ioComms.Rank()), latDatLBM(latDatLBM), propertyCache(propertyCache), path(outputPath), net(ioComms) + ioComms(ioComms_), localRank(ioComms->Rank()), latDatLBM(latDatLBM), propertyCache(propertyCache), path(outputPath) { /** * Open the file, unless it already exists, for writing only, creating it if it doesn't exist. */ - file = net::MpiFile::Open(ioComms, path, MPI_MODE_EXCL | MPI_MODE_WRONLY | MPI_MODE_CREATE); + file = ioComms->OpenFile(path, MPI_MODE_EXCL | MPI_MODE_WRONLY | MPI_MODE_CREATE); - if (ioComms.OnIORank()) + if (ioComms->OnIORank()) { // Write out the header information from core 0 buffer.resize(io::formats::colloids::MagicLength); @@ -60,10 +61,10 @@ namespace hemelb writer << (uint32_t) io::formats::HemeLbMagicNumber; writer << (uint32_t) io::formats::colloids::MagicNumber; writer << (uint32_t) io::formats::colloids::VersionNumber; - file.Write(buffer); + file->Write(buffer); } - HEMELB_MPI_CALL(MPI_File_seek_shared, (file, 0, MPI_SEEK_END)); + HEMELB_MPI_CALL(MPI_File_seek_shared, (*file, 0, MPI_SEEK_END)); // add an element into scanMap for each neighbour rank with zero for both counts // sorting the list of neighbours allows the position in the map to be predicted @@ -132,35 +133,35 @@ namespace hemelb // Find how far we currently are into the file. MPI_Offset positionBeforeWriting; - HEMELB_MPI_CALL(MPI_File_get_position_shared, (file, &positionBeforeWriting)); + HEMELB_MPI_CALL(MPI_File_get_position_shared, (*file, &positionBeforeWriting)); log::Logger::Log("from offsetEOF: %i\n", positionBeforeWriting); // Go past the header (which we'll write at the end) unsigned int sizeOfHeader = io::formats::colloids::HeaderLength; - HEMELB_MPI_CALL(MPI_File_seek_shared, (file, sizeOfHeader, MPI_SEEK_END)); + HEMELB_MPI_CALL(MPI_File_seek_shared, (*file, sizeOfHeader, MPI_SEEK_END)); // Collective write: the effect is as though all writes are done // in serialised order, i.e. as if rank 0 writes first, followed // by rank 1, and so on, until all ranks have written their data - HEMELB_MPI_CALL(MPI_File_write_ordered, (file, &buffer.front(), count, MPI_CHAR, MPI_STATUS_IGNORE)); + HEMELB_MPI_CALL(MPI_File_write_ordered, (*file, &buffer.front(), count, MPI_CHAR, MPI_STATUS_IGNORE)); // the collective ordered write modifies the shared file pointer // it should point to the byte following the highest rank's data // (should be true for all ranks but) we only need it for rank 0 MPI_Offset positionAferWriting; - HEMELB_MPI_CALL(MPI_File_get_position_shared, (file, &positionAferWriting)); + HEMELB_MPI_CALL(MPI_File_get_position_shared, (*file, &positionAferWriting)); log::Logger::Log("new offsetEOF: %i\n", positionBeforeWriting); // Now write the header section, only on rank 0. - if (ioComms.OnIORank()) + if (ioComms->OnIORank()) { writer << (uint32_t) io::formats::colloids::HeaderLength; writer << (uint32_t) io::formats::colloids::RecordLength; writer << (uint64_t) (positionAferWriting - positionBeforeWriting - io::formats::colloids::HeaderLength); writer << (uint64_t) timestep; - HEMELB_MPI_CALL(MPI_File_write_at, (file, positionBeforeWriting, &buffer[count], sizeOfHeader, MPI_CHAR, MPI_STATUS_IGNORE)); + HEMELB_MPI_CALL(MPI_File_write_at, (*file, positionBeforeWriting, &buffer[count], sizeOfHeader, MPI_CHAR, MPI_STATUS_IGNORE)); } for (scanMapConstIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) @@ -285,19 +286,22 @@ namespace hemelb { return; } - - for (scanMapIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) + { - const proc_t& neighbourRank = iterMap->first; - if (neighbourRank != localRank) - { - unsigned int& numberOfParticlesToRecv = iterMap->second.first; - net.RequestSendR(numberOfParticlesToSend, neighbourRank); - net.RequestReceiveR(numberOfParticlesToRecv, neighbourRank); - } + comm::Async commQ(ioComms); + + for (scanMapIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) + { + const proc_t& neighbourRank = iterMap->first; + if (neighbourRank != localRank) + { + unsigned int& numberOfParticlesToRecv = iterMap->second.first; + commQ.Isend(numberOfParticlesToSend, neighbourRank); + commQ.Irecv(numberOfParticlesToRecv, neighbourRank); + } } - net.Dispatch(); - + } + unsigned int numberOfParticles = 0; for (scanMapConstIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) numberOfParticles += iterMap->second.first; @@ -305,18 +309,21 @@ namespace hemelb std::vector::iterator iterSendBegin = particles.begin(); std::vector::iterator iterRecvBegin = particles.begin() + numberOfParticlesToSend; + // Handle the request list ourselves to force the template instaniation to be correct. + auto commQ = ioComms->MakeRequestList(); for (scanMapConstIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) { const proc_t& neighbourRank = iterMap->first; if (neighbourRank != localRank) { const unsigned int& numberOfParticlesToRecv = iterMap->second.first; - net.RequestSend(& ((PersistedParticle&) *iterSendBegin), numberOfParticlesToSend, neighbourRank); - net.RequestReceive(& ((PersistedParticle&) * (iterRecvBegin)), numberOfParticlesToRecv, neighbourRank); + commQ->push_back(ioComms->Isend(&*iterSendBegin, numberOfParticlesToSend, neighbourRank)); + commQ->push_back(ioComms->Irecv(&*iterRecvBegin, numberOfParticlesToRecv, neighbourRank)); + iterRecvBegin += numberOfParticlesToRecv; } } - net.Dispatch(); + commQ->WaitAll(); // remove particles owned by unknown ranks std::vector::iterator newEndOfParticles = @@ -351,20 +358,22 @@ namespace hemelb { return; } - - // exchange counts - for (scanMapIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) + { - const proc_t& neighbourRank = iterMap->first; - if (neighbourRank != localRank) - { - unsigned int& numberOfVelocitiesToSend = iterMap->second.first; - unsigned int& numberOfVelocitiesToRecv = iterMap->second.second; - net.RequestSendR(numberOfVelocitiesToSend, neighbourRank); - net.RequestReceiveR(numberOfVelocitiesToRecv, neighbourRank); - } + comm::Async commQ(ioComms); + // exchange counts + for (scanMapIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) + { + const proc_t& neighbourRank = iterMap->first; + if (neighbourRank != localRank) + { + unsigned int& numberOfVelocitiesToSend = iterMap->second.first; + unsigned int& numberOfVelocitiesToRecv = iterMap->second.second; + commQ.Isend(numberOfVelocitiesToSend, neighbourRank); + commQ.Irecv(numberOfVelocitiesToRecv, neighbourRank); + } + } } - net.Dispatch(); // sum counts unsigned int numberOfIncomingVelocities = 0; @@ -373,6 +382,7 @@ namespace hemelb velocityBuffer.resize(numberOfIncomingVelocities); // exchange velocities + auto commQ = ioComms->MakeRequestList(); std::vector::iterator iterSendBegin = particles.begin(); std::vector > >::iterator iterRecvBegin = velocityBuffer.begin(); for (scanMapConstIterType iterMap = scanMap.begin(); iterMap != scanMap.end(); iterMap++) @@ -382,12 +392,12 @@ namespace hemelb { const unsigned int& numberOfVelocitiesToSend = iterMap->second.first; const unsigned int& numberOfVelocitiesToRecv = iterMap->second.second; - net.RequestSend(& ((Particle&) *iterSendBegin), numberOfVelocitiesToSend, neighbourRank); - net.RequestReceive(& (* (iterRecvBegin)), numberOfVelocitiesToRecv, neighbourRank); + commQ->push_back(ioComms->Isend(&*iterSendBegin, numberOfVelocitiesToSend, neighbourRank)); + commQ->push_back(ioComms->Irecv(&*iterRecvBegin, numberOfVelocitiesToRecv, neighbourRank)); iterRecvBegin += numberOfVelocitiesToRecv; } } - net.Dispatch(); + commQ->WaitAll(); // sum velocities velocityMap.clear(); diff --git a/Code/colloids/ParticleSet.h b/Code/colloids/ParticleSet.h index 4eb1fb512..196fa1245 100644 --- a/Code/colloids/ParticleSet.h +++ b/Code/colloids/ParticleSet.h @@ -11,9 +11,8 @@ #include "geometry/LatticeData.h" #include "io/xml/XmlAbstractionLayer.h" #include "lb/MacroscopicPropertyCache.h" -#include "net/mpi.h" #include "colloids/Particle.h" -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" #include "units.h" namespace hemelb @@ -30,7 +29,7 @@ namespace hemelb lb::MacroscopicPropertyCache& propertyCache, const hemelb::lb::LbmParameters *lbmParams, std::vector& neighbourProcessors, - const net::IOCommunicator& ioComms_, + comm::Communicator::ConstPtr ioComms_, const std::string& outputPath); /** destructor - de-allocates all Particle objects created by this Set */ @@ -61,7 +60,7 @@ namespace hemelb const void OutputInformation(const LatticeTimeStep timestep); private: - const net::IOCommunicator& ioComms; + comm::Communicator::ConstPtr ioComms; /** cached copy of local rank (obtained from topology) */ const proc_t localRank; @@ -94,8 +93,6 @@ namespace hemelb */ lb::MacroscopicPropertyCache& propertyCache; - /** abstracts communication via MPI */ - net::Net net; /** * Reusable output buffer. */ @@ -107,7 +104,7 @@ namespace hemelb /** * MPI File handle to write with */ - net::MpiFile file; + comm::MpiFile::Ptr file; }; } } diff --git a/Code/colloids/PersistedParticle.h b/Code/colloids/PersistedParticle.h index 2e7303e8b..cb12c3f9d 100644 --- a/Code/colloids/PersistedParticle.h +++ b/Code/colloids/PersistedParticle.h @@ -9,7 +9,7 @@ #include "io/xml/XmlAbstractionLayer.h" #include "units.h" -#include "net/MpiDataType.h" +#include "comm/MpiDataType.h" namespace hemelb { @@ -61,7 +61,7 @@ namespace hemelb LatticePosition globalPosition; }; } - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); diff --git a/Code/comm/Async.h b/Code/comm/Async.h new file mode 100644 index 000000000..0301c6c81 --- /dev/null +++ b/Code/comm/Async.h @@ -0,0 +1,71 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_COMM_ASYNC_H +#define HEMELB_COMM_ASYNC_H + +#include +#include "comm/Communicator.h" +#include "comm/Request.h" + +namespace hemelb +{ + namespace comm + { + // Simple class that manages a bunch of asynchronous + // communications. Construct it and add the comms. Their + // completion is waited on when the object is destructed. + class Async + { + public: + typedef std::shared_ptr Ptr; + + inline static Ptr New(Communicator::ConstPtr c) + { + return std::make_shared(c); + } + + inline Async(Communicator::ConstPtr c) : comms(c), q(c->MakeRequestList()) + { + } + inline ~Async() + { + q->WaitAll(); + } + + inline void Wait() { + q->WaitAll(); + q->clear(); + } + + inline Communicator::ConstPtr GetComm() const { + return comms; + } + + template + void Isend(Ts&&... args) + { + q->push_back(comms->Isend(std::forward(args)...)); + } + template + void Issend(Ts&&... args) + { + q->push_back(comms->Issend(std::forward(args)...)); + } + template + void Irecv(Ts&&... args) + { + q->push_back(comms->Irecv(std::forward(args)...)); + } + + private: + Communicator::ConstPtr comms; + RequestList::Ptr q; + }; + + } +} + +#endif diff --git a/Code/comm/AsyncConcern.h b/Code/comm/AsyncConcern.h new file mode 100644 index 000000000..2b0baf093 --- /dev/null +++ b/Code/comm/AsyncConcern.h @@ -0,0 +1,51 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_COMM_ASYNC_CONCERN_H +#define HEMELB_COMM_ASYNC_CONCERN_H + +#include "comm/Async.h" +#include "timestep/Actor.h" + +namespace hemelb +{ + namespace comm + { + class AsyncConcern : public timestep::Actor + { + public: + AsyncConcern(Async::Ptr async) : mAsync(async) + { + } + inline virtual void BeginAll() { + } + inline virtual void Begin() { + } + inline virtual void Receive() { + } + inline virtual void PreSend() { + } + inline virtual void Send() { + } + inline virtual void PreWait() { + } + + inline virtual void Wait() { + mAsync->Wait(); + } + + inline virtual void End() { + } + inline virtual void EndAll() { + } + + private: + Async::Ptr mAsync; + }; + + } +} + +#endif diff --git a/Code/comm/CMakeLists.txt b/Code/comm/CMakeLists.txt new file mode 100644 index 000000000..42f997e88 --- /dev/null +++ b/Code/comm/CMakeLists.txt @@ -0,0 +1,10 @@ + +# This file is part of HemeLB and is Copyright (C) +# the HemeLB team and/or their institutions, as detailed in the +# file AUTHORS. This software is provided under the terms of the +# license in the file LICENSE. +add_library(hemelb_comm + MpiDataType.cc MpiEnvironment.cc MpiError.cc + MpiCommunicator.cc MpiGroup.cc MpiFile.cc + MpiRequest.cc +) diff --git a/Code/comm/Communicator.h b/Code/comm/Communicator.h new file mode 100644 index 000000000..a487c68bf --- /dev/null +++ b/Code/comm/Communicator.h @@ -0,0 +1,214 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_COMM_COMMUNICATOR_H +#define HEMELB_COMM_COMMUNICATOR_H + +#include +#include +#include + +namespace hemelb +{ + // TODO: remove when net is dead + namespace net + { + class CoalescePointPoint; + class SeparatedPointPoint; + class ImmediatePointPoint; + } + namespace comm + { + class Group; + class Request; + class RequestList; + class MpiFile; + + // Base class for communicators (MPI, null, and mock) + class Communicator + { + public: + typedef std::shared_ptr Ptr; + typedef std::shared_ptr ConstPtr; + /** + * Class has virtual methods so should have virtual d'tor. + */ + virtual ~Communicator() {}; + + /** + * Returns the local rank on the communicator + * @return + */ + virtual int Rank() const = 0; + + inline bool OnIORank() const + { + return Rank() == GetIORank(); + } + + inline int GetIORank() const + { + return 0; + } + + /** + * Returns the size of the communicator (i.e. total number of procs involved). + * @return + */ + virtual int Size() const = 0; + + /** + * Abort - should try to bring down all tasks, but no guarantees + * @param errCode + */ + virtual void Abort(int errCode) const = 0; + + /** + * Duplicate the communicator - see MPI_COMM_DUP + * @return + */ + virtual Ptr Duplicate() const = 0; + + virtual std::shared_ptr GetGroup() const = 0; + virtual Ptr Create(std::shared_ptr grp) const = 0; + /** + * Opens a file with MPI_File_open. A collective operation + * @param filename + * @param mode + * @param info + * @return + */ + virtual std::shared_ptr OpenFile(const std::string& filename, int mode, + const MPI_Info info = MPI_INFO_NULL) const = 0; + + virtual std::shared_ptr MakeRequestList() const = 0; + virtual void Barrier() const = 0; + virtual std::shared_ptr Ibarrier() const = 0; + + virtual bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const = 0; + + template + void Broadcast(T& val, const int root) const; + template + void Broadcast(std::vector& vals, const int root) const; + + template + std::shared_ptr Ibcast(T& val, const int root) const; + template + std::shared_ptr Ibcast(std::vector& vals, const int root) const; + + template + T AllReduce(const T& val, const MPI_Op& op) const; + template + std::vector AllReduce(const std::vector& vals, const MPI_Op& op) const; + + template + std::shared_ptr Iallreduce(const T& val, const MPI_Op& op, T& out) const; + + template + std::shared_ptr Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const; + + template + T Reduce(const T& val, const MPI_Op& op, const int root) const; + template + std::vector Reduce(const std::vector& vals, const MPI_Op& op, const int root) const; + + template + std::vector Gather(const T& val, const int root) const; + + template + std::vector GatherV(const std::vector senddata, const std::vector recvcounts, + const int root) const; + template + std::vector AllGatherV(const std::vector senddata, const std::vector recvcounts) const; + + template + std::vector AllGather(const T& val) const; + + template + std::vector AllToAll(const std::vector& vals) const; + + template + void Send(const std::vector& val, int dest, int tag=0) const; + template + void Send(const T& val, int dest, int tag=0) const; + template + void Send(const T* valPtr, int count, int dest, int tag) const; + + template + void Recv(std::vector& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + template + void Recv(T& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + template + void Recv(T* val, int count, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; + + template + std::shared_ptr Isend(const T& val, int dest, int tag=0) const; + template + std::shared_ptr Isend(const std::vector& vals, int dest, int tag=0) const; + template + std::shared_ptr Isend(const T* valPtr, int count, int dest, int tag=0) const; + + template + std::shared_ptr Issend(const T& val, int dest, int tag=0) const; + template + std::shared_ptr Issend(const std::vector& vals, int dest, int tag=0) const; + template + std::shared_ptr Issend(const T* valPtr, int count, int dest, int tag=0) const; + + template + std::shared_ptr Irecv(T& val, int source, int tag=0) const; + template + std::shared_ptr Irecv(std::vector& vals, int source, int tag=0) const; + template + std::shared_ptr Irecv(T* valPtr, int count, int source, int tag=0) const; + + + protected: + virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; + virtual std::shared_ptr IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const = 0; + virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const = 0; + virtual std::shared_ptr IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const = 0; + virtual std::shared_ptr IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const = 0; + virtual void ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const = 0; + virtual void GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype, + int root) const = 0; + virtual void GathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype, + int root) const = 0; + virtual void AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const = 0; + virtual void AllgathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype) const = 0; + virtual void AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const = 0; + virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const = 0; + virtual void SsendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const = 0; + virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int src, int tag, MPI_Status* stat) const = 0; + virtual std::shared_ptr IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const = 0; + virtual std::shared_ptr IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const = 0; + virtual std::shared_ptr IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int tag) const = 0; + }; + + // bool operator==(const Communicator& comm1, const Communicator& comm2); + // bool operator!=(const Communicator& comm1, const Communicator& comm2); + + template + struct is_std_vector : public std::false_type {}; + template + struct is_std_vector > : public std::true_type {}; + } +} + +#include "comm/Communicator.hpp" + +#endif /* HEMELB_COMM_MPICOMMUNICATOR_H */ diff --git a/Code/comm/Communicator.hpp b/Code/comm/Communicator.hpp new file mode 100644 index 000000000..47617181d --- /dev/null +++ b/Code/comm/Communicator.hpp @@ -0,0 +1,285 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. +#ifndef HEMELB_COMM_COMMUNICATOR_HPP +#define HEMELB_COMM_COMMUNICATOR_HPP + +#include "comm/MpiDataType.h" + +namespace hemelb +{ + namespace comm + { + template + void Communicator::Broadcast(T& val, const int root) const + { + BcastImpl(&val, 1, MpiDataType(), root); + } + template + void Communicator::Broadcast(std::vector& vals, const int root) const + { + BcastImpl(vals.data(), vals.size(), MpiDataType(), root); + } + + template + std::shared_ptr Communicator::Ibcast(T& val, const int root) const + { + return IbcastImpl(&val, 1, MpiDataType(), root); + } + template + std::shared_ptr Communicator::Ibcast(std::vector& vals, const int root) const + { + return IbcastImpl(vals.data(), vals.size(), MpiDataType(), root); + } + + template + T Communicator::AllReduce(const T& val, const MPI_Op& op) const + { + T ans; + AllreduceImpl(&val, &ans, 1, MpiDataType(), op); + return ans; + } + + template + std::vector Communicator::AllReduce(const std::vector& vals, const MPI_Op& op) const + { + std::vector ans(vals.size()); + AllreduceImpl(vals.data(), ans.data(), vals.size(), MpiDataType(), op); + return ans; + } + + template + std::shared_ptr Communicator::Iallreduce(const T& val, const MPI_Op& op, T& out) const + { + return IallreduceImpl(&val, &out, 1, MpiDataType(), op); + } + + template + std::shared_ptr Communicator::Ireduce(const T& val, const MPI_Op& op, const int root, T& out) const + { + return IreduceImpl(&val, &out, 1, MpiDataType(), op, root); + } + + template + T Communicator::Reduce(const T& val, const MPI_Op& op, const int root) const + { + T ans; + ReduceImpl(&val, &ans, 1, MpiDataType(), op, root); + return ans; + } + + template + std::vector Communicator::Reduce(const std::vector& vals, const MPI_Op& op, + const int root) const + { + std::vector ans; + T* recvbuf = nullptr; + + if (Rank() == root) + { + // Standard says the address of receive buffer only matters at the root. + ans.resize(vals.size()); + recvbuf = ans.data(); + } + + ReduceImpl(vals.data(), recvbuf, vals.size(), MpiDataType(), op, root); + return ans; + } + + template + std::vector Communicator::Gather(const T& val, const int root) const + { + std::vector ans; + T* recvbuf = nullptr; + + if (Rank() == root) + { + // Standard says the address of receive buffer only matters at the root. + ans.resize(Size()); + recvbuf = ans.data(); + } + + GatherImpl(&val, 1, MpiDataType(), + recvbuf, 1, MpiDataType(), + root); + return ans; + } + + template + std::vector Communicator::GatherV(const std::vector senddata, + const std::vector recvcounts, + const int root) const + { + const int np = Size(); + const int sendcount = senddata.size(); + std::vector displs; + std::vector ans; + T* recvbuf = nullptr; + if (Rank() == root) + { + // Compute the displacements from the counts + displs.resize(np); + int total = 0; + for(size_t i = 0; i < np; ++i) { + displs[i] = total; + total += recvcounts[i]; + } + // set up recv buffer + ans.resize(total); + recvbuf = ans.data(); + } + + GathervImpl(senddata.data(), sendcount, MpiDataType(), + recvbuf, recvcounts.data(), displs.data(), MpiDataType(), + root); + return ans; + } + + template + std::vector Communicator::AllGatherV(const std::vector senddata, + const std::vector recvcounts) const + { + const int np = Size(); + const int sendcount = senddata.size(); + std::vector displs; + // Compute the displacements from the counts + displs.resize(np); + int total = 0; + for(size_t i = 0; i < np; ++i) { + displs[i] = total; + total += recvcounts[i]; + } + // set up recv buffer + std::vector ans(total); + + AllgathervImpl(senddata.data(), sendcount, MpiDataType(), + ans.data(), recvcounts.data(), displs.data(), MpiDataType()); + return ans; + } + + template + std::vector Communicator::AllGather(const T& val) const + { + std::vector ans(Size()); + T* recvbuf = ans.data(); + + AllgatherImpl(&val, 1, MpiDataType(), + recvbuf, 1, MpiDataType()); + return ans; + } + + template + std::vector Communicator::AllToAll(const std::vector& vals) const + { + std::vector ans(vals.size()); + AlltoallImpl(vals.data(), 1, MpiDataType(), + ans.data(), 1, MpiDataType()); + return ans; + } + + // Send implementations + template + void Communicator::Send(const T* valPtr, int count, int dest, int tag) const + { + SendImpl(valPtr, count, MpiDataType(), dest, tag); + } + template + void Communicator::Send(const T& val, int dest, int tag) const + { + Send(&val, 1, dest, tag); + } + template + void Communicator::Send(const std::vector& vals, int dest, int tag) const + { + Send(vals.data(), vals.size(), dest, tag); + } + + // Recv implementations + template + void Communicator::Recv(T* valPtr, int count, int src, int tag, MPI_Status* stat) const + { + RecvImpl(valPtr, count, MpiDataType(), src, tag, stat); + } + template + void Communicator::Recv(T& val, int src, int tag, MPI_Status* stat) const + { + Recv(&val, 1, src, tag, stat); + } + template + void Communicator::Recv(std::vector& vals, int src, int tag, MPI_Status* stat) const + { + Recv(vals.data(), vals.size(), src, tag, stat); + } + + // Isend implementations + template + std::shared_ptr Communicator::Isend(const T* valPtr, int count, int dest, int tag) const + { + static_assert(!std::is_pointer::value, + "You are trying to communicate a pointer type"); + static_assert(!is_std_vector::value, + "You are trying to communicate a std::vector type"); + + return IsendImpl(valPtr, count, MpiDataType(), dest, tag); + } + template + std::shared_ptr Communicator::Isend(const T& val, int dest, int tag) const + { + return Isend(&val, 1, dest, tag); + } + template + std::shared_ptr Communicator::Isend(const std::vector& vals, int dest, int tag) const + { + return Isend(vals.data(), vals.size(), dest, tag); + } + + // Issend implementations + template + std::shared_ptr Communicator::Issend(const T* valPtr, int count, int dest, int tag) const + { + static_assert(!std::is_pointer::value, + "You are trying to communicate a pointer type"); + static_assert(!is_std_vector::value, + "You are trying to communicate a std::vector type"); + + return IssendImpl(valPtr, count, MpiDataType(), dest, tag); + } + template + std::shared_ptr Communicator::Issend(const T& val, int dest, int tag) const + { + return Issend(&val, 1, dest, tag); + } + template + std::shared_ptr Communicator::Issend(const std::vector& vals, int dest, int tag) const + { + return Issend(vals.data(), vals.size(), dest, tag); + } + + // Irecv implementations + template + std::shared_ptr Communicator::Irecv(T* valPtr, int count, int source, int tag) const + { + static_assert(!std::is_pointer::value, + "You are trying to communicate a pointer type"); + static_assert(!is_std_vector::value, + "You are trying to communicate a std::vector type"); + + return IrecvImpl(valPtr, count, MpiDataType(), source, tag); + } + template + std::shared_ptr Communicator::Irecv(T& val, int source, int tag) const + { + return Irecv(&val, 1, source, tag); + } + template + std::shared_ptr Communicator::Irecv(std::vector& vals, int source, int tag) const + { + return Irecv(&vals[0], vals.size(), source, tag); + } + + } +} + +#endif // HEMELB_COMM_COMMUNICATOR_HPP diff --git a/Code/comm/Group.h b/Code/comm/Group.h new file mode 100644 index 000000000..fb9667d8a --- /dev/null +++ b/Code/comm/Group.h @@ -0,0 +1,50 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_COMM_GROUP_H +#define HEMELB_COMM_GROUP_H + +#include "units.h" +#include +#include + +namespace hemelb +{ + namespace comm + { + class Group + { + public: + typedef std::shared_ptr Ptr; + /** + * Returns the local rank within the group + * @return + */ + virtual int Rank() const = 0; + + /** + * Returns the size of the group + * @return + */ + virtual int Size() const = 0; + + /** + * Exclude the provided ranks + * @param ranksToExclude + * @return + */ + virtual Group::Ptr Exclude(const std::vector& ranksToExclude) const = 0; + /** + * Include the provided ranks + * @param ranksToExclude + * @return + */ + virtual Group::Ptr Include(const std::vector& ranksToInclude) const = 0; + }; + } +} + +#endif /* HEMELB_COMM_GROUP_H */ diff --git a/Code/comm/MapAllToAll.h b/Code/comm/MapAllToAll.h new file mode 100644 index 000000000..eebe86b68 --- /dev/null +++ b/Code/comm/MapAllToAll.h @@ -0,0 +1,88 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_NET_MAPALLTOALL_H +#define HEMELB_NET_MAPALLTOALL_H + +#include +#include "comm/Communicator.h" +#include "comm/Request.h" + +namespace hemelb +{ + namespace comm + { + template + void MapAllToAll(Communicator::ConstPtr comm, + const std::map& valsToSend, + std::map& receivedVals, + const int tag = 1234) + { + receivedVals.clear(); + + auto sendReqs = comm->MakeRequestList(); + int localRank = comm->Rank(); + int nSends = valsToSend.size(); + // Is localRank in the map of send destinations? + if (valsToSend.find(localRank) != valsToSend.end()) + // If so, reduce the number of sends by 1. + nSends -= 1; + // Set up a container for all the Status objs + sendReqs->resize(nSends); + + typename std::map::const_iterator iter = valsToSend.begin(), + end = valsToSend.end(); + int i = 0; + for (; iter != end; ++iter) { + int rank = iter->first; + const T& val = iter->second; + if (rank == localRank) + { + receivedVals[localRank] = val; + } + else + { + // Synchronous to ensure we know when this is matched + sendReqs->set(i, comm->Issend(val, rank, tag)); + i++; + } + } + + bool allProcsHaveHadAllSendsMatched = false; + bool mySendsMatched = false; + Request::Ptr barrierReq; + + // Allocate outside of loop to ensure compiler isn't reinitialising this + // all the time. + MPI_Status status; + while(!mySendsMatched || !allProcsHaveHadAllSendsMatched) + { + if (comm->Iprobe(MPI_ANY_SOURCE, tag, &status)) { + // There is a message waiting + comm->Recv(receivedVals[status.MPI_SOURCE], status.MPI_SOURCE, status.MPI_TAG); + } + + if (!mySendsMatched) + { + if (sendReqs->TestAll()) + { + mySendsMatched = true; + barrierReq = comm->Ibarrier(); + } + } + else + { + allProcsHaveHadAllSendsMatched = barrierReq->Test(); + } + } + } + } +} + +#endif /* HEMELB_NET_MAPALLTOALL_H */ diff --git a/Code/comm/MpiCommunicator.cc b/Code/comm/MpiCommunicator.cc new file mode 100644 index 000000000..3b299e82e --- /dev/null +++ b/Code/comm/MpiCommunicator.cc @@ -0,0 +1,317 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#include "comm/MpiCommunicator.h" +#include "comm/MpiError.h" +#include "comm/MpiFile.h" +#include "comm/MpiRequest.h" +#include "comm/MpiGroup.h" + +namespace hemelb +{ + namespace comm + { + + namespace + { + void Deleter(MPI_Comm* comm) + { + int finalized; + HEMELB_MPI_CALL(MPI_Finalized, (&finalized)); + if (!finalized) + HEMELB_MPI_CALL(MPI_Comm_free, (comm)); + delete comm; + } + } + + MpiCommunicator::MpiCommunicator() : commPtr(), rank(0), size(0) + { + } + + MpiCommunicator::MpiCommunicator(MPI_Comm communicator, bool owner) : commPtr(), rank(0), size(0) + { + if (communicator == MPI_COMM_NULL) + return; + + if (owner) + { + commPtr.reset(new MPI_Comm(communicator), Deleter); + } + else + { + commPtr.reset(new MPI_Comm(communicator)); + } + + HEMELB_MPI_CALL(MPI_Comm_rank, (*commPtr, &rank)); + HEMELB_MPI_CALL(MPI_Comm_size, (*commPtr, &size)); + } + + MpiCommunicator::operator MPI_Comm() const + { + return *commPtr; + } + + MpiCommunicator::~MpiCommunicator() + { + } + int MpiCommunicator::Rank() const + { + return rank; + } + + int MpiCommunicator::Size() const + { + return size; + } + + void MpiCommunicator::Abort(int errCode) const + { + HEMELB_MPI_CALL(MPI_Abort, (*commPtr, errCode)); + } + + Communicator::Ptr MpiCommunicator::Duplicate() const + { + MPI_Comm newComm; + HEMELB_MPI_CALL(MPI_Comm_dup, (*commPtr, &newComm)); + return std::make_shared(newComm, true); + } + Group::Ptr MpiCommunicator::GetGroup() const + { + MPI_Group grp; + HEMELB_MPI_CALL(MPI_Comm_group, (*commPtr, &grp)); + return std::make_shared(grp, true); + } + + Communicator::Ptr MpiCommunicator::Create(std::shared_ptr grp) const + { + auto mpiGrp = std::dynamic_pointer_cast(grp); + + MPI_Comm newComm; + HEMELB_MPI_CALL(MPI_Comm_create, (*commPtr, *mpiGrp, &newComm)); + return std::make_shared(newComm, true); + } + + MpiFile::Ptr MpiCommunicator::OpenFile(const std::string& filename, int mode, + const MPI_Info info) const + { + MPI_File ans; + HEMELB_MPI_CALL( + MPI_File_open, + (*this, filename.c_str(), mode, info, &ans) + ); + return std::make_shared(shared_from_this(), ans); + } + + RequestList::Ptr MpiCommunicator::MakeRequestList() const + { + return std::make_shared(); + } + + void MpiCommunicator::Barrier() const + { + HEMELB_MPI_CALL(MPI_Barrier, (*commPtr)); + } + + std::shared_ptr MpiCommunicator::Ibarrier() const + { + MPI_Request req; + HEMELB_MPI_CALL(MPI_Ibarrier, (*commPtr, &req)); + return std::make_shared(req); + } + + bool MpiCommunicator::Iprobe(int source, int tag, MPI_Status* stat) const + { + int flag; + HEMELB_MPI_CALL( + MPI_Iprobe, + (source, tag, *commPtr, &flag, stat) + ); + return flag; + } + + + void MpiCommunicator::BcastImpl(void* buf, int count, MPI_Datatype dt, + int root) const + { + HEMELB_MPI_CALL( + MPI_Bcast, + (buf, count, dt, root, *commPtr) + ); + } + + std::shared_ptr MpiCommunicator::IbcastImpl(void* buf, int count, MPI_Datatype dt, + int root) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Ibcast, + (buf, count, dt, root, *commPtr, &req) + ); + return std::make_shared(req); + } + + void MpiCommunicator::AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, + MPI_Op op) const + { + HEMELB_MPI_CALL( + MPI_Allreduce, + (send, ans, count, dt, op, *commPtr) + ); + } + + std::shared_ptr MpiCommunicator::IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, + MPI_Op op) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Iallreduce, + (send, ans, count, dt, op, *commPtr, &req) + ); + return std::make_shared(req); + } + + std::shared_ptr MpiCommunicator::IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, + MPI_Op op, int root) const + { + MPI_Request req; + HEMELB_MPI_CALL( + MPI_Ireduce, + (send, ans, count, dt, op, root, *commPtr, &req) + ); + return std::make_shared(req); + } + + void MpiCommunicator::ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, + MPI_Op op, int root) const + { + HEMELB_MPI_CALL( + MPI_Reduce, + (send, ans, count, dt, op, root, *commPtr) + ); + } + + void MpiCommunicator::GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype, + int root) const + { + HEMELB_MPI_CALL( + MPI_Gather, + (send, sendcount, sendtype, + recv, recvcount, recvtype, + root, *commPtr) + ); + } + + void MpiCommunicator::GathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype, + int root) const + { + HEMELB_MPI_CALL( + MPI_Gatherv, + (sendbuf, sendcount, sendtype, + recvbuf, recvcounts, displs, recvtype, + root, *commPtr) + ); + } + void MpiCommunicator::AllgathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype) const + { + HEMELB_MPI_CALL( + MPI_Allgatherv, + (sendbuf, sendcount, sendtype, + recvbuf, recvcounts, displs, recvtype, + *commPtr) + ); + } + void MpiCommunicator::AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const + { + HEMELB_MPI_CALL( + MPI_Allgather, + (send, sendcount, sendtype, + recv, recvcount, recvtype, + *commPtr) + ); + } + + void MpiCommunicator::AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const + { + HEMELB_MPI_CALL( + MPI_Alltoall, + (send, sendcount, sendtype, + recv, recvcount, recvtype, + *commPtr) + ); + } + + void MpiCommunicator::SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const + { + HEMELB_MPI_CALL( + MPI_Send, + (sendbuf, sendcount, sendtype, + dest, tag, *commPtr) + ); + } + + void MpiCommunicator::SsendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const + { + HEMELB_MPI_CALL( + MPI_Ssend, + (sendbuf, sendcount, sendtype, + dest, tag, *commPtr) + ); + } + void MpiCommunicator::RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int src, int tag, MPI_Status* stat) const + { + HEMELB_MPI_CALL( + MPI_Recv, + (recvbuf, recvcount, recvtype, + src, tag, *commPtr, stat) + ); + } + + std::shared_ptr MpiCommunicator::IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const + { + MPI_Request ans; + HEMELB_MPI_CALL( + MPI_Isend, + (sendbuf, sendcount, sendtype, + dest, tag, *commPtr, &ans) + ); + return std::make_shared(ans); + } + + std::shared_ptr MpiCommunicator::IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const + { + MPI_Request ans; + HEMELB_MPI_CALL( + MPI_Issend, + (sendbuf, sendcount, sendtype, + dest, tag, *commPtr, &ans) + ); + return std::make_shared(ans); + } + + std::shared_ptr MpiCommunicator::IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int tag) const + { + MPI_Request ans; + HEMELB_MPI_CALL( + MPI_Irecv, + (recvbuf, recvcount, recvtype, + source, tag, *commPtr, &ans) + ); + return std::make_shared(ans); + } + + } +} + diff --git a/Code/comm/MpiCommunicator.h b/Code/comm/MpiCommunicator.h new file mode 100644 index 000000000..9aa06b6c1 --- /dev/null +++ b/Code/comm/MpiCommunicator.h @@ -0,0 +1,119 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_COMM_MPICOMMUNICATOR_H +#define HEMELB_COMM_MPICOMMUNICATOR_H + +#include "comm/Communicator.h" + +namespace hemelb +{ + namespace comm + { + + // MPI communicator + class MpiCommunicator : public Communicator, public std::enable_shared_from_this + { + public: + /** + * Constructor for an uninitialised communicator, equivalent to + * MPI_COMM_NULL + * @param communicator + */ + MpiCommunicator(); + /** + * Constructor to get data needed from an MPI communicator + * @param communicator + */ + MpiCommunicator(MPI_Comm communicator, bool willOwn); + + operator MPI_Comm() const; + /** + * Class has virtual methods so should have virtual d'tor. + */ + virtual ~MpiCommunicator(); + + /** + * Returns the local rank on the communicator + * @return + */ + virtual int Rank() const; + + /** + * Returns the size of the communicator (i.e. total number of procs involved). + * @return + */ + virtual int Size() const; + + /** + * Abort - should try to bring down all tasks, but no guarantees + * @param errCode + */ + virtual void Abort(int errCode) const; + + /** + * Duplicate the communicator - see MPI_COMM_DUP + * @return + */ + virtual Communicator::Ptr Duplicate() const; + + virtual std::shared_ptr GetGroup() const; + virtual Communicator::Ptr Create(std::shared_ptr grp) const; + + virtual std::shared_ptr OpenFile(const std::string& filename, int mode, + const MPI_Info info = MPI_INFO_NULL) const; + virtual std::shared_ptr MakeRequestList() const; + + virtual void Barrier() const; + virtual std::shared_ptr Ibarrier() const; + + virtual bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const; + + private: + + virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const; + virtual std::shared_ptr IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const; + virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const; + virtual std::shared_ptr IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const; + virtual std::shared_ptr IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const; + virtual void ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const; + virtual void GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype, + int root) const; + virtual void GathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype, + int root) const; + virtual void AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const; + virtual void AllgathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype) const; + virtual void AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const; + virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const; + virtual void SsendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const; + virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int src, int tag, MPI_Status* stat) const; + virtual std::shared_ptr IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const; + virtual std::shared_ptr IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const; + virtual std::shared_ptr IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int tag) const; + + std::shared_ptr commPtr; + int rank; + int size; + }; + + // bool operator==(const Communicator& comm1, const Communicator& comm2); + // bool operator!=(const Communicator& comm1, const Communicator& comm2); + + } +} + + +#endif /* HEMELB_COMM_MPICOMMUNICATOR_H */ diff --git a/Code/net/MpiDataType.cc b/Code/comm/MpiDataType.cc similarity index 97% rename from Code/net/MpiDataType.cc rename to Code/comm/MpiDataType.cc index 883d1c163..d25cc95db 100644 --- a/Code/net/MpiDataType.cc +++ b/Code/comm/MpiDataType.cc @@ -4,11 +4,11 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/MpiDataType.h" +#include "comm/MpiDataType.h" namespace hemelb { - namespace net + namespace comm { // Specializations of the above getters for built in types. // These mappings are taken directly from the MPI standard version 2.2, diff --git a/Code/net/MpiDataType.h b/Code/comm/MpiDataType.h similarity index 94% rename from Code/net/MpiDataType.h rename to Code/comm/MpiDataType.h index b82b3acd3..f28eeaa41 100644 --- a/Code/net/MpiDataType.h +++ b/Code/comm/MpiDataType.h @@ -4,8 +4,8 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIDATATYPE_H -#define HEMELB_NET_MPIDATATYPE_H +#ifndef HEMELB_COMM_MPIDATATYPE_H +#define HEMELB_COMM_MPIDATATYPE_H #include #if HEMELB_HAVE_CSTDINT @@ -34,7 +34,7 @@ (&(typeInstances->name), elementDisplacements + elementCounter) \ ); \ elementDisplacements[elementCounter] -= instanceAddr; \ - elementTypes[elementCounter] = ::hemelb::net::MpiDataType(typeInstances->name); \ + elementTypes[elementCounter] = ::hemelb::comm::MpiDataType(typeInstances->name); \ ++elementCounter #define HEMELB_MPI_TYPE_ADD_MEMBER(name) HEMELB_MPI_TYPE_ADD_MEMBER_N(name, 1) @@ -59,7 +59,7 @@ namespace hemelb { - namespace net + namespace comm { /* * There are templated functions for getting the MPI_Datatype @@ -73,7 +73,7 @@ namespace hemelb * You must specialize the template, e.g. * * template<> - * MPI_Datatype hemelb::net::MpiDataTypeTraits::RegisterMpiDataType() + * MPI_Datatype hemelb::comm::MpiDataTypeTraits::RegisterMpiDataType() * { * // Create the type * MPI_Type_commit(&type); @@ -85,7 +85,7 @@ namespace hemelb * Important note: to ensure C++ standard compliance, you MUST declare your * specialisations before use and you MUST ensure that the definition * is compiled exactly once (standard ODR). These MUST both be in the - * namespace hemelb::net. + * namespace hemelb::comm. * * Declaration is best done in the relevant header file. These templates are * only used by MpiCommunicator's templated communication methods so the @@ -162,4 +162,4 @@ namespace hemelb } } -#endif // HEMELB_NET_MPIDATATYPE_H +#endif // HEMELB_COMM_MPIDATATYPE_H diff --git a/Code/net/MpiEnvironment.cc b/Code/comm/MpiEnvironment.cc similarity index 79% rename from Code/net/MpiEnvironment.cc rename to Code/comm/MpiEnvironment.cc index d5116704a..1aad4ea73 100644 --- a/Code/net/MpiEnvironment.cc +++ b/Code/comm/MpiEnvironment.cc @@ -5,13 +5,13 @@ // license in the file LICENSE. #include -#include "net/MpiEnvironment.h" -#include "net/MpiError.h" -#include "net/MpiCommunicator.h" +#include "comm/MpiEnvironment.h" +#include "comm/MpiError.h" +#include "comm/MpiCommunicator.h" namespace hemelb { - namespace net + namespace comm { MpiEnvironment::MpiEnvironment(int& argc, char**& argv) : @@ -33,6 +33,11 @@ namespace hemelb } } + Communicator::Ptr MpiEnvironment::World() + { + return std::make_shared(MPI_COMM_WORLD, false); + } + bool MpiEnvironment::Initialized() { int flag; @@ -53,7 +58,7 @@ namespace hemelb void MpiEnvironment::Abort(int errorCode) { - MpiCommunicator::World().Abort(errorCode); + World()->Abort(errorCode); } } diff --git a/Code/net/MpiEnvironment.h b/Code/comm/MpiEnvironment.h similarity index 89% rename from Code/net/MpiEnvironment.h rename to Code/comm/MpiEnvironment.h index d04dbc3b1..80aa89993 100644 --- a/Code/net/MpiEnvironment.h +++ b/Code/comm/MpiEnvironment.h @@ -4,13 +4,16 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIENVIRONMENT_H -#define HEMELB_NET_MPIENVIRONMENT_H +#ifndef HEMELB_COMM_MPIENVIRONMENT_H +#define HEMELB_COMM_MPIENVIRONMENT_H + +#include "comm/Communicator.h" namespace hemelb { - namespace net + namespace comm { + /** * Manage the MPI environment and provide query/abort functions. * @@ -35,6 +38,7 @@ namespace hemelb */ ~MpiEnvironment(); + static Communicator::Ptr World(); /** * Query if MPI is initialised * @return @@ -63,4 +67,4 @@ namespace hemelb }; } } -#endif //HEMELB_NET_MPIENVIRONMENT_H +#endif //HEMELB_COMM_MPIENVIRONMENT_H diff --git a/Code/net/MpiError.cc b/Code/comm/MpiError.cc similarity index 94% rename from Code/net/MpiError.cc rename to Code/comm/MpiError.cc index 3f0a416ce..1e8dd72d4 100644 --- a/Code/net/MpiError.cc +++ b/Code/comm/MpiError.cc @@ -4,12 +4,12 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/MpiError.h" +#include "comm/MpiError.h" #include namespace hemelb { - namespace net + namespace comm { MpiError::MpiError(const char* mpiFunc_, const int errorCode_, const char* fileName_, const int lineNo_) : diff --git a/Code/net/MpiError.h b/Code/comm/MpiError.h similarity index 82% rename from Code/net/MpiError.h rename to Code/comm/MpiError.h index df783a738..47e57908d 100644 --- a/Code/net/MpiError.h +++ b/Code/comm/MpiError.h @@ -4,15 +4,15 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIERROR_H -#define HEMELB_NET_MPIERROR_H +#ifndef HEMELB_COMM_MPIERROR_H +#define HEMELB_COMM_MPIERROR_H #include #include "Exception.h" namespace hemelb { - namespace net + namespace comm { /** * Indicate an error to do with MPI. @@ -39,7 +39,7 @@ namespace hemelb { \ int _check_result = mpiFunc args; \ if (_check_result != MPI_SUCCESS) \ - throw ::hemelb::net::MpiError(#mpiFunc, _check_result, __FILE__, __LINE__); \ + throw ::hemelb::comm::MpiError(#mpiFunc, _check_result, __FILE__, __LINE__); \ } -#endif // HEMELB_NET_MPIERROR_H +#endif // HEMELB_COMM_MPIERROR_H diff --git a/Code/net/MpiFile.cc b/Code/comm/MpiFile.cc similarity index 56% rename from Code/net/MpiFile.cc rename to Code/comm/MpiFile.cc index 3ada89c12..0a51c90f4 100644 --- a/Code/net/MpiFile.cc +++ b/Code/comm/MpiFile.cc @@ -4,13 +4,12 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/MpiFile.h" -#include "net/MpiCommunicator.h" -#include "net/MpiConstness.h" +#include "comm/MpiFile.h" +#include "comm/MpiCommunicator.h" namespace hemelb { - namespace net + namespace comm { namespace { @@ -25,23 +24,12 @@ namespace hemelb } - MpiFile::MpiFile(const MpiCommunicator& parentComm, MPI_File fh) : - comm(&parentComm) + MpiFile::MpiFile(Communicator::ConstPtr parentComm, MPI_File fh) : + comm(parentComm) { filePtr.reset(new MPI_File(fh), Deleter); } - MpiFile MpiFile::Open(const MpiCommunicator& comm, const std::string& filename, int mode, - const MPI_Info info) - { - MPI_File ans; - HEMELB_MPI_CALL( - MPI_File_open, - (comm, MpiConstCast(filename.c_str()), mode, info, &ans) - ); - return MpiFile(comm, ans); - } - void MpiFile::Close() { @@ -56,16 +44,16 @@ namespace hemelb return *filePtr; } - const MpiCommunicator& MpiFile::GetCommunicator() const + Communicator::ConstPtr MpiFile::GetCommunicator() const { - return *comm; + return comm; } void MpiFile::SetView(MPI_Offset disp, MPI_Datatype etype, MPI_Datatype filetype, const std::string& datarep, MPI_Info info) { HEMELB_MPI_CALL( MPI_File_set_view, - (*filePtr, disp, etype, filetype, MpiConstCast(datarep.c_str()), info) + (*filePtr, disp, etype, filetype, datarep.c_str(), info) ); } } diff --git a/Code/net/MpiFile.h b/Code/comm/MpiFile.h similarity index 60% rename from Code/net/MpiFile.h rename to Code/comm/MpiFile.h index 71da5dd53..b1d034ea8 100644 --- a/Code/net/MpiFile.h +++ b/Code/comm/MpiFile.h @@ -4,32 +4,24 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIFILE_H -#define HEMELB_NET_MPIFILE_H +#ifndef HEMELB_COMM_MPIFILE_H +#define HEMELB_COMM_MPIFILE_H -#include -#include "net/MpiError.h" -#include "net/MpiCommunicator.h" +#include +#include "comm/MpiError.h" +#include "comm/Communicator.h" namespace hemelb { - namespace net + namespace comm { class MpiFile { public: + typedef std::shared_ptr Ptr; + typedef std::shared_ptr ConstPtr; MpiFile(); - - /** - * Opens a file with MPI_File_open. A collective operation on comm. - * @param comm - * @param filename - * @param mode - * @param info - * @return - */ - static MpiFile Open(const MpiCommunicator& comm, const std::string& filename, int mode, - const MPI_Info info = MPI_INFO_NULL); + MpiFile(Communicator::ConstPtr parentComm, MPI_File fh); /** * Closes the file with MPI_File_close. @@ -45,7 +37,7 @@ namespace hemelb void SetView(MPI_Offset disp, MPI_Datatype etype, MPI_Datatype filetype, const std::string& datarep, MPI_Info info); - const MpiCommunicator& GetCommunicator() const; + Communicator::ConstPtr GetCommunicator() const; template void Read(std::vector& buffer, MPI_Status* stat = MPI_STATUS_IGNORE); @@ -56,16 +48,17 @@ namespace hemelb void Write(const std::vector& buffer, MPI_Status* stat = MPI_STATUS_IGNORE); template void WriteAt(MPI_Offset offset, const std::vector& buffer, MPI_Status* stat = MPI_STATUS_IGNORE); - protected: - MpiFile(const MpiCommunicator& parentComm, MPI_File fh); + private: + friend class MpiCommunicator; + - const MpiCommunicator* comm; - boost::shared_ptr filePtr; + Communicator::ConstPtr comm; + std::shared_ptr filePtr; }; } } -#include "net/MpiFile.hpp" +#include "comm/MpiFile.hpp" -#endif /* HEMELB_NET_MPIFILE_H */ +#endif /* HEMELB_COMM_MPIFILE_H */ diff --git a/Code/net/MpiFile.hpp b/Code/comm/MpiFile.hpp similarity index 68% rename from Code/net/MpiFile.hpp rename to Code/comm/MpiFile.hpp index c75ce6d09..eb0f089de 100644 --- a/Code/net/MpiFile.hpp +++ b/Code/comm/MpiFile.hpp @@ -4,14 +4,14 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIFILE_HPP -#define HEMELB_NET_MPIFILE_HPP +#ifndef HEMELB_COMM_MPIFILE_HPP +#define HEMELB_COMM_MPIFILE_HPP -#include "net/MpiFile.h" +#include "comm/MpiFile.h" namespace hemelb { - namespace net + namespace comm { template @@ -19,7 +19,7 @@ namespace hemelb { HEMELB_MPI_CALL( MPI_File_read, - (*filePtr, &buffer[0], buffer.size(), MpiDataType(), stat) + (*filePtr, buffer.data(), buffer.size(), MpiDataType(), stat) ); } template @@ -27,7 +27,7 @@ namespace hemelb { HEMELB_MPI_CALL( MPI_File_read_at, - (*filePtr, offset, &buffer[0], buffer.size(), MpiDataType(), stat) + (*filePtr, offset, buffer.data(), buffer.size(), MpiDataType(), stat) ); } @@ -36,7 +36,7 @@ namespace hemelb { HEMELB_MPI_CALL( MPI_File_write, - (*filePtr, MpiConstCast(&buffer[0]), buffer.size(), MpiDataType(), stat) + (*filePtr, buffer.data(), buffer.size(), MpiDataType(), stat) ); } template @@ -44,7 +44,7 @@ namespace hemelb { HEMELB_MPI_CALL( MPI_File_write_at, - (*filePtr, offset, MpiConstCast(&buffer[0]), buffer.size(), MpiDataType(), stat) + (*filePtr, offset, buffer.data(), buffer.size(), MpiDataType(), stat) ); } diff --git a/Code/net/MpiGroup.cc b/Code/comm/MpiGroup.cc similarity index 71% rename from Code/net/MpiGroup.cc rename to Code/comm/MpiGroup.cc index 196d14b42..5a1ab2202 100644 --- a/Code/net/MpiGroup.cc +++ b/Code/comm/MpiGroup.cc @@ -4,12 +4,12 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/MpiGroup.h" -#include "net/MpiConstness.h" +#include "comm/MpiGroup.h" +#include "comm/MpiError.h" namespace hemelb { - namespace net + namespace comm { namespace { @@ -41,20 +41,20 @@ namespace hemelb return size; } - MpiGroup MpiGroup::Exclude(const std::vector& ranksToExclude) + Group::Ptr MpiGroup::Exclude(const std::vector& ranksToExclude) const { MPI_Group ans; HEMELB_MPI_CALL(MPI_Group_excl, - (*groupPtr, ranksToExclude.size(), MpiConstCast(&ranksToExclude.front()), &ans)) - return MpiGroup(ans, true); + (*groupPtr, ranksToExclude.size(), ranksToExclude.data(), &ans)); + return std::make_shared(ans, true); } - MpiGroup MpiGroup::Include(const std::vector& ranksToInclude) + Group::Ptr MpiGroup::Include(const std::vector& ranksToInclude) const { MPI_Group ans; HEMELB_MPI_CALL(MPI_Group_incl, - (*groupPtr, ranksToInclude.size(), MpiConstCast(&ranksToInclude.front()), &ans)) - return MpiGroup(ans, true); + (*groupPtr, ranksToInclude.size(), ranksToInclude.data(), &ans)); + return std::make_shared(ans, true); } MpiGroup::MpiGroup(MPI_Group grp, bool own) diff --git a/Code/net/MpiGroup.h b/Code/comm/MpiGroup.h similarity index 69% rename from Code/net/MpiGroup.h rename to Code/comm/MpiGroup.h index 07bdf374e..d08b0ef4a 100644 --- a/Code/net/MpiGroup.h +++ b/Code/comm/MpiGroup.h @@ -4,50 +4,57 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#ifndef HEMELB_NET_MPIGROUP_H -#define HEMELB_NET_MPIGROUP_H +#ifndef HEMELB_COMM_MPIGROUP_H +#define HEMELB_COMM_MPIGROUP_H -#include "units.h" -#include "net/mpi.h" -#include "net/MpiCommunicator.h" -#include +#include "comm/Group.h" +#include +#include namespace hemelb { - namespace net + namespace comm { - class MpiGroup + class MpiGroup : public Group { public: /** * Default c'tor - initialises equivalent to MPI_GROUP_NULL */ MpiGroup(); + // Direct construction. + /** + * Construct from an MPI_Group + * @param grp + * @param own - Whether this instance is responsible for deleting the + * group when all copies are destroyed. + */ + MpiGroup(MPI_Group grp, bool own); /** * Returns the local rank within the group * @return */ - int Rank() const; + virtual int Rank() const; /** * Returns the size of the group * @return */ - int Size() const; + virtual int Size() const; /** * Exclude the provided ranks * @param ranksToExclude * @return */ - MpiGroup Exclude(const std::vector& ranksToExclude); + virtual Group::Ptr Exclude(const std::vector& ranksToExclude) const; /** * Include the provided ranks * @param ranksToExclude * @return */ - MpiGroup Include(const std::vector& ranksToInclude); + virtual Group::Ptr Include(const std::vector& ranksToInclude) const; /** * Implicit cast to the underlying MPI_group @@ -59,21 +66,9 @@ namespace hemelb } private: - // Allow Communicators access to the c'tor. - friend class MpiCommunicator; - - // Direct construction. - /** - * Construct from an MPI_Group - * @param grp - * @param own - Whether this instance is responsible for deleting the - * group when all copies are destroyed. - */ - MpiGroup(MPI_Group grp, bool own); - - boost::shared_ptr groupPtr; + std::shared_ptr groupPtr; }; } } -#endif /* HEMELB_NET_MPIGROUP_H */ +#endif /* HEMELB_COMM_MPIGROUP_H */ diff --git a/Code/comm/MpiRequest.cc b/Code/comm/MpiRequest.cc new file mode 100644 index 000000000..c27c1cb1d --- /dev/null +++ b/Code/comm/MpiRequest.cc @@ -0,0 +1,90 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#include "comm/MpiRequest.h" + +namespace hemelb +{ + namespace comm + { + MpiRequest::MpiRequest() : + req(MPI_REQUEST_NULL) + { + } + MpiRequest::MpiRequest(MPI_Request req_) : + req(req_) + { + } + + MpiRequest::operator bool() const + { + return req != MPI_REQUEST_NULL; + } + + void MpiRequest::Wait() + { + HEMELB_MPI_CALL(MPI_Wait, (&req, MPI_STATUS_IGNORE)); + } + + bool MpiRequest::Test() + { + int flag; + HEMELB_MPI_CALL(MPI_Test, (&req, &flag, MPI_STATUS_IGNORE)); + return flag; + } + + size_t MpiRequestList::size() const + { + return reqs.size(); + } + + void MpiRequestList::resize(size_t i) + { + reqs.resize(i); + } + void MpiRequestList::clear() + { + reqs.clear(); + } + + void MpiRequestList::push_back(Request::Ptr r) + { + auto mr = std::dynamic_pointer_cast(r); + //MpiRequest&& mr = dynamic_cast(r); + reqs.push_back(mr->req); + mr->req = MPI_REQUEST_NULL; + } + + void MpiRequestList::set(size_t i, Request::Ptr r) { + auto mr = std::dynamic_pointer_cast(r); + //MpiRequest&& mr = dynamic_cast(r); + reqs[i] = mr->req; + mr->req = MPI_REQUEST_NULL; + } + + void MpiRequestList::WaitAll() + { + HEMELB_MPI_CALL( + MPI_Waitall, + (reqs.size(), reqs.data(), MPI_STATUSES_IGNORE) + ); + } + + bool MpiRequestList::TestAll() + { + int flag; + HEMELB_MPI_CALL( + MPI_Testall, + (reqs.size(), reqs.data(), &flag, MPI_STATUSES_IGNORE) + ); + return flag; + } + + } +} diff --git a/Code/comm/MpiRequest.h b/Code/comm/MpiRequest.h new file mode 100644 index 000000000..8432e1c80 --- /dev/null +++ b/Code/comm/MpiRequest.h @@ -0,0 +1,69 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_NET_MPIREQUEST_H +#define HEMELB_NET_MPIREQUEST_H + +#include +#include "comm/Request.h" +#include "comm/MpiError.h" + +namespace hemelb +{ + namespace comm + { + /** + * + */ + class MpiRequest : public Request + { + public: + MpiRequest(); + MpiRequest(MPI_Request req); + MpiRequest(MpiRequest&& req); + MpiRequest& operator=(MpiRequest&& req); + MpiRequest(const MpiRequest&) = delete; + MpiRequest& operator=(const MpiRequest&) = delete; + + /** + * Allow implicit casts to MPI_Request + * @return The underlying MPI_Request + */ + operator MPI_Request() const + { + return req; + } + operator bool() const; + + virtual void Wait(); + + virtual bool Test(); + + private: + friend class MpiRequestList; + MPI_Request req; + }; + + class MpiRequestList : public RequestList + { + public: + virtual size_t size() const; + virtual void resize(size_t i); + virtual void push_back(Request::Ptr); + virtual void set(size_t i, Request::Ptr); + virtual void clear(); + virtual void WaitAll(); + virtual bool TestAll(); + private: + std::vector reqs; + }; + + } +} +#endif // HEMELB_NET_MPIREQUEST_H diff --git a/Code/lb/StabilityTester.cc b/Code/comm/MpiStatus.cc similarity index 54% rename from Code/lb/StabilityTester.cc rename to Code/comm/MpiStatus.cc index 29b890cf7..43f172336 100644 --- a/Code/lb/StabilityTester.cc +++ b/Code/comm/MpiStatus.cc @@ -1,17 +1,21 @@ - // This file is part of HemeLB and is Copyright (C) // the HemeLB team and/or their institutions, as detailed in the // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "lb/StabilityTester.h" -#include "lb/LbmParameters.h" +#include "net/MpiStatus.h" namespace hemelb { - namespace lb + namespace net { - - + MpiStatus::MpiStatus() : + statPtr() + { + } + MpiStatus::MpiStatus(MPI_Status stat) : statPtr() + { + statPtr.reset(new MPI_Status(stat)); + } } } diff --git a/Code/comm/MpiStatus.h b/Code/comm/MpiStatus.h new file mode 100644 index 000000000..1c060963c --- /dev/null +++ b/Code/comm/MpiStatus.h @@ -0,0 +1,44 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_NET_MPISTATUS_H +#define HEMELB_NET_MPISTATUS_H + +#include "net/MpiError.h" +#include + +namespace hemelb +{ + namespace net + { + class MpiRequest; + + class MpiStatus + { + public: + MpiStatus(); + MpiStatus(MPI_Status stat); + + /** + * Allow implicit casts to MPI_Status + * @return The underlying MPI_Status + */ + operator MPI_Status() const + { + return *statPtr; + } + + private: + std::shared_ptr statPtr; + // Request needs to be able to access statPtr. + friend class MpiRequest; + }; + } +} +#endif // HEMELB_NET_MPISTATUS_H diff --git a/Code/comm/Request.h b/Code/comm/Request.h new file mode 100644 index 000000000..5011e19b1 --- /dev/null +++ b/Code/comm/Request.h @@ -0,0 +1,56 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_COMM_REQUEST_H +#define HEMELB_COMM_REQUEST_H + +#include +#include + +namespace hemelb +{ + namespace comm + { + class RequestList; + /** + * + */ + class Request + { + public: + typedef std::shared_ptr Ptr; + + Request() = default; + Request(Request&& req) = default; + Request& operator=(Request&& req) = default; + Request(const Request&) = delete; + Request& operator=(const Request&) = delete; + operator bool() const; + + virtual void Wait() = 0; + virtual bool Test() = 0; + }; + + class RequestList + { + public: + typedef std::shared_ptr Ptr; + + virtual size_t size() const = 0; + virtual void resize(size_t i) = 0; + virtual void push_back(Request::Ptr) = 0; + virtual void clear() = 0; + virtual void set(size_t i, Request::Ptr) = 0; + + virtual void WaitAll() = 0; + virtual bool TestAll() = 0; + }; + } +} +#endif // HEMELB_COMM_REQUEST_H diff --git a/Code/configuration/CommandLine.cc b/Code/configuration/CommandLine.cc index 1aabeda46..81bbe4e72 100644 --- a/Code/configuration/CommandLine.cc +++ b/Code/configuration/CommandLine.cc @@ -13,7 +13,7 @@ namespace hemelb { CommandLine::CommandLine(int aargc, const char * const * const aargv) : - inputFile("input.xml"), outputDir(""), images(10), steeringSessionId(1), debugMode(false), argc(aargc), + inputFile("input.xml"), outputDir(""), debugMode(false), argc(aargc), argv(aargv) { @@ -37,16 +37,6 @@ namespace hemelb { outputDir = std::string(paramValue); } - else if (std::strcmp(paramName, "-i") == 0) - { - char *dummy; - images = (unsigned int) (strtoul(paramValue, &dummy, 10)); - } - else if (std::strcmp(paramName, "-ss") == 0) - { - char *dummy; - steeringSessionId = (unsigned int) (strtoul(paramValue, &dummy, 10)); - } else if (std::strcmp(paramName, "-debug") == 0) { debugMode = std::strcmp(paramName, "0") == 0 ? false : true; @@ -65,7 +55,6 @@ namespace hemelb ans.append("-in \t Path to the configuration xml file (default is config.xml)\n"); ans.append("-out \t Path to the output folder (default is based on input file, e.g. config_xml_results)\n"); ans.append("-i \t Number of images to create (default is 10)\n"); - ans.append("-ss \t Steering session identifier (default is 1)\n"); return ans; } } diff --git a/Code/configuration/CommandLine.h b/Code/configuration/CommandLine.h index c337110fe..d46d104c8 100644 --- a/Code/configuration/CommandLine.h +++ b/Code/configuration/CommandLine.h @@ -21,8 +21,6 @@ namespace hemelb * Arguments should be: * - -in input xml configuration file (default input.xml) * - -out output folder (empty default, but the hemelb::io::PathManager will guess a value from the input file if not given.) - * - -i number of images (default 10) - * - -ss steering session i.d. (default 1) */ class CommandLine { @@ -38,13 +36,7 @@ namespace hemelb * Report to standard output an error message describing the usage */ static std::string GetUsage(); - /** - * @return Number of images that should be produced - */ - unsigned int NumberOfImages() const - { - return (images); - } + /** * The output directory that should be used for result files. * Empty default, but the hemelb::io::PathManager will guess a value from the input file if not given.) @@ -61,13 +53,6 @@ namespace hemelb { return (inputFile); } - /** - * @return A unique integer representing the steering session to which to attach. - */ - int GetSteeringSessionId() const - { - return (steeringSessionId); - } /** * @return Whether the user requested a debug mode. @@ -102,8 +87,6 @@ namespace hemelb private: std::string inputFile; //! local or full path to input file std::string outputDir; //! local or full path to input file - unsigned int images; //! images to produce - int steeringSessionId; //! unique identifier for steering session bool debugMode; //! Use debugger int argc; //! count of command line arguments, including program name const char * const * const argv; //! command line arguments diff --git a/Code/configuration/SimConfig.cc b/Code/configuration/SimConfig.cc index 0ae5e926e..34cfca60a 100644 --- a/Code/configuration/SimConfig.cc +++ b/Code/configuration/SimConfig.cc @@ -83,8 +83,6 @@ namespace hemelb inlets = DoIOForInOutlets(topNode.GetChildOrThrow("inlets")); outlets = DoIOForInOutlets(topNode.GetChildOrThrow("outlets")); - DoIOForVisualisation(topNode.GetChildOrThrow("visualisation")); - // Optional element io::xml::Element propertiesEl = topNode.GetChildOrNull("properties"); if (propertiesEl != io::xml::Element::Missing()) @@ -288,24 +286,6 @@ namespace hemelb return newIolet; } - void SimConfig::DoIOForVisualisation(const io::xml::Element& visEl) - { - GetDimensionalValue(visEl.GetChildOrThrow("centre"), "m", visualisationCentre); - - io::xml::Element orientationEl = visEl.GetChildOrThrow("orientation"); - GetDimensionalValue(orientationEl.GetChildOrThrow("longitude"), - "deg", - visualisationLongitude); - GetDimensionalValue(orientationEl.GetChildOrThrow("latitude"), "deg", visualisationLatitude); - - io::xml::Element displayEl = visEl.GetChildOrThrow("display"); - displayEl.GetAttributeOrThrow("zoom", visualisationZoom); - displayEl.GetAttributeOrThrow("brightness", visualisationBrightness); - - io::xml::Element rangeEl = visEl.GetChildOrThrow("range"); - GetDimensionalValue(rangeEl.GetChildOrThrow("maxvelocity"), "m/s", maxVelocity); - GetDimensionalValue(rangeEl.GetChildOrThrow("maxstress"), "Pa", maxStress); - } void SimConfig::DoIOForProperties(const io::xml::Element& propertiesEl) { diff --git a/Code/configuration/SimConfig.h b/Code/configuration/SimConfig.h index 770d4e07f..7030f1e28 100644 --- a/Code/configuration/SimConfig.h +++ b/Code/configuration/SimConfig.h @@ -65,10 +65,6 @@ namespace hemelb void Save(std::string path); // TODO this method should be able to be CONST // but because it uses DoIo, which uses one function signature for both reading and writing, it cannot be. - const util::Vector3D & GetVisualisationCentre() const - { - return visualisationCentre; - } const std::vector & GetInlets() const { return inlets; @@ -81,30 +77,6 @@ namespace hemelb { return stressType; } - float GetVisualisationLongitude() const - { - return visualisationLongitude; - } - float GetVisualisationLatitude() const - { - return visualisationLatitude; - } - float GetVisualisationZoom() const - { - return visualisationZoom; - } - float GetVisualisationBrightness() const - { - return visualisationBrightness; - } - float GetMaximumVelocity() const - { - return maxVelocity; - } - float GetMaximumStress() const - { - return maxStress; - } const std::string & GetDataFilePath() const { return dataFilePath; @@ -244,7 +216,6 @@ namespace hemelb extraction::SurfacePointSelector* DoIOForSurfacePoint(const io::xml::Element&); void DoIOForInitialConditions(io::xml::Element parent); - void DoIOForVisualisation(const io::xml::Element& visEl); /** * Reads monitoring configuration from XML file @@ -271,13 +242,6 @@ namespace hemelb io::xml::Document* rawXmlDoc; std::string dataFilePath; - util::Vector3D visualisationCentre; - float visualisationLongitude; - float visualisationLatitude; - float visualisationZoom; - float visualisationBrightness; - float maxVelocity; - float maxStress; lb::StressTypes stressType; std::vector propertyOutputs; std::string colloidConfigPath; diff --git a/Code/debug/Debugger.cc b/Code/debug/Debugger.cc index beb6f5b0e..db719c72c 100644 --- a/Code/debug/Debugger.cc +++ b/Code/debug/Debugger.cc @@ -13,12 +13,12 @@ namespace hemelb namespace debug { - Debugger* Debugger::Init(bool active, const char * const executable, const net::MpiCommunicator& comm) + Debugger* Debugger::Init(bool active, const char * const executable, comm::Communicator::ConstPtr comm) { /* Static member function that implements the singleton pattern. - * Use the namespace function PlatformDebuggerFactory to - * actually construct the instance. It should be defined in the - * appropriate platform subdirectory. + * + * PlatformDebugger.h is configured to have an appropriate + * typedef for the current platform. */ if (Debugger::singleton == NULL) { @@ -40,7 +40,7 @@ namespace hemelb // Init static members Debugger* Debugger::singleton = NULL; - Debugger::Debugger(const char* const executable, const net::MpiCommunicator& comm) : + Debugger::Debugger(const char* const executable, comm::Communicator::ConstPtr comm) : mExecutable(executable), mCommunicator(comm) { } diff --git a/Code/debug/Debugger.h b/Code/debug/Debugger.h index acf3889c8..15839a273 100644 --- a/Code/debug/Debugger.h +++ b/Code/debug/Debugger.h @@ -8,7 +8,7 @@ #define HEMELB_DEBUG_DEBUGGER_H #include -#include "net/MpiCommunicator.h" +#include "comm/Communicator.h" namespace hemelb { @@ -22,20 +22,20 @@ namespace hemelb */ public: // the singleton pattern - static Debugger* Init(bool active, const char *const, const net::MpiCommunicator& comm); + static Debugger* Init(bool active, const char *const, comm::Communicator::ConstPtr comm); static Debugger* Get(void); virtual void BreakHere(void) = 0; virtual void Print(const char* iFormat, ...) = 0; protected: - Debugger(const char* const executable, const net::MpiCommunicator& comm); + Debugger(const char* const executable, comm::Communicator::ConstPtr comm); virtual ~Debugger(); virtual void Attach() = 0; std::string mExecutable; - const net::MpiCommunicator mCommunicator; + comm::Communicator::ConstPtr mCommunicator; // Singleton pattern static Debugger* singleton; diff --git a/Code/debug/OSX/MPIdebug.applescript b/Code/debug/OSX/MPIdebug.applescript index 486c9180b..5eec12f9e 100644 --- a/Code/debug/OSX/MPIdebug.applescript +++ b/Code/debug/OSX/MPIdebug.applescript @@ -19,9 +19,9 @@ on run args end repeat set lldbPath to do shell script "which lldb" - if lldbPath is equal to "" + if lldbPath is equal to "" then set gdbPath to do shell script "which gdb" - if gdbPath is equal to "" + if gdbPath is equal to "" then -- Error - no known debugger! return "Cannot find a debugger" else @@ -69,22 +69,17 @@ on CreateTabs(pIds) end CreateTabs on lldbRun(binary, pIds) - set debugger to "lldb" - + set debuggerCommandFile to myDir() & "/resume.lldb" + set debugger to "lldb -s " & debuggerCommandFile + set tabList to CreateTabs(pIds) tell application "Terminal" -- run commands in tabs repeat with i from 1 to count pIds set pId to item i of pIds set curTab to item i of tabList - set cmd to debugger & " -f " & binary & " -p " & pId + set cmd to debugger & " -p " & pId & " " & binary set newTab to do script cmd in curTab - delay 1 - do script ("frame select -r 3") in curTab - do script ("expr amWaiting = 0") in curTab - do script ("breakpoint set -F hemelb::debug::ActiveDebugger::BreakHere()") in curTab - do script ("breakpoint command add -o 'finish' 1") in curTab - do script ("continue") in curTab end repeat tell application "System Events" to tell process "Terminal" to keystroke "}" using command down diff --git a/Code/debug/OSX/OsxDebugger.cc b/Code/debug/OSX/OsxDebugger.cc index 2d642cac8..709469976 100644 --- a/Code/debug/OSX/OsxDebugger.cc +++ b/Code/debug/OSX/OsxDebugger.cc @@ -5,14 +5,14 @@ // license in the file LICENSE. #include - +#include "Exception.h" #include "debug/OSX/OsxDebugger.h" #include namespace hemelb { namespace debug { - OsxDebugger::OsxDebugger(const char* const executable, const net::MpiCommunicator& comm) : + OsxDebugger::OsxDebugger(const char* const executable, comm::Communicator::ConstPtr comm) : ActiveDebugger(executable, comm) { } diff --git a/Code/debug/OSX/OsxDebugger.h b/Code/debug/OSX/OsxDebugger.h index 1fe56f5a7..494587631 100644 --- a/Code/debug/OSX/OsxDebugger.h +++ b/Code/debug/OSX/OsxDebugger.h @@ -22,7 +22,7 @@ namespace hemelb const std::string GetPlatformScript(void) const; // C'tor... - OsxDebugger(const char* const executable, const net::MpiCommunicator& comm); + OsxDebugger(const char* const executable, comm::Communicator::ConstPtr comm); // ... which the factory function needs to be able to get at. friend class Debugger; diff --git a/Code/debug/OSX/resume.lldb b/Code/debug/OSX/resume.lldb new file mode 100644 index 000000000..2e557434f --- /dev/null +++ b/Code/debug/OSX/resume.lldb @@ -0,0 +1,11 @@ +# Get out of sleep and disable the waiting flag +frame select -r 3 +expr amWaiting = 0 + +# Set up a breakpoint for our break function and a command to finish +# it automatically +breakpoint set -M ActiveDebugger::BreakHere() +breakpoint command add -o 'finish' 1 + +# Carry on +continue diff --git a/Code/debug/common/ActiveDebugger.cc b/Code/debug/common/ActiveDebugger.cc index 165f6e5fb..7a0c5c3cd 100644 --- a/Code/debug/common/ActiveDebugger.cc +++ b/Code/debug/common/ActiveDebugger.cc @@ -19,8 +19,6 @@ #include #include -#include "net/mpi.h" - #include "debug/common/ActiveDebugger.h" namespace hemelb @@ -28,7 +26,7 @@ namespace hemelb namespace debug { - ActiveDebugger::ActiveDebugger(const char* const executable, const net::MpiCommunicator& comm) : + ActiveDebugger::ActiveDebugger(const char* const executable, comm::Communicator::ConstPtr comm) : Debugger(executable, comm), mAmAttached(false), mPIds() { } @@ -69,7 +67,7 @@ namespace hemelb // To rank 0 GatherProcessIds(); - if (mCommunicator.Rank() == 0) + if (mCommunicator->Rank() == 0) { childPid = fork(); // Fork gives the PID of the child to the parent and zero to the child @@ -84,7 +82,7 @@ namespace hemelb while (amWaiting) sleep(5); - if (mCommunicator.Rank() == 0) + if (mCommunicator->Rank() == 0) { // Reap the spawner int deadPid = waitpid(childPid, NULL, 0); @@ -99,7 +97,7 @@ namespace hemelb { int pId = getpid(); - mPIds = mCommunicator.Gather(pId, 0); + mPIds = mCommunicator->Gather(pId, 0); } void ActiveDebugger::SpawnDebuggers(void) diff --git a/Code/debug/common/ActiveDebugger.h b/Code/debug/common/ActiveDebugger.h index cfb110e4c..ab1441965 100644 --- a/Code/debug/common/ActiveDebugger.h +++ b/Code/debug/common/ActiveDebugger.h @@ -33,7 +33,7 @@ namespace hemelb typedef std::vector VoS;// Vector of Strings // C'tor - ActiveDebugger(const char* const executable, const net::MpiCommunicator& comm); + ActiveDebugger(const char* const executable, comm::Communicator::ConstPtr comm); bool mAmAttached;// Indicate attachment state VoI mPIds;// vector of process IDs diff --git a/Code/debug/linux/LinuxDebugger.cc b/Code/debug/linux/LinuxDebugger.cc index 8f8aa66d4..ea9e2bbbb 100644 --- a/Code/debug/linux/LinuxDebugger.cc +++ b/Code/debug/linux/LinuxDebugger.cc @@ -7,6 +7,7 @@ #include #include "debug/linux/LinuxDebugger.h" +#include "Exception.h" #include #include @@ -14,7 +15,7 @@ namespace hemelb { namespace debug { - LinuxDebugger::LinuxDebugger(const char* const executable, const net::MpiCommunicator& comm) : + LinuxDebugger::LinuxDebugger(const char* const executable, comm::Communicator::ConstPtr comm) : ActiveDebugger(executable, comm) {} const std::string LinuxDebugger::GetBinaryPath(void) const diff --git a/Code/debug/linux/LinuxDebugger.h b/Code/debug/linux/LinuxDebugger.h index 06e44eecb..0bb626a0d 100644 --- a/Code/debug/linux/LinuxDebugger.h +++ b/Code/debug/linux/LinuxDebugger.h @@ -22,14 +22,12 @@ namespace hemelb const std::string GetPlatformScript(void) const; // C'tor... - LinuxDebugger(const char* const executable, const net::MpiCommunicator& comm); + LinuxDebugger(const char* const executable, comm::Communicator::ConstPtr comm); // ... which the factory function needs to be able to get at. friend class Debugger; }; - // Factory. Don't be calling this. - Debugger* PlatformDebuggerFactory(const char* const executable, const net::MpiCommunicator& comm); } } diff --git a/Code/debug/none/NullDebugger.cc b/Code/debug/none/NullDebugger.cc index 4c9a487e3..f010d2bcc 100644 --- a/Code/debug/none/NullDebugger.cc +++ b/Code/debug/none/NullDebugger.cc @@ -22,7 +22,7 @@ namespace hemelb { } - NullDebugger::NullDebugger(const char* const executable, const net::MpiCommunicator& comm) : + NullDebugger::NullDebugger(const char* const executable, comm::Communicator::ConstPtr comm) : Debugger(executable, comm) { } diff --git a/Code/debug/none/NullDebugger.h b/Code/debug/none/NullDebugger.h index b36d07107..8cfeabf73 100644 --- a/Code/debug/none/NullDebugger.h +++ b/Code/debug/none/NullDebugger.h @@ -21,7 +21,7 @@ namespace hemelb protected: void Attach(void); - NullDebugger(const char* const executable, const net::MpiCommunicator& comm); + NullDebugger(const char* const executable, comm::Communicator::ConstPtr); friend class Debugger; }; diff --git a/Code/extraction/LocalPropertyOutput.cc b/Code/extraction/LocalPropertyOutput.cc index 19c01db9d..b4c0b4b5e 100644 --- a/Code/extraction/LocalPropertyOutput.cc +++ b/Code/extraction/LocalPropertyOutput.cc @@ -9,7 +9,7 @@ #include "io/formats/formats.h" #include "io/formats/extraction.h" #include "io/writers/xdr/XdrMemWriter.h" -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" #include "constants.h" namespace hemelb @@ -18,13 +18,13 @@ namespace hemelb { LocalPropertyOutput::LocalPropertyOutput(IterableDataSource& dataSource, const PropertyOutputFile* outputSpec, - const net::IOCommunicator& ioComms) : + comm::Communicator::ConstPtr ioComms) : comms(ioComms), dataSource(dataSource), outputSpec(outputSpec) { // Open the file as write-only, create it if it doesn't exist, don't create if the file // already exists. - outputFile = net::MpiFile::Open(comms, outputSpec->filename, - MPI_MODE_WRONLY | MPI_MODE_CREATE | MPI_MODE_EXCL); + outputFile = comms->OpenFile(outputSpec->filename, + MPI_MODE_WRONLY | MPI_MODE_CREATE | MPI_MODE_EXCL); // Count sites on this task uint64_t siteCount = 0; dataSource.Reset(); @@ -53,7 +53,7 @@ namespace hemelb writeLength *= siteCount; // The IO proc also writes the iteration number - if (comms.OnIORank()) + if (comms->OnIORank()) { writeLength += 8; } @@ -61,17 +61,17 @@ namespace hemelb //! @TODO: These two MPI calls can be replaced with one // Everyone needs to know the total length written during one iteration. - allCoresWriteLength = comms.AllReduce(writeLength, MPI_SUM); + allCoresWriteLength = comms->AllReduce(writeLength, MPI_SUM); // Only the root process needs to know the total number of sites written // Note this has a garbage value on other procs. - uint64_t allSiteCount = comms.Reduce(siteCount, MPI_SUM, - comms.GetIORank()); + uint64_t allSiteCount = comms->Reduce(siteCount, MPI_SUM, + comms->GetIORank()); unsigned totalHeaderLength = 0; // Write the header information on the IO proc. - if (comms.OnIORank()) + if (comms->OnIORank()) { // Compute the length of the field header unsigned fieldHeaderLength = 0; @@ -125,29 +125,29 @@ namespace hemelb } // Write from the buffer - outputFile.WriteAt(0, headerBuffer); + outputFile->WriteAt(0, headerBuffer); } // Calculate where each core should start writing - if (comms.OnIORank()) + if (comms->OnIORank()) { // For core 0 this is easy: it passes the value for core 1 to the core. localDataOffsetIntoFile = totalHeaderLength; - if (comms.Size() > 1) + if (comms->Size() > 1) { - comms.Send(localDataOffsetIntoFile+writeLength, 1, 1); + comms->Send(localDataOffsetIntoFile+writeLength, 1, 1); } } else { // Receive the writing start position from the previous core. - comms.Receive(localDataOffsetIntoFile, comms.Rank()-1, 1); + comms->Recv(localDataOffsetIntoFile, comms->Rank()-1, 1); // Send the next core its start position. - if (comms.Rank() != (comms.Size() - 1)) + if (comms->Rank() != (comms->Size() - 1)) { - comms.Send(localDataOffsetIntoFile+writeLength, comms.Rank() + 1, 1); + comms->Send(localDataOffsetIntoFile+writeLength, comms->Rank() + 1, 1); } } @@ -188,7 +188,7 @@ namespace hemelb io::writers::xdr::XdrMemWriter xdrWriter(&buffer[0], buffer.size()); // Firstly, the IO proc must write the iteration number. - if (comms.OnIORank()) + if (comms->OnIORank()) { xdrWriter << (uint64_t) timestepNumber; } @@ -252,7 +252,7 @@ namespace hemelb break; case OutputField::MpiRank: xdrWriter - << static_cast (comms.Rank()); + << static_cast (comms->Rank()); break; default: // This should never trip. It only occurs when a new OutputField field is added and no @@ -264,7 +264,7 @@ namespace hemelb } // Actually do the MPI writing. - outputFile.WriteAt(localDataOffsetIntoFile, buffer); + outputFile->WriteAt(localDataOffsetIntoFile, buffer); // Set the offset to the right place for writing on the next iteration. localDataOffsetIntoFile += allCoresWriteLength; diff --git a/Code/extraction/LocalPropertyOutput.h b/Code/extraction/LocalPropertyOutput.h index 60494204c..352670981 100644 --- a/Code/extraction/LocalPropertyOutput.h +++ b/Code/extraction/LocalPropertyOutput.h @@ -9,15 +9,12 @@ #include "extraction/IterableDataSource.h" #include "extraction/PropertyOutputFile.h" -#include "net/mpi.h" -#include "net/MpiFile.h" + +#include "comm/Communicator.h" +#include "comm/MpiFile.h" namespace hemelb { - namespace net - { - class IOCommunicator; - } namespace extraction { /** @@ -32,7 +29,8 @@ namespace hemelb * @param offset * @return */ - LocalPropertyOutput(IterableDataSource& dataSource, const PropertyOutputFile* outputSpec, const net::IOCommunicator& ioComms); + LocalPropertyOutput(IterableDataSource& dataSource, const PropertyOutputFile* outputSpec, + comm::Communicator::ConstPtr ioComms); /** * Tidies up the LocalPropertyOutput (close files etc). @@ -72,11 +70,11 @@ namespace hemelb */ double GetOffset(OutputField::FieldType field) const; - const net::IOCommunicator& comms; + comm::Communicator::ConstPtr comms; /** * The MPI file to write into. */ - net::MpiFile outputFile; + comm::MpiFile::Ptr outputFile; /** * The data source to use for file output. diff --git a/Code/extraction/PropertyActor.cc b/Code/extraction/PropertyActor.cc index b4a9caa0b..cb1edbff0 100644 --- a/Code/extraction/PropertyActor.cc +++ b/Code/extraction/PropertyActor.cc @@ -14,7 +14,7 @@ namespace hemelb const std::vector& propertyOutputs, IterableDataSource& dataSource, reporting::Timers& timers, - const net::IOCommunicator& ioComms) : + comm::Communicator::ConstPtr ioComms) : simulationState(simulationState), timers(timers) { propertyWriter = new PropertyWriter(dataSource, propertyOutputs, ioComms); @@ -82,7 +82,7 @@ namespace hemelb } } - void PropertyActor::EndIteration() + void PropertyActor::EndAll() { timers[reporting::Timers::extractionWriting].Start(); propertyWriter->Write(simulationState.GetTimeStep()); diff --git a/Code/extraction/PropertyActor.h b/Code/extraction/PropertyActor.h index 15bccb913..ee7173742 100644 --- a/Code/extraction/PropertyActor.h +++ b/Code/extraction/PropertyActor.h @@ -11,13 +11,13 @@ #include "io/PathManager.h" #include "lb/MacroscopicPropertyCache.h" #include "lb/SimulationState.h" -#include "net/IteratedAction.h" +#include "timestep/Actor.h" namespace hemelb { namespace extraction { - class PropertyActor : public net::IteratedAction + class PropertyActor : public timestep::Actor { public: /** @@ -31,7 +31,7 @@ namespace hemelb const std::vector& propertyOutputs, IterableDataSource& dataSource, reporting::Timers& timers, - const net::IOCommunicator& ioComms); + comm::Communicator::ConstPtr ioComms); ~PropertyActor(); @@ -41,11 +41,26 @@ namespace hemelb */ void SetRequiredProperties(lb::MacroscopicPropertyCache& propertyCache); - /** - * Override the iterated actor end of iteration method to perform writing. - */ - void EndIteration(); - + inline virtual void BeginAll() { + } + inline virtual void Begin() { + } + inline virtual void Receive() { + } + inline virtual void PreSend() { + } + inline virtual void Send() { + } + inline virtual void PreWait() { + } + inline virtual void Wait() { + } + inline virtual void End() { + } + + // Override the iterated actor end of iteration method to perform writing. + virtual void EndAll(); + private: const lb::SimulationState& simulationState; PropertyWriter* propertyWriter; diff --git a/Code/extraction/PropertyWriter.cc b/Code/extraction/PropertyWriter.cc index db9e9a539..7386848e4 100644 --- a/Code/extraction/PropertyWriter.cc +++ b/Code/extraction/PropertyWriter.cc @@ -12,7 +12,7 @@ namespace hemelb { PropertyWriter::PropertyWriter(IterableDataSource& dataSource, const std::vector& propertyOutputs, - const net::IOCommunicator& ioComms) + comm::Communicator::ConstPtr ioComms) { for (unsigned outputNumber = 0; outputNumber < propertyOutputs.size(); ++outputNumber) { diff --git a/Code/extraction/PropertyWriter.h b/Code/extraction/PropertyWriter.h index ffb43c617..19fffd5ae 100644 --- a/Code/extraction/PropertyWriter.h +++ b/Code/extraction/PropertyWriter.h @@ -9,7 +9,7 @@ #include "extraction/LocalPropertyOutput.h" #include "extraction/PropertyOutputFile.h" -#include "net/mpi.h" +#include "comm/Communicator.h" namespace hemelb { @@ -23,7 +23,7 @@ namespace hemelb * @param propertyOutputs * @return */ - PropertyWriter(IterableDataSource& dataSource, const std::vector& propertyOutputs, const net::IOCommunicator& ioComms); + PropertyWriter(IterableDataSource& dataSource, const std::vector& propertyOutputs, comm::Communicator::ConstPtr ioComms); /** * Destructor; deallocates memory used to store property info. diff --git a/Code/functionaltests/pythontests/resources/poiseuille_flow_test_master.xml b/Code/functionaltests/pythontests/resources/poiseuille_flow_test_master.xml index 3442aabfb..1fe5eb01c 100644 --- a/Code/functionaltests/pythontests/resources/poiseuille_flow_test_master.xml +++ b/Code/functionaltests/pythontests/resources/poiseuille_flow_test_master.xml @@ -18,12 +18,6 @@ - - - - - - diff --git a/Code/geometry/BlockTraverserWithVisitedBlockTracker.cc b/Code/geometry/BlockTraverserWithVisitedBlockTracker.cc index 73f615c9a..ea4404aa8 100644 --- a/Code/geometry/BlockTraverserWithVisitedBlockTracker.cc +++ b/Code/geometry/BlockTraverserWithVisitedBlockTracker.cc @@ -7,7 +7,6 @@ #include "geometry/BlockTraverserWithVisitedBlockTracker.h" #include "geometry/LatticeData.h" #include "util/Vector3D.h" -#include "vis/rayTracer/RayTracer.h" namespace hemelb { diff --git a/Code/geometry/GeometryReader.cc b/Code/geometry/GeometryReader.cc index de8624c46..bd8f67118 100644 --- a/Code/geometry/GeometryReader.cc +++ b/Code/geometry/GeometryReader.cc @@ -17,44 +17,21 @@ #include "geometry/decomposition/OptimisedDecomposition.h" #include "geometry/GeometryReader.h" #include "lb/lattices/D3Q27.h" -#include "net/net.h" -#include "net/IOCommunicator.h" #include "log/Logger.h" #include "util/utilityFunctions.h" #include "constants.h" +#include "comm/Group.h" +#include "comm/Async.h" + namespace hemelb { namespace geometry { - GeometryReader::GeometryReader(const bool reserveSteeringCore, - const lb::lattices::LatticeInfo& latticeInfo, - reporting::Timers &atimings, const net::IOCommunicator& ioComm) : + GeometryReader::GeometryReader(const lb::lattices::LatticeInfo& latticeInfo, + reporting::Timers &atimings, comm::Communicator::ConstPtr ioComm) : latticeInfo(latticeInfo), hemeLbComms(ioComm), timings(atimings) { - // This rank should participate in the domain decomposition if - // - there's no steering core (then all ranks are involved) - // - we're not on core 0 (the only core that might ever not participate) - // - there's only one processor (so core 0 has to participate) - - // Create our own group, without the root node if we're not running with it. - if (reserveSteeringCore && ioComm.Size() > 1) - { - participateInTopology = !ioComm.OnIORank(); - - std::vector lExclusions(1); - lExclusions[0] = 0; - net::MpiGroup computeGroup = hemeLbComms.Group().Exclude(lExclusions); - // Create a communicator just for the domain decomposition. - computeComms = ioComm.Create(computeGroup); - // Note that on the steering core, this is a null communicator. - } - else - { - participateInTopology = true; - computeComms = ioComm; - } - } GeometryReader::~GeometryReader() @@ -84,13 +61,13 @@ namespace hemelb ); // Open the file. - file = net::MpiFile::Open(hemeLbComms, dataFilePath, MPI_MODE_RDONLY, fileInfo); + file = hemeLbComms->OpenFile(dataFilePath, MPI_MODE_RDONLY, fileInfo); log::Logger::Log("Opened config file %s", dataFilePath.c_str()); // TODO: Why is there this fflush? fflush( NULL); // Set the view to the file. - file.SetView(0, MPI_CHAR, MPI_CHAR, "native", fileInfo); + file->SetView(0, MPI_CHAR, MPI_CHAR, "native", fileInfo); log::Logger::Log("Reading file preamble"); Geometry geometry = ReadPreamble(); @@ -98,54 +75,43 @@ namespace hemelb log::Logger::Log("Reading file header"); ReadHeader(geometry.GetBlockCount()); - // Close the file - only the ranks participating in the topology need to read it again. - file.Close(); + // Close the file + file->Close(); timings[hemelb::reporting::Timers::initialDecomposition].Start(); log::Logger::Log("Beginning initial decomposition"); principalProcForEachBlock.resize(geometry.GetBlockCount()); - if (!participateInTopology) - { - // If we are the steering core, mark them all as unknown. - for (site_t block = 0; block < geometry.GetBlockCount(); ++block) - { - principalProcForEachBlock[block] = -1; - } - } - else + // Get an initial base-level decomposition of the domain macro-blocks over processors. + // This will later be improved upon by ParMetis. + decomposition::BasicDecomposition basicDecomposer(geometry, + latticeInfo, + hemeLbComms, + fluidSitesOnEachBlock); + basicDecomposer.Decompose(principalProcForEachBlock); + + if (ShouldValidate()) { - // Get an initial base-level decomposition of the domain macro-blocks over processors. - // This will later be improved upon by ParMetis. - decomposition::BasicDecomposition basicDecomposer(geometry, - latticeInfo, - computeComms, - fluidSitesOnEachBlock); - basicDecomposer.Decompose(principalProcForEachBlock); - - if (ShouldValidate()) - { - basicDecomposer.Validate(principalProcForEachBlock); - } + basicDecomposer.Validate(principalProcForEachBlock); } + timings[hemelb::reporting::Timers::initialDecomposition].Stop(); // Perform the initial read-in. log::Logger::Log("Reading in my blocks"); - if (participateInTopology) + // TODO: can we remove the close/reopen? As far as I can tell, + // it's only the set view we are changing + + // Reopen in the file. Read in blocks local to this node. + file = hemeLbComms->OpenFile(dataFilePath, MPI_MODE_RDONLY, fileInfo); + + ReadInBlocksWithHalo(geometry, principalProcForEachBlock, hemeLbComms->Rank()); + + if (ShouldValidate()) { - // Reopen in the file just between the nodes in the topology decomposition. Read in blocks - // local to this node. - file = net::MpiFile::Open(computeComms, dataFilePath, MPI_MODE_RDONLY, fileInfo); - - ReadInBlocksWithHalo(geometry, principalProcForEachBlock, computeComms.Rank()); - - if (ShouldValidate()) - { - ValidateGeometry(geometry); - } + ValidateGeometry(geometry); } - + timings[hemelb::reporting::Timers::fileRead].Stop(); hemelb::log::Logger::Log("Begin optimising the domain decomposition."); @@ -153,19 +119,16 @@ namespace hemelb // Having done an initial decomposition of the geometry, and read in the data, we optimise the // domain decomposition. - if (participateInTopology) + log::Logger::Log("Beginning domain decomposition optimisation"); + OptimiseDomainDecomposition(geometry, principalProcForEachBlock); + log::Logger::Log("Ending domain decomposition optimisation"); + + if (ShouldValidate()) { - log::Logger::Log("Beginning domain decomposition optimisation"); - OptimiseDomainDecomposition(geometry, principalProcForEachBlock); - log::Logger::Log("Ending domain decomposition optimisation"); - - if (ShouldValidate()) - { - ValidateGeometry(geometry); - } - file.Close(); + ValidateGeometry(geometry); } - + file->Close(); + // Finish up - close the file, set the timings, deallocate memory. HEMELB_MPI_CALL(MPI_Info_free, (&fileInfo)); @@ -177,12 +140,12 @@ namespace hemelb std::vector GeometryReader::ReadOnAllTasks(unsigned nBytes) { std::vector buffer(nBytes); - const net::MpiCommunicator& comm = file.GetCommunicator(); - if (comm.Rank() == HEADER_READING_RANK) + auto comm = file->GetCommunicator(); + if (comm->Rank() == HEADER_READING_RANK) { - file.Read(buffer); + file->Read(buffer); } - comm.Broadcast(buffer, HEADER_READING_RANK); + comm->Broadcast(buffer, HEADER_READING_RANK); return buffer; } @@ -321,11 +284,11 @@ namespace hemelb // Next we spread round the lists of which blocks each core needs access to. log::Logger::Log("Informing reading cores of block needs"); - net::Net net = net::Net(computeComms); + Needs needs(geometry.GetBlockCount(), readBlock, - util::NumericalFunctions::min(READING_GROUP_SIZE, computeComms.Size()), - net, + util::NumericalFunctions::min(READING_GROUP_SIZE, hemeLbComms->Size()), + hemeLbComms, ShouldValidate()); timings[hemelb::reporting::Timers::readBlocksPrelim].Stop(); @@ -366,40 +329,38 @@ namespace hemelb std::vector compressedBlockData; proc_t readingCore = GetReadingCoreForBlock(blockNumber); - net::Net net = net::Net(computeComms); - - if (readingCore == computeComms.Rank()) { - timings[hemelb::reporting::Timers::readBlock].Start(); - // Read the data. - compressedBlockData.resize(bytesPerCompressedBlock[blockNumber]); - file.ReadAt(offsetSoFar, compressedBlockData); - - // Spread it. - for (std::vector::const_iterator receiver = procsWantingThisBlock.begin(); receiver - != procsWantingThisBlock.end(); receiver++) - { - if (*receiver != computeComms.Rank()) - { - - net.RequestSendV(compressedBlockData, *receiver); - } - } - timings[hemelb::reporting::Timers::readBlock].Stop(); - } - else if (neededOnThisRank) - { - compressedBlockData.resize(bytesPerCompressedBlock[blockNumber]); - - net.RequestReceiveV(compressedBlockData, readingCore); - + comm::Async requestQ(hemeLbComms); + + if (readingCore == hemeLbComms->Rank()) + { + timings[hemelb::reporting::Timers::readBlock].Start(); + // Read the data. + compressedBlockData.resize(bytesPerCompressedBlock[blockNumber]); + file->ReadAt(offsetSoFar, compressedBlockData); + + // Spread it. + for (std::vector::const_iterator receiver = procsWantingThisBlock.begin(); receiver + != procsWantingThisBlock.end(); receiver++) + { + if (*receiver != hemeLbComms->Rank()) + { + requestQ.Isend(compressedBlockData, *receiver); + } + } + timings[hemelb::reporting::Timers::readBlock].Stop(); + } + else if (neededOnThisRank) + { + compressedBlockData.resize(bytesPerCompressedBlock[blockNumber]); + requestQ.Irecv(compressedBlockData, readingCore); + } + else + { + return; + } + timings[hemelb::reporting::Timers::readNet].Start(); } - else - { - return; - } - timings[hemelb::reporting::Timers::readNet].Start(); - net.Dispatch(); timings[hemelb::reporting::Timers::readNet].Stop(); timings[hemelb::reporting::Timers::readParse].Start(); if (neededOnThisRank) @@ -590,7 +551,7 @@ namespace hemelb proc_t GeometryReader::GetReadingCoreForBlock(site_t blockNumber) { return proc_t(blockNumber % util::NumericalFunctions::min(READING_GROUP_SIZE, - computeComms.Size())); + hemeLbComms->Size())); } /** @@ -649,13 +610,13 @@ namespace hemelb // Reduce using a minimum to find the actual processor for each site (ignoring the // invalid entries). - std::vector procForSiteRecv = computeComms.AllReduce(myProcForSite, MPI_MIN); - std::vector siteDataRecv = computeComms.AllReduce(dummySiteData, MPI_MIN); + std::vector procForSiteRecv = hemeLbComms->AllReduce(myProcForSite, MPI_MIN); + std::vector siteDataRecv = hemeLbComms->AllReduce(dummySiteData, MPI_MIN); for (site_t site = 0; site < geometry.GetSitesPerBlock(); ++site) { - if (procForSiteRecv[site] == ConvertTopologyRankToGlobalRank(computeComms.Rank()) - && (myProcForSite[site] != ConvertTopologyRankToGlobalRank(computeComms.Rank()))) + if (procForSiteRecv[site] == hemeLbComms->Rank() + && (myProcForSite[site] != hemeLbComms->Rank())) { log::Logger::Log("Other cores think this core has site %li on block %li but it disagrees.", site, @@ -757,7 +718,7 @@ namespace hemelb const std::vector& procForEachBlock) { decomposition::OptimisedDecomposition optimiser(timings, - computeComms, + hemeLbComms, geometry, latticeInfo, procForEachBlock, @@ -804,7 +765,7 @@ namespace hemelb // going to be moved to the current proc. idx_t moveIndex = 0; - for (proc_t fromProc = 0; fromProc < computeComms.Size(); ++fromProc) + for (proc_t fromProc = 0; fromProc < hemeLbComms->Size(); ++fromProc) { for (idx_t moveNumber = 0; moveNumber < movesPerProc[fromProc]; ++moveNumber) { @@ -812,15 +773,15 @@ namespace hemelb idx_t toProc = movesList[3 * moveIndex + 2]; ++moveIndex; - if (toProc == (idx_t) computeComms.Rank()) + if (toProc == (idx_t) hemeLbComms->Rank()) { - newProcForEachBlock[block] = computeComms.Rank(); + newProcForEachBlock[block] = hemeLbComms->Rank(); } } } // Reread the blocks into the GlobalLatticeData now. - ReadInBlocksWithHalo(geometry, newProcForEachBlock, computeComms.Rank()); + ReadInBlocksWithHalo(geometry, newProcForEachBlock, hemeLbComms->Rank()); } void GeometryReader::ImplementMoves(Geometry& geometry, @@ -846,8 +807,7 @@ namespace hemelb if (geometry.Blocks[block].Sites[siteIndex].targetProcessor != SITE_OR_BLOCK_SOLID) { // ... set its rank to be the rank it had before optimisation. - geometry.Blocks[block].Sites[siteIndex].targetProcessor - = ConvertTopologyRankToGlobalRank(originalProc); + geometry.Blocks[block].Sites[siteIndex].targetProcessor = originalProc; } } } @@ -857,7 +817,7 @@ namespace hemelb idx_t moveIndex = 0; // For each source proc, go through as many moves as it had. - for (proc_t fromProc = 0; fromProc < computeComms.Size(); ++fromProc) + for (proc_t fromProc = 0; fromProc < hemeLbComms->Size(); ++fromProc) { for (idx_t moveNumber = 0; moveNumber < movesFromEachProc[fromProc]; ++moveNumber) { @@ -873,8 +833,7 @@ namespace hemelb // lFromProc. if (ShouldValidate()) { - if (geometry.Blocks[block].Sites[site].targetProcessor - != ConvertTopologyRankToGlobalRank((proc_t) fromProc)) + if (geometry.Blocks[block].Sites[site].targetProcessor != proc_t(fromProc)) { log::Logger::Log("Block %ld, site %ld from move %u was originally on proc %i, not proc %u.", block, @@ -886,8 +845,7 @@ namespace hemelb } // Implement the move. - geometry.Blocks[block].Sites[site].targetProcessor - = ConvertTopologyRankToGlobalRank((proc_t) toProc); + geometry.Blocks[block].Sites[site].targetProcessor = proc_t(toProc); } ++moveIndex; @@ -895,15 +853,6 @@ namespace hemelb } } - proc_t GeometryReader::ConvertTopologyRankToGlobalRank(proc_t topologyRankIn) const - { - // If the global rank is not equal to the topology rank, we are not using rank 0 for - // LBM. - return (hemeLbComms.Rank() == computeComms.Rank()) - ? topologyRankIn - : (topologyRankIn + 1); - } - bool GeometryReader::ShouldValidate() const { #ifdef HEMELB_VALIDATE_GEOMETRY diff --git a/Code/geometry/GeometryReader.h b/Code/geometry/GeometryReader.h index 6a5d0c40d..440645b77 100644 --- a/Code/geometry/GeometryReader.h +++ b/Code/geometry/GeometryReader.h @@ -13,7 +13,6 @@ #include "io/writers/xdr/XdrReader.h" #include "lb/lattices/LatticeInfo.h" #include "lb/LbmParameters.h" -#include "net/mpi.h" #include "geometry/ParmetisHeader.h" #include "reporting/Timers.h" #include "util/Vector3D.h" @@ -21,7 +20,7 @@ #include "geometry/Geometry.h" #include "geometry/needs/Needs.h" -#include "net/MpiFile.h" +#include "comm/MpiFile.h" namespace hemelb { @@ -33,8 +32,8 @@ namespace hemelb public: typedef util::Vector3D BlockLocation; - GeometryReader(const bool reserveSteeringCore, const lb::lattices::LatticeInfo&, - reporting::Timers &timings, const net::IOCommunicator& ioComm); + GeometryReader(const lb::lattices::LatticeInfo&, + reporting::Timers &timings, comm::Communicator::ConstPtr ioComm); ~GeometryReader(); Geometry LoadAndDecompose(const std::string& dataFilePath); @@ -148,8 +147,6 @@ namespace hemelb const std::vector& movesFromEachProc, const std::vector& movesList) const; - proc_t ConvertTopologyRankToGlobalRank(proc_t topologyRank) const; - /** * True if we should validate the geometry. * @return @@ -164,14 +161,11 @@ namespace hemelb //! Info about the connectivity of the lattice. const lb::lattices::LatticeInfo& latticeInfo; //! File accessed to read in the geometry data. - net::MpiFile file; + comm::MpiFile::Ptr file; //! Information about the file, to give cues and hints to MPI. - const net::IOCommunicator& hemeLbComms; //! HemeLB's main communicator - net::MpiCommunicator computeComms; //! Communication info for all ranks that will need a slice of the geometry (i.e. all non-steering cores) - //! True iff this rank is participating in the domain decomposition. - bool participateInTopology; - + comm::Communicator::ConstPtr hemeLbComms; //! HemeLB's main communicator + //! The number of fluid sites on each block in the geometry std::vector fluidSitesOnEachBlock; //! The number of bytes each block takes up while still compressed. diff --git a/Code/geometry/LatticeData.cc b/Code/geometry/LatticeData.cc index 01266dc06..65bc392d9 100644 --- a/Code/geometry/LatticeData.cc +++ b/Code/geometry/LatticeData.cc @@ -7,19 +7,19 @@ #include #include -#include "debug/Debugger.h" #include "log/Logger.h" -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" #include "geometry/BlockTraverser.h" #include "geometry/LatticeData.h" #include "geometry/neighbouring/NeighbouringLatticeData.h" #include "util/utilityFunctions.h" +#include "comm/Async.h" namespace hemelb { namespace geometry { - LatticeData::LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const net::IOCommunicator& comms_) : + LatticeData::LatticeData(const lb::lattices::LatticeInfo& latticeInfo, comm::Communicator::ConstPtr comms_) : latticeInfo(latticeInfo), neighbouringData(new neighbouring::NeighbouringLatticeData(latticeInfo)), comms(comms_) { } @@ -29,7 +29,7 @@ namespace hemelb delete neighbouringData; } - LatticeData::LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const Geometry& readResult, const net::IOCommunicator& comms_) : + LatticeData::LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const Geometry& readResult, comm::Communicator::ConstPtr comms_) : latticeInfo(latticeInfo), neighbouringData(new neighbouring::NeighbouringLatticeData(latticeInfo)), comms(comms_) { SetBasicDetails(readResult.GetBlockDimensions(), @@ -39,7 +39,7 @@ namespace hemelb // if debugging then output beliefs regarding geometry and neighbour list if (log::Logger::ShouldDisplay()) { - proc_t localRank = comms.Rank(); + proc_t localRank = comms->Rank(); for (std::vector::iterator itNeighProc = neighbouringProcs.begin(); itNeighProc != neighbouringProcs.end(); ++itNeighProc) { @@ -82,7 +82,7 @@ namespace hemelb std::vector domainEdgeWallDistance[COLLISION_TYPES]; std::vector midDomainWallDistance[COLLISION_TYPES]; - proc_t localRank = comms.Rank(); + proc_t localRank = comms->Rank(); // Iterate over all blocks in site units for (BlockTraverser blockTraverser(*this); blockTraverser.CurrentLocationValid(); blockTraverser.TraverseOne()) { @@ -275,9 +275,9 @@ namespace hemelb void LatticeData::CollectFluidSiteDistribution() { hemelb::log::Logger::Log("Gathering lattice info."); - fluidSitesOnEachProcessor = comms.AllGather(localFluidSites); + fluidSitesOnEachProcessor = comms->AllGather(localFluidSites); totalFluidSites = 0; - for (proc_t ii = 0; ii < comms.Size(); ++ii) + for (proc_t ii = 0; ii < comms->Size(); ++ii) { totalFluidSites += fluidSitesOnEachProcessor[ii]; } @@ -305,7 +305,7 @@ namespace hemelb siteSet.TraverseOne()) { if (block.GetProcessorRankForSite(siteSet.GetCurrentIndex()) - == comms.Rank()) + == comms->Rank()) { util::Vector3D globalCoords = blockSet.GetCurrentLocation() * GetBlockSize() + siteSet.GetCurrentLocation(); @@ -320,8 +320,8 @@ namespace hemelb } - std::vector siteMins = comms.AllReduce(localMins, MPI_MIN); - std::vector siteMaxes = comms.AllReduce(localMaxes, MPI_MAX); + std::vector siteMins = comms->AllReduce(localMins, MPI_MIN); + std::vector siteMaxes = comms->AllReduce(localMaxes, MPI_MAX); for (unsigned ii = 0; ii < 3; ++ii) { @@ -335,7 +335,7 @@ namespace hemelb // Allocate the index in which to put the distribution functions received from the other // process. std::vector > sharedDistributionLocationForEachProc = - std::vector >(comms.Size()); + std::vector >(comms->Size()); site_t totalSharedDistributionsSoFar = 0; // Set the remaining neighbouring processor data. for (size_t neighbourId = 0; neighbourId < neighbouringProcs.size(); neighbourId++) @@ -353,7 +353,7 @@ namespace hemelb void LatticeData::InitialiseNeighbourLookup(std::vector >& sharedFLocationForEachProc) { - const proc_t localRank = comms.Rank(); + const proc_t localRank = comms->Rank(); neighbourIndices.resize(latticeInfo.GetNumVectors() * localFluidSites); for (BlockTraverser blockTraverser(*this); blockTraverser.CurrentLocationValid(); blockTraverser.TraverseOne()) { @@ -433,7 +433,7 @@ namespace hemelb void LatticeData::InitialisePointToPointComms(std::vector >& sharedFLocationForEachProc) { - proc_t localRank = comms.Rank(); + proc_t localRank = comms->Rank(); // point-to-point communications are performed to match data to be // sent to/receive from different partitions; in this way, the // communication of the locations of the interface-dependent fluid @@ -441,7 +441,7 @@ namespace hemelb // propagate to different partitions is avoided (only their values // will be communicated). It's here! // Allocate the request variable. - net::Net tempNet(comms); + comm::Async commQ(comms); for (size_t neighbourId = 0; neighbourId < neighbouringProcs.size(); neighbourId++) { NeighbouringProcessor* neigh_proc_p = &neighbouringProcs[neighbourId]; @@ -451,21 +451,19 @@ namespace hemelb // other processor. if (neigh_proc_p->Rank > localRank) { - tempNet.RequestSendV(sharedFLocationForEachProc[neigh_proc_p->Rank], neigh_proc_p->Rank); + commQ.Isend(sharedFLocationForEachProc[neigh_proc_p->Rank], neigh_proc_p->Rank); } else { sharedFLocationForEachProc[neigh_proc_p->Rank].resize(neigh_proc_p->SharedDistributionCount * 4); - tempNet.RequestReceiveV(sharedFLocationForEachProc[neigh_proc_p->Rank], neigh_proc_p->Rank); + commQ.Irecv(sharedFLocationForEachProc[neigh_proc_p->Rank], neigh_proc_p->Rank); } } - - tempNet.Dispatch(); } void LatticeData::InitialiseReceiveLookup(std::vector >& sharedFLocationForEachProc) { - proc_t localRank = comms.Rank(); + proc_t localRank = comms->Rank(); streamingIndicesForReceivedDistributions.resize(totalSharedFs); site_t f_count = GetLocalFluidSiteCount() * latticeInfo.GetNumVectors(); site_t sharedSitesSeen = 0; @@ -594,7 +592,7 @@ namespace hemelb // get the rank of the processor that owns the site procId = block.GetProcessorRankForSite(localSiteIndex); - if (procId != comms.Rank()) + if (procId != comms->Rank()) return false; if (procId == SITE_OR_BLOCK_SOLID) // means that the site is solid return false; @@ -653,20 +651,28 @@ namespace hemelb blockCoords.x = blockIJData / blockCounts.y; } - void LatticeData::SendAndReceive(hemelb::net::Net* net) + void LatticeData::Receive(comm::Async::Ptr commQ) { for (std::vector::const_iterator it = neighbouringProcs.begin(); it != neighbouringProcs.end(); ++it) { - // Request the receive into the appropriate bit of FOld. - net->RequestReceive(GetFOld( (*it).FirstSharedDistribution), - (int) ( ( (*it).SharedDistributionCount)), - (*it).Rank); - // Request the send from the right bit of FNew. - net->RequestSend(GetFNew( (*it).FirstSharedDistribution), - (int) ( ( (*it).SharedDistributionCount)), - (*it).Rank); - + // Start the receive into the appropriate bit of FOld. + commQ->Irecv(GetFOld( (*it).FirstSharedDistribution), + (int) ( ( (*it).SharedDistributionCount)), + (*it).Rank, + 1); + } + } + void LatticeData::Send(comm::Async::Ptr commQ) + { + for (std::vector::const_iterator it = neighbouringProcs.begin(); + it != neighbouringProcs.end(); ++it) + { + // Start the send from the right bit of FNew. + commQ->Isend(GetFNew( (*it).FirstSharedDistribution), + (int) ( ( (*it).SharedDistributionCount)), + (*it).Rank, + 1); } } @@ -704,7 +710,7 @@ namespace hemelb int LatticeData::GetLocalRank() const { - return comms.Rank(); + return comms->Rank(); } } diff --git a/Code/geometry/LatticeData.h b/Code/geometry/LatticeData.h index 121b0e803..5f82f2a46 100644 --- a/Code/geometry/LatticeData.h +++ b/Code/geometry/LatticeData.h @@ -10,7 +10,7 @@ #include #include -#include "net/net.h" +#include "comm/Async.h" #include "constants.h" #include "configuration/SimConfig.h" #include "geometry/Block.h" @@ -39,7 +39,7 @@ namespace hemelb template friend class lb::LBM; //! Let the LBM have access to internals so it can initialise the distribution arrays. template friend class Site; //! Let the inner classes have access to site-related data that's otherwise private. - LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const Geometry& readResult, const net::IOCommunicator& comms); + LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const Geometry& readResult, comm::Communicator::ConstPtr comms); virtual ~LatticeData(); @@ -51,7 +51,9 @@ namespace hemelb oldDistributions.swap(newDistributions); } - void SendAndReceive(net::Net* net); + void Receive(comm::Async::Ptr commQ); + void Send(comm::Async::Ptr commQ); + void CopyReceived(); /** @@ -337,7 +339,7 @@ namespace hemelb * class for the purpose of testing. * @return */ - LatticeData(const lb::lattices::LatticeInfo& latticeInfo, const net::IOCommunicator& comms); + LatticeData(const lb::lattices::LatticeInfo& latticeInfo, comm::Communicator::ConstPtr comms); void SetBasicDetails(util::Vector3D blocks, site_t blockSize); @@ -574,7 +576,7 @@ namespace hemelb std::vector neighbourIndices; //! Data about neighbouring fluid sites. std::vector streamingIndicesForReceivedDistributions; //! The indices to stream to for distributions received from other processors. neighbouring::NeighbouringLatticeData *neighbouringData; - const net::IOCommunicator& comms; + comm::Communicator::ConstPtr comms; }; } } diff --git a/Code/geometry/SiteData.cc b/Code/geometry/SiteData.cc index d6258106f..fc75106bd 100644 --- a/Code/geometry/SiteData.cc +++ b/Code/geometry/SiteData.cc @@ -8,7 +8,7 @@ namespace hemelb { - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() diff --git a/Code/geometry/SiteData.h b/Code/geometry/SiteData.h index ccac22b15..0f6f7d8d5 100644 --- a/Code/geometry/SiteData.h +++ b/Code/geometry/SiteData.h @@ -8,11 +8,11 @@ #define HEMELB_GEOMETRY_SITEDATA_H #include "geometry/SiteDataBare.h" -#include "net/MpiDataType.h" +#include "comm/MpiDataType.h" namespace hemelb { - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); diff --git a/Code/geometry/decomposition/BasicDecomposition.cc b/Code/geometry/decomposition/BasicDecomposition.cc index ed74d3bcf..0f86b361b 100644 --- a/Code/geometry/decomposition/BasicDecomposition.cc +++ b/Code/geometry/decomposition/BasicDecomposition.cc @@ -5,7 +5,6 @@ // license in the file LICENSE. #include "geometry/decomposition/BasicDecomposition.h" -#include "net/mpi.h" namespace hemelb { @@ -16,7 +15,7 @@ namespace hemelb BasicDecomposition::BasicDecomposition(const Geometry& geometry, const lb::lattices::LatticeInfo& latticeInfo, - const net::MpiCommunicator& communicator, + comm::Communicator::ConstPtr communicator, const std::vector& fluidSitesOnEachBlock) : geometry(geometry), latticeInfo(latticeInfo), communicator(communicator), fluidSitesOnEachBlock(fluidSitesOnEachBlock) @@ -42,7 +41,7 @@ namespace hemelb DivideBlocks(procAssignedToEachBlock, unvisitedFluidBlockCount, geometry, - communicator.Size(), + communicator->Size(), fluidSitesOnEachBlock); } @@ -50,7 +49,7 @@ namespace hemelb { log::Logger::Log("Validating procForEachBlock"); - std::vector procForEachBlockRecv = communicator.AllReduce(procAssignedToEachBlock, MPI_MAX); + std::vector procForEachBlockRecv = communicator->AllReduce(procAssignedToEachBlock, MPI_MAX); for (site_t block = 0; block < geometry.GetBlockCount(); ++block) { @@ -74,7 +73,7 @@ namespace hemelb // required on each unit. proc_t currentUnit = 0; - site_t targetBlocksPerUnit = (site_t) ceil((double) unassignedBlocks / (double) (communicator.Size())); + site_t targetBlocksPerUnit = (site_t) ceil((double) unassignedBlocks / (double) (communicator->Size())); // Create an array to monitor whether each block has been assigned yet. std::vector blockAssigned(geometry.GetBlockCount(), false); diff --git a/Code/geometry/decomposition/BasicDecomposition.h b/Code/geometry/decomposition/BasicDecomposition.h index 43f1a85f5..02fb48ca8 100644 --- a/Code/geometry/decomposition/BasicDecomposition.h +++ b/Code/geometry/decomposition/BasicDecomposition.h @@ -9,7 +9,7 @@ #include "geometry/Geometry.h" #include "lb/lattices/LatticeInfo.h" -#include "net/MpiCommunicator.h" +#include "comm/Communicator.h" #include "units.h" #include "util/Vector3D.h" @@ -36,7 +36,7 @@ namespace hemelb */ BasicDecomposition(const Geometry& geometry, const lb::lattices::LatticeInfo& latticeInfo, - const net::MpiCommunicator& communicator, + comm::Communicator::ConstPtr communicator, const std::vector& fluidSitesOnEachBlock); /** @@ -111,7 +111,7 @@ namespace hemelb const Geometry& geometry; //! The geometry being decomposed. const lb::lattices::LatticeInfo& latticeInfo; //! The lattice to decompose for. - const net::MpiCommunicator& communicator; //! The communicator object being decomposed over. + comm::Communicator::ConstPtr communicator; //! The communicator object being decomposed over. const std::vector& fluidSitesOnEachBlock; //! The number of fluid sites on each block in the geometry. }; } /* namespace decomposition */ diff --git a/Code/geometry/decomposition/OptimisedDecomposition.cc b/Code/geometry/decomposition/OptimisedDecomposition.cc index 33e01b8f6..f29a022c4 100644 --- a/Code/geometry/decomposition/OptimisedDecomposition.cc +++ b/Code/geometry/decomposition/OptimisedDecomposition.cc @@ -8,7 +8,9 @@ #include "geometry/decomposition/DecompositionWeights.h" #include "lb/lattices/D3Q27.h" #include "log/Logger.h" -#include "net/net.h" +#include "comm/MpiCommunicator.h" +#include "comm/Async.h" +#include "Exception.h" namespace hemelb { @@ -17,7 +19,7 @@ namespace hemelb namespace decomposition { OptimisedDecomposition::OptimisedDecomposition( - reporting::Timers& timers, net::MpiCommunicator& comms, const Geometry& geometry, + reporting::Timers& timers, comm::Communicator::ConstPtr comms, const Geometry& geometry, const lb::lattices::LatticeInfo& latticeInfo, const std::vector& procForEachBlock, const std::vector& fluidSitesOnEachBlock) : timers(timers), comms(comms), geometry(geometry), latticeInfo(latticeInfo), @@ -42,7 +44,7 @@ namespace hemelb } // Populate the adjacency data arrays (for ParMetis) and validate if appropriate - idx_t localVertexCount = vtxDistribn[comms.Rank() + 1] - vtxDistribn[comms.Rank()]; + idx_t localVertexCount = vtxDistribn[comms->Rank() + 1] - vtxDistribn[comms->Rank()]; PopulateAdjacencyData(localVertexCount); @@ -111,7 +113,7 @@ namespace hemelb // comm* is a pointer to the MPI communicator of the processes involved // Initialise the partition vector. - partitionVector = std::vector(localVertexCount, comms.Rank()); + partitionVector = std::vector(localVertexCount, comms->Rank()); // Weight all vertices evenly. //std::vector < idx_t > vertexWeight(localVertexCount, 1); @@ -120,7 +122,7 @@ namespace hemelb PopulateVertexWeightData(localVertexCount); // Set the weights of each partition to be even, and to sum to 1. - idx_t desiredPartitionSize = comms.Size(); + idx_t desiredPartitionSize = comms->Size(); std::vector domainWeights(desiredPartitionSize, (real_t) (1.0) / ((real_t) (desiredPartitionSize))); @@ -129,7 +131,6 @@ namespace hemelb idx_t weightFlag = 2; idx_t numberingFlag = 0; idx_t edgesCut = 0; - idx_t nDims = 3; idx_t options[4] = { 0, 0, 0, 0 }; if (ShouldValidate()) { @@ -157,7 +158,8 @@ namespace hemelb adjacenciesPerVertex.reserve(1); localAdjacencies.reserve(1); vertexWeights.reserve(1); - MPI_Comm communicator = comms; + auto mpi_comms = std::dynamic_pointer_cast(comms); + MPI_Comm communicator = *mpi_comms; ParMETIS_V3_PartKway(&vtxDistribn[0], &adjacenciesPerVertex[0], &localAdjacencies[0], @@ -175,10 +177,10 @@ namespace hemelb &communicator); log::Logger::Log("ParMetis returned."); - if (comms.Rank() == comms.Size() - 1) + if (comms->Rank() == comms->Size() - 1) { log::Logger::Log("ParMetis cut %d edges.", edgesCut); - if (edgesCut < 1 && comms.Size() > 2) + if (edgesCut < 1 && comms->Size() > 2) { throw Exception() << "The decomposition using ParMetis returned an edge cut of 0 even though there are multiple processes. " @@ -205,7 +207,7 @@ namespace hemelb blockK); // Only consider sites on this processor. - if (procForEachBlock[blockNumber] != comms.Rank()) + if (procForEachBlock[blockNumber] != comms->Rank()) { continue; } @@ -301,14 +303,14 @@ namespace hemelb WallSiteCounter, IOSiteCounter, WallIOSiteCounter, - comms.Rank(), + comms->Rank(), TotalSites, TotalCoreWeight); } void OptimisedDecomposition::PopulateSiteDistribution() { - vtxDistribn.resize(comms.Size() + 1, 0); + vtxDistribn.resize(comms->Size() + 1, 0); // Firstly, count the sites per processor. Do this off-by-one // to be compatible with ParMetis. for (site_t block = 0; block < geometry.GetBlockCount(); ++block) @@ -320,7 +322,7 @@ namespace hemelb } // Now make the count cumulative, again off-by-one. - for (proc_t rank = 0; rank < comms.Size(); ++rank) + for (proc_t rank = 0; rank < comms->Size(); ++rank) { vtxDistribn[rank + 1] += vtxDistribn[rank]; } @@ -364,7 +366,7 @@ namespace hemelb blockJ, blockK); // ... considering only the ones which live on this proc... - if (procForEachBlock[blockNumber] != comms.Rank()) + if (procForEachBlock[blockNumber] != comms->Rank()) { continue; } @@ -451,14 +453,14 @@ namespace hemelb // Right. Let's count how many sites we're going to have to move. Count the local number of // sites to be moved, and collect the site id and the destination processor. std::vector moveData; - const idx_t myLowest = vtxDistribn[comms.Rank()]; - const idx_t myHighest = vtxDistribn[comms.Rank() + 1] - 1; + const idx_t myLowest = vtxDistribn[comms->Rank()]; + const idx_t myHighest = vtxDistribn[comms->Rank() + 1] - 1; // For each local fluid site... for (idx_t ii = 0; ii <= (myHighest - myLowest); ++ii) { // ... if it's going elsewhere... - if (partitionVector[ii] != comms.Rank()) + if (partitionVector[ii] != comms->Rank()) { // ... get its id on the local processor... idx_t localFluidSiteId = myLowest + ii; @@ -514,7 +516,7 @@ namespace hemelb // If we've ended up on an impossible block, or one that doesn't live on this rank, // inform the user. if (fluidSiteBlock >= geometry.GetBlockCount() - || procForEachBlock[fluidSiteBlock] != comms.Rank()) + || procForEachBlock[fluidSiteBlock] != comms->Rank()) { log::Logger::Log("Partition element %i wrongly assigned to block %u of %i (block on processor %i)", ii, @@ -557,12 +559,10 @@ namespace hemelb { timers[hemelb::reporting::Timers::moveForcingNumbers].Start(); - net::Net netForMoveSending(comms); - // We also need to force some data upon blocks, i.e. when they're receiving data from a new // block they didn't previously want to know about. std::map > blockForcedUponX; - std::vector numberOfBlocksIForceUponX(comms.Size(), 0); + std::vector numberOfBlocksIForceUponX(comms->Size(), 0); for (idx_t moveNumber = 0; moveNumber < (idx_t) (moveData.size()); moveNumber += 3) { proc_t target_proc = moveData[moveNumber + 2]; @@ -581,33 +581,35 @@ namespace hemelb // Now find how many blocks are being forced upon us from every other core. log::Logger::Log("Moving forcing block numbers"); - std::vector blocksForcedOnMe = comms.AllToAll(numberOfBlocksIForceUponX); + std::vector blocksForcedOnMe = comms->AllToAll(numberOfBlocksIForceUponX); timers[hemelb::reporting::Timers::moveForcingNumbers].Stop(); + // Now get all the blocks being forced upon me. timers[hemelb::reporting::Timers::moveForcingData].Start(); - // Now get all the blocks being forced upon me. - std::map > blocksForcedOnMeByEachProc; - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + std::map > blocksForcedOnMeByEachProc; { - if (blocksForcedOnMe[otherProc] > 0) - { - blocksForcedOnMeByEachProc[otherProc] = - std::vector(blocksForcedOnMe[otherProc]); - netForMoveSending.RequestReceiveV(blocksForcedOnMeByEachProc[otherProc], otherProc); - } - if (numberOfBlocksIForceUponX[otherProc] > 0) - { - netForMoveSending.RequestSendV(blockForcedUponX[otherProc], otherProc); - } - log::Logger::Log("I'm forcing %i blocks on proc %i.", - numberOfBlocksIForceUponX[otherProc], - otherProc); - } - - log::Logger::Log("Moving forcing block ids"); - netForMoveSending.Dispatch(); + comm::Async requestQ(comms); + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) + { + if (blocksForcedOnMe[otherProc] > 0) + { + blocksForcedOnMeByEachProc[otherProc] = + std::vector(blocksForcedOnMe[otherProc]); + requestQ.Irecv(blocksForcedOnMeByEachProc[otherProc], otherProc); + } + if (numberOfBlocksIForceUponX[otherProc] > 0) + { + requestQ.Isend(blockForcedUponX[otherProc], otherProc); + } + log::Logger::Log("I'm forcing %i blocks on proc %i.", + numberOfBlocksIForceUponX[otherProc], + otherProc); + } + log::Logger::Log("Moving forcing block ids"); + } + // Now go through every block forced upon me and add it to the list of ones I want. - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { if (blocksForcedOnMe[otherProc] > 0) { @@ -681,7 +683,7 @@ namespace hemelb log::Logger::Log("Calculating block requirements"); timers[hemelb::reporting::Timers::blockRequirements].Start(); // Populate numberOfBlocksRequiredFrom - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { numberOfBlocksRequiredFrom[otherProc] = blockIdsIRequireFromX.count(otherProc) == 0 ? 0 : @@ -692,22 +694,23 @@ namespace hemelb } // Now perform the exchange s.t. each core knows how many blocks are required of it from // each other core. - numberOfBlocksXRequiresFromMe = comms.AllToAll(numberOfBlocksRequiredFrom); + numberOfBlocksXRequiresFromMe = comms->AllToAll(numberOfBlocksRequiredFrom); // Awesome. Now we need to get a list of all the blocks wanted from each core by each other // core. - net::Net netForMoveSending(comms); - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) - { - blockIdsXRequiresFromMe[otherProc] = - std::vector(numberOfBlocksXRequiresFromMe[otherProc]); - log::Logger::Log("Proc %i requires %i blocks from me", - otherProc, - blockIdsXRequiresFromMe[otherProc].size()); - netForMoveSending.RequestReceiveV(blockIdsXRequiresFromMe[otherProc], otherProc); - netForMoveSending.RequestSendV(blockIdsIRequireFromX[otherProc], otherProc); + { + comm::Async requestQ(comms); + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) + { + blockIdsXRequiresFromMe[otherProc] = + std::vector(numberOfBlocksXRequiresFromMe[otherProc]); + log::Logger::Log("Proc %i requires %i blocks from me", + otherProc, + blockIdsXRequiresFromMe[otherProc].size()); + requestQ.Irecv(blockIdsXRequiresFromMe[otherProc], otherProc); + requestQ.Isend(blockIdsIRequireFromX[otherProc], otherProc); + } } - netForMoveSending.Dispatch(); timers[hemelb::reporting::Timers::blockRequirements].Stop(); } @@ -724,16 +727,16 @@ namespace hemelb // block has no moves. for (site_t blockId = 0; blockId < geometry.GetBlockCount(); ++blockId) { - if (procForEachBlock[blockId] == comms.Rank()) + if (procForEachBlock[blockId] == comms->Rank()) { movesForEachLocalBlock[blockId] = 0; } } - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) { for (site_t blockNum = 0; - blockNum < (site_t) ( ( ( ( (blockIdsXRequiresFromMe[otherProc].size()))))); + blockNum < site_t(blockIdsXRequiresFromMe[otherProc].size()); ++blockNum) { site_t blockId = blockIdsXRequiresFromMe[otherProc][blockNum]; @@ -749,7 +752,7 @@ namespace hemelb } - for (site_t moveNumber = 0; moveNumber < (site_t) ( ( ( ( (moveData.size()))))); + for (site_t moveNumber = 0; moveNumber < site_t(moveData.size()); moveNumber += 3) { site_t blockId = moveData[moveNumber]; @@ -763,29 +766,30 @@ namespace hemelb movesForEachLocalBlock[blockId]++; } - net::Net netForMoveSending(comms); - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) - { - for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); - it != blockIdsIRequireFromX[otherProc].end(); ++it) - { - netForMoveSending.RequestReceiveR(movesForEachBlockWeCareAbout[*it], otherProc); - log::Logger::Log("I want the move count for block %i from proc %i", - *it, - otherProc); - } - for (std::vector::iterator it = blockIdsXRequiresFromMe[otherProc].begin(); - it != blockIdsXRequiresFromMe[otherProc].end(); ++it) - { - netForMoveSending.RequestSendR(movesForEachLocalBlock[*it], otherProc); - log::Logger::Log("I'm sending move count for block %i to proc %i", - *it, - otherProc); - } - } - - log::Logger::Log("Sending move counts"); - netForMoveSending.Dispatch(); + { + comm::Async requestQ(comms); + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) + { + for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); + it != blockIdsIRequireFromX[otherProc].end(); ++it) + { + requestQ.Irecv(movesForEachBlockWeCareAbout[*it], otherProc); + log::Logger::Log("I want the move count for block %i from proc %i", + *it, + otherProc); + } + for (std::vector::iterator it = blockIdsXRequiresFromMe[otherProc].begin(); + it != blockIdsXRequiresFromMe[otherProc].end(); ++it) + { + requestQ.Isend(movesForEachLocalBlock[*it], otherProc); + log::Logger::Log("I'm sending move count for block %i to proc %i", + *it, + otherProc); + } + } + + log::Logger::Log("Sending move counts"); + } timers[hemelb::reporting::Timers::moveCountsSending].Stop(); } @@ -808,48 +812,48 @@ namespace hemelb movesList.resize(totalMovesToReceive * 3); idx_t localMoveId = 0; - net::Net netForMoveSending(comms); - - for (proc_t otherProc = 0; otherProc < (proc_t) ( ( ( ( (comms.Size()))))); ++otherProc) - { - allMoves[otherProc] = 0; - for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); - it != blockIdsIRequireFromX[otherProc].end(); ++it) - { - if (movesForEachBlockWeCareAbout[*it] > 0) - { - netForMoveSending.RequestReceive(&movesList[localMoveId * 3], - 3 * movesForEachBlockWeCareAbout[*it], - otherProc); - localMoveId += movesForEachBlockWeCareAbout[*it]; - allMoves[otherProc] += movesForEachBlockWeCareAbout[*it]; - log::Logger::Log("Expect %i moves from from proc %i about block %i", - movesForEachBlockWeCareAbout[*it], - otherProc, - *it); - } - } - - for (std::vector::iterator it = blockIdsXRequiresFromMe[otherProc].begin(); - it != blockIdsXRequiresFromMe[otherProc].end(); ++it) - { - if (moveDataForEachBlock[*it].size() > 0) - { - netForMoveSending.RequestSendV(moveDataForEachBlock[*it], otherProc); - log::Logger::Log("Sending %i moves from to proc %i about block %i", - moveDataForEachBlock[*it].size() / 3, - otherProc, - *it); - } - } - - log::Logger::Log("%i moves from proc %i", - allMoves[otherProc], - otherProc); - } - - log::Logger::Log("Sending move data"); - netForMoveSending.Dispatch(); + { + comm::Async requestQ(comms); + for (proc_t otherProc = 0; otherProc < proc_t(comms->Size()); ++otherProc) + { + allMoves[otherProc] = 0; + for (std::vector::iterator it = blockIdsIRequireFromX[otherProc].begin(); + it != blockIdsIRequireFromX[otherProc].end(); ++it) + { + if (movesForEachBlockWeCareAbout[*it] > 0) + { + requestQ.Irecv(&movesList[localMoveId * 3], + 3 * movesForEachBlockWeCareAbout[*it], + otherProc, 0); + localMoveId += movesForEachBlockWeCareAbout[*it]; + allMoves[otherProc] += movesForEachBlockWeCareAbout[*it]; + log::Logger::Log("Expect %i moves from from proc %i about block %i", + movesForEachBlockWeCareAbout[*it], + otherProc, + *it); + } + } + + for (std::vector::iterator it = blockIdsXRequiresFromMe[otherProc].begin(); + it != blockIdsXRequiresFromMe[otherProc].end(); ++it) + { + if (moveDataForEachBlock[*it].size() > 0) + { + requestQ.Isend(moveDataForEachBlock[*it], otherProc); + log::Logger::Log("Sending %i moves from to proc %i about block %i", + moveDataForEachBlock[*it].size() / 3, + otherProc, + *it); + } + } + + log::Logger::Log("%i moves from proc %i", + allMoves[otherProc], + otherProc); + } + + log::Logger::Log("Sending move data"); + } timers[hemelb::reporting::Timers::moveDataSending].Stop(); } @@ -865,7 +869,7 @@ namespace hemelb */ void OptimisedDecomposition::PopulateMovesList() { - allMoves = std::vector(comms.Size()); + allMoves = std::vector(comms->Size()); // Create a map for looking up block Ids: the map is from the contiguous site index // of the last fluid site on the block, to the block id. @@ -909,8 +913,8 @@ namespace hemelb // Now we want to spread this info around so that each core knows which blocks each other // requires from it. - std::vector numberOfBlocksRequiredFrom(comms.Size(), 0); - std::vector numberOfBlocksXRequiresFromMe(comms.Size(), 0); + std::vector numberOfBlocksRequiredFrom(comms->Size(), 0); + std::vector numberOfBlocksXRequiresFromMe(comms->Size(), 0); std::map > blockIdsXRequiresFromMe; GetBlockRequirements(numberOfBlocksRequiredFrom, blockIdsIRequireFromX, @@ -945,14 +949,14 @@ namespace hemelb // Right. Let's count how many sites we're going to have to move. Count the local number of // sites to be moved, and collect the site id and the destination processor. std::vector moveData; - const idx_t myLowest = vtxDistribn[comms.Rank()]; - const idx_t myHighest = vtxDistribn[comms.Rank() + 1] - 1; + const idx_t myLowest = vtxDistribn[comms->Rank()]; + const idx_t myHighest = vtxDistribn[comms->Rank() + 1] - 1; // For each local fluid site... for (idx_t ii = 0; ii <= (myHighest - myLowest); ++ii) { // ... if it's going elsewhere... - if (partitionVector[ii] != comms.Rank()) + if (partitionVector[ii] != comms->Rank()) { // ... get its id on the local processor... idx_t localFluidSiteId = myLowest + ii; @@ -991,7 +995,7 @@ namespace hemelb /* Below is a modified version of PopulateMovesList(). */ void OptimisedDecomposition::LoadDecomposition() { - allMoves = std::vector(comms.Size()); + allMoves = std::vector(comms->Size()); // Create a map for looking up block Ids: the map is from the contiguous site index // of the last fluid site on the block, to the block id. @@ -1035,8 +1039,8 @@ namespace hemelb // Now we want to spread this info around so that each core knows which blocks each other // requires from it. - std::vector numberOfBlocksRequiredFrom(comms.Size(), 0); - std::vector numberOfBlocksXRequiresFromMe(comms.Size(), 0); + std::vector numberOfBlocksRequiredFrom(comms->Size(), 0); + std::vector numberOfBlocksXRequiresFromMe(comms->Size(), 0); std::map > blockIdsXRequiresFromMe; GetBlockRequirements(numberOfBlocksRequiredFrom, blockIdsIRequireFromX, @@ -1078,9 +1082,9 @@ namespace hemelb { log::Logger::Log("Validating the vertex distribution."); // vtxDistribn should be the same on all cores. - std::vector vtxDistribnRecv = comms.AllReduce(vtxDistribn, MPI_MIN); + std::vector vtxDistribnRecv = comms->AllReduce(vtxDistribn, MPI_MIN); - for (proc_t rank = 0; rank < comms.Size() + 1; ++rank) + for (proc_t rank = 0; rank < comms->Size() + 1; ++rank) { if (vtxDistribn[rank] != vtxDistribnRecv[rank]) { @@ -1102,12 +1106,12 @@ namespace hemelb log::Logger::Log("Validating the graph adjacency structure"); // Create an array of lists to store all of this node's adjacencies, arranged by the // proc the adjacent vertex is on. - std::vector > adjByNeighProc(comms.Size(), + std::vector > adjByNeighProc(comms->Size(), std::multimap()); // The adjacency data should correspond across all cores. for (idx_t index = 0; index < localVertexCount; ++index) { - idx_t vertex = vtxDistribn[comms.Rank()] + index; + idx_t vertex = vtxDistribn[comms->Rank()] + index; // Iterate over each adjacency (of each vertex). for (idx_t adjNumber = 0; adjNumber < (adjacenciesPerVertex[index + 1] - adjacenciesPerVertex[index]); @@ -1116,7 +1120,7 @@ namespace hemelb idx_t adjacentVertex = localAdjacencies[adjacenciesPerVertex[index] + adjNumber]; proc_t adjacentProc = -1; // Calculate the proc of the neighbouring vertex. - for (proc_t proc = 0; proc < comms.Size(); ++proc) + for (proc_t proc = 0; proc < comms->Size(); ++proc) { if (vtxDistribn[proc] <= adjacentVertex && vtxDistribn[proc + 1] > adjacentVertex) { @@ -1140,18 +1144,18 @@ namespace hemelb } // Create variables for the neighbour data to go into. - std::vector counts(comms.Size()); - std::vector > data(comms.Size()); + std::vector counts(comms->Size()); + std::vector > data(comms->Size()); log::Logger::Log("Validating neighbour data"); // Now spread and compare the adjacency information. Larger ranks send data to smaller // ranks which receive the data and compare it. - for (proc_t neigh = 0; neigh < (proc_t) ( ( ( ( (comms.Size()))))); ++neigh) + for (proc_t neigh = 0; neigh < proc_t(comms->Size()); ++neigh) { SendAdjacencyDataToLowerRankedProc(neigh, counts[neigh], data[neigh], adjByNeighProc[neigh]); - if (neigh < comms.Rank()) + if (neigh < comms->Rank()) { // Sending arrays don't perform comparison. continue; @@ -1168,11 +1172,11 @@ namespace hemelb std::vector& neighboursAdjacencyData, std::multimap& expectedAdjacencyData) { - if (neighbouringProc < comms.Rank()) + if (neighbouringProc < comms->Rank()) { // Send the array length. neighboursAdjacencyCount = 2 * expectedAdjacencyData.size(); - comms.Send(neighboursAdjacencyCount, neighbouringProc, 42); + comms->Send(neighboursAdjacencyCount, neighbouringProc, 42); // Create a sendable array (std::lists aren't organised in a sendable format). neighboursAdjacencyData.resize(neighboursAdjacencyCount); unsigned int adjacencyIndex = 0; @@ -1184,15 +1188,15 @@ namespace hemelb ++adjacencyIndex; } // Send the data to the neighbouringProc. - comms.Send(neighboursAdjacencyData, neighbouringProc, 43); + comms->Send(neighboursAdjacencyData, neighbouringProc, 43); } else // If this is a greater rank number than the neighbouringProc, receive the data. - if (neighbouringProc > comms.Rank()) + if (neighbouringProc > comms->Rank()) { - comms.Receive(neighboursAdjacencyCount, neighbouringProc, 42); + comms->Recv(neighboursAdjacencyCount, neighbouringProc, 42); neighboursAdjacencyData.resize(neighboursAdjacencyCount); - comms.Receive(neighboursAdjacencyData, neighbouringProc, 43); + comms->Recv(neighboursAdjacencyData, neighbouringProc, 43); } else // Neigh == mTopologyRank, i.e. neighbouring vertices on the same proc // Duplicate the data. @@ -1269,7 +1273,7 @@ namespace hemelb // Reduce finding the maximum across all nodes. Note that we have to use the maximum // because some cores will have -1 for a block (indicating that it has no neighbours on // that block. - std::vector firstSiteIndexPerBlockRecv = comms.AllReduce(firstSiteIndexPerBlock, + std::vector firstSiteIndexPerBlockRecv = comms->AllReduce(firstSiteIndexPerBlock, MPI_MAX); for (site_t block = 0; block < geometry.GetBlockCount(); ++block) diff --git a/Code/geometry/decomposition/OptimisedDecomposition.h b/Code/geometry/decomposition/OptimisedDecomposition.h index 8ad4e134f..0853ab664 100644 --- a/Code/geometry/decomposition/OptimisedDecomposition.h +++ b/Code/geometry/decomposition/OptimisedDecomposition.h @@ -12,7 +12,7 @@ #include "lb/lattices/LatticeInfo.h" #include "geometry/ParmetisHeader.h" #include "reporting/Timers.h" -#include "net/MpiCommunicator.h" +#include "comm/Communicator.h" #include "geometry/SiteData.h" #include "geometry/GeometryBlock.h" @@ -25,7 +25,7 @@ namespace hemelb class OptimisedDecomposition { public: - OptimisedDecomposition(reporting::Timers& timers, net::MpiCommunicator& comms, + OptimisedDecomposition(reporting::Timers& timers, comm::Communicator::ConstPtr comms, const Geometry& geometry, const lb::lattices::LatticeInfo& latticeInfo, const std::vector& procForEachBlock, @@ -206,7 +206,7 @@ namespace hemelb std::map > moveDataForEachBlock); reporting::Timers& timers; //! Timers for reporting. - net::MpiCommunicator& comms; //! Communicator + comm::Communicator::ConstPtr comms; //! Communicator const Geometry& geometry; //! The geometry being optimised. const lb::lattices::LatticeInfo& latticeInfo; //! The lattice info to optimise for. const std::vector& procForEachBlock; //! The processor assigned to each block at the moment diff --git a/Code/geometry/needs/Needs.cc b/Code/geometry/needs/Needs.cc index 8c9ed0bab..44dee944c 100644 --- a/Code/geometry/needs/Needs.cc +++ b/Code/geometry/needs/Needs.cc @@ -15,9 +15,9 @@ namespace hemelb Needs::Needs(const site_t blockCount, const std::vector& readBlock, const proc_t readingGroupSize, - net::InterfaceDelegationNet & net, + comm::Communicator::ConstPtr comm, bool shouldValidate_) : - procsWantingBlocksBuffer(blockCount), communicator(net.GetCommunicator()), readingGroupSize(readingGroupSize), shouldValidate(shouldValidate_) + procsWantingBlocksBuffer(blockCount), communicator(comm), readingGroupSize(readingGroupSize), shouldValidate(shouldValidate_) { // Compile the blocks needed here into an array of indices, instead of an array of bools std::vector > blocksNeededHere(readingGroupSize); @@ -31,38 +31,30 @@ namespace hemelb // Share the counts of needed blocks int blocksNeededSize[readingGroupSize]; - std::vector blocksNeededSizes(communicator.Size()); + std::vector blocksNeededSizes; for (proc_t readingCore = 0; readingCore < readingGroupSize; readingCore++) { blocksNeededSize[readingCore] = blocksNeededHere[readingCore].size(); - net.RequestGatherSend(blocksNeededSize[readingCore], readingCore); - - } - if (communicator.Rank() < readingGroupSize) - { - net.RequestGatherReceive(blocksNeededSizes); + auto tmp = communicator->Gather(blocksNeededSize[readingCore], readingCore); + if (readingCore == communicator->Rank()) + blocksNeededSizes = std::move(tmp); } - net.Dispatch(); + // Communicate the arrays of needed blocks - + std::vector blocksNeededOn; for (proc_t readingCore = 0; readingCore < readingGroupSize; readingCore++) { - net.RequestGatherVSend(blocksNeededHere[readingCore], readingCore); + auto tmp = communicator->GatherV(blocksNeededHere[readingCore], blocksNeededSizes, readingCore); + if (readingCore == communicator->Rank()) + blocksNeededOn = std::move(tmp); } - std::vector blocksNeededOn; - - if (communicator.Rank() < readingGroupSize) - { - net.RequestGatherVReceive(blocksNeededOn, blocksNeededSizes); - } - net.Dispatch(); - if (communicator.Rank() < readingGroupSize) + if (communicator->Rank() < readingGroupSize) { int needsPassed = 0; // Transpose the blocks needed on cores matrix - for (proc_t sendingCore = 0; sendingCore < communicator.Size(); sendingCore++) + for (proc_t sendingCore = 0; sendingCore < communicator->Size(); sendingCore++) { for (int needForThisSendingCore = 0; needForThisSendingCore < blocksNeededSizes[sendingCore]; ++needForThisSendingCore) @@ -87,12 +79,12 @@ namespace hemelb { int neededHere = readBlock[block]; proc_t readingCore = GetReadingCoreForBlock(block); - std::vector procsWantingThisBlockBuffer = communicator.Gather(neededHere, readingCore); + std::vector procsWantingThisBlockBuffer = communicator->Gather(neededHere, readingCore); - if (communicator.Rank() == readingCore) + if (communicator->Rank() == readingCore) { - for (proc_t needingProcOld = 0; needingProcOld < communicator.Size(); needingProcOld++) + for (proc_t needingProcOld = 0; needingProcOld < communicator->Size(); needingProcOld++) { bool found = false; for (std::vector::iterator needingProc = procsWantingBlocksBuffer[block].begin(); diff --git a/Code/geometry/needs/Needs.h b/Code/geometry/needs/Needs.h index f6c8ab31e..a878823ef 100644 --- a/Code/geometry/needs/Needs.h +++ b/Code/geometry/needs/Needs.h @@ -7,8 +7,9 @@ #ifndef HEMELB_GEOMETRY_NEEDS_NEEDS_H #define HEMELB_GEOMETRY_NEEDS_NEEDS_H #include -#include "net/net.h" -#include "net/IOCommunicator.h" +#include "units.h" +#include "comm/Communicator.h" + namespace hemelb { namespace geometry @@ -20,6 +21,7 @@ namespace hemelb * Class defining HemeLB needs communication Used by geometry reader to know where to send which blocks. */ + class Needs { public: @@ -28,13 +30,13 @@ namespace hemelb * @param BlockCount Count of blocks * @param readBlock Which cores need which blocks, as an array of booleans. * @param readingGroupSize Number sof cores to use for reading blocks - * @param net Instance of Net communication class to use. + * @param comm MPI communicator. */ Needs(const site_t blockCount, - const std::vector& readBlock, - const proc_t readingGroupSize, - net::InterfaceDelegationNet &net, - bool shouldValidate); // Temporarily during the refactor, constructed just to abstract the block sharing bit + const std::vector& readBlock, + const proc_t readingGroupSize, + comm::Communicator::ConstPtr comm, + bool shouldValidate); // Temporarily during the refactor, constructed just to abstract the block sharing bit /*** * Which processors need a given block? @@ -56,7 +58,7 @@ namespace hemelb proc_t GetReadingCoreForBlock(const site_t blockNumber) const; private: std::vector > procsWantingBlocksBuffer; - const net::MpiCommunicator & communicator; + comm::Communicator::ConstPtr communicator; const proc_t readingGroupSize; bool shouldValidate; void Validate(const site_t blockCount, const std::vector& readBlock); diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.cc b/Code/geometry/neighbouring/NeighbouringDataManager.cc index e45a341f7..fdb635d3e 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.cc +++ b/Code/geometry/neighbouring/NeighbouringDataManager.cc @@ -8,8 +8,9 @@ #include "geometry/neighbouring/NeighbouringDataManager.h" #include "geometry/LatticeData.h" - #include "log/Logger.h" +#include "comm/MapAllToAll.h" + namespace hemelb { namespace geometry @@ -19,10 +20,9 @@ namespace hemelb NeighbouringDataManager::NeighbouringDataManager( const LatticeData & localLatticeData, NeighbouringLatticeData & neighbouringLatticeData, - net::InterfaceDelegationNet & net) : + comm::Async::Ptr cq) : localLatticeData(localLatticeData), neighbouringLatticeData(neighbouringLatticeData), - net(net), needsEachProcHasFromMe(net.Size()), - needsHaveBeenShared(false) + commQ(cq), needsHaveBeenShared(false) { } void NeighbouringDataManager::RegisterNeededSite(site_t globalId, @@ -43,60 +43,68 @@ namespace hemelb { return localLatticeData.ProcProvidingSiteByGlobalNoncontiguousId(site); } - + + // TODO: figure out if this needs to force the commQ to wait + // (current behaviour) or if it can create a new queue and wait + // on that only. void NeighbouringDataManager::TransferNonFieldDependentInformation() { // Ordering is important here, to ensure the requests are registered in the same order // on the sending and receiving procs. // But, the needsEachProcHasFromMe is always ordered, // by the same order, as the neededSites, so this should be OK. - for (std::vector::iterator localNeed = neededSites.begin(); + for (IdVec::iterator localNeed = neededSites.begin(); localNeed != neededSites.end(); localNeed++) { proc_t source = ProcForSite(*localNeed); NeighbouringSite site = neighbouringLatticeData.GetSite(*localNeed); - - net.RequestReceiveR(site.GetSiteData().GetWallIntersectionData(), source); - net.RequestReceiveR(site.GetSiteData().GetIoletIntersectionData(), source); - net.RequestReceiveR(site.GetSiteData().GetIoletId(), source); - net.RequestReceiveR(site.GetSiteData().GetSiteType(), source); - net.RequestReceive(site.GetWallDistances(), - localLatticeData.GetLatticeInfo().GetNumVectors() - 1, - source); - net.RequestReceiveR(site.GetWallNormal(), source); + commQ->Irecv(site.GetSiteData().GetWallIntersectionData(), source); + commQ->Irecv(site.GetSiteData().GetIoletIntersectionData(), source); + commQ->Irecv(site.GetSiteData().GetIoletId(), source); + commQ->Irecv(site.GetSiteData().GetSiteType(), source); + commQ->Irecv(site.GetWallDistances(), + localLatticeData.GetLatticeInfo().GetNumVectors() - 1, + source); + commQ->Irecv(site.GetWallNormal(), source); } - for (proc_t other = 0; other < net.Size(); other++) + + for (IdsMap::const_iterator iter = needsEachProcHasFromMe.begin(); + iter != needsEachProcHasFromMe.end(); + ++iter) { - for (std::vector::iterator needOnProcFromMe = - needsEachProcHasFromMe[other].begin(); - needOnProcFromMe != needsEachProcHasFromMe[other].end(); needOnProcFromMe++) + proc_t other = iter->first; + const IdVec& neededIds = iter->second; + for (IdVec::const_iterator needOnProcFromMe = neededIds.begin(); + needOnProcFromMe != neededIds.end(); + ++needOnProcFromMe) { site_t localContiguousId = - localLatticeData.GetLocalContiguousIdFromGlobalNoncontiguousId(*needOnProcFromMe); + localLatticeData.GetLocalContiguousIdFromGlobalNoncontiguousId(*needOnProcFromMe); - Site site = - const_cast(localLatticeData).GetSite(localContiguousId); - // have to cast away the const, because no respect for const-ness for sends in MPI - net.RequestSendR(site.GetSiteData().GetWallIntersectionData(), other); - net.RequestSendR(site.GetSiteData().GetIoletIntersectionData(), other); - net.RequestSendR(site.GetSiteData().GetIoletId(), other); - net.RequestSendR(site.GetSiteData().GetSiteType(), other); - net.RequestSend(site.GetWallDistances(), + const Site site = localLatticeData.GetSite(localContiguousId); + const SiteData& sd = site.GetSiteData(); + + commQ->Isend(sd.GetWallIntersectionData(), other); + commQ->Isend(sd.GetIoletIntersectionData(), other); + commQ->Isend(sd.GetIoletId(), other); + commQ->Isend(sd.GetSiteType(), other); + commQ->Isend(site.GetWallDistances(), localLatticeData.GetLatticeInfo().GetNumVectors() - 1, other); - net.RequestSendR(site.GetWallNormal(), other); + commQ->Isend(site.GetWallNormal(), other); } } - net.Dispatch(); + commQ->Wait(); } void NeighbouringDataManager::TransferFieldDependentInformation() { - RequestComms(); - net.Dispatch(); + Receive(); + Send(); + commQ->Wait(); } - void NeighbouringDataManager::RequestComms() + void NeighbouringDataManager::Receive() { /*if (needsHaveBeenShared == false) { @@ -108,31 +116,36 @@ namespace hemelb // on the sending and receiving procs. // But, the needsEachProcHasFromMe is always ordered, // by the same order, as the neededSites, so this should be OK. - for (std::vector::iterator localNeed = neededSites.begin(); + for (IdVec::iterator localNeed = neededSites.begin(); localNeed != neededSites.end(); localNeed++) { proc_t source = ProcForSite(*localNeed); NeighbouringSite site = neighbouringLatticeData.GetSite(*localNeed); - net.RequestReceive(site.GetFOld(localLatticeData.GetLatticeInfo().GetNumVectors()), + commQ->Irecv(site.GetFOld(localLatticeData.GetLatticeInfo().GetNumVectors()), localLatticeData.GetLatticeInfo().GetNumVectors(), source); } - for (proc_t other = 0; other < net.Size(); other++) + } + void NeighbouringDataManager::Send() + { + const unsigned Q = localLatticeData.GetLatticeInfo().GetNumVectors(); + for (IdsMap::const_iterator iter = needsEachProcHasFromMe.begin(); + iter != needsEachProcHasFromMe.end(); + ++iter) { - for (std::vector::iterator needOnProcFromMe = - needsEachProcHasFromMe[other].begin(); - needOnProcFromMe != needsEachProcHasFromMe[other].end(); needOnProcFromMe++) + proc_t other = iter->first; + const IdVec& neededIds = iter->second; + for (IdVec::const_iterator needOnProcFromMe = neededIds.begin(); + needOnProcFromMe != neededIds.end(); ++needOnProcFromMe) { site_t localContiguousId = localLatticeData.GetLocalContiguousIdFromGlobalNoncontiguousId(*needOnProcFromMe); - Site site = - const_cast(localLatticeData).GetSite(localContiguousId); + const Site site = localLatticeData.GetSite(localContiguousId); // have to cast away the const, because no respect for const-ness for sends in MPI - net.RequestSend(const_cast(site.GetFOld(localLatticeData.GetLatticeInfo().GetNumVectors())), - localLatticeData.GetLatticeInfo().GetNumVectors(), + commQ->Isend(site.GetFOld(Q), + Q, other); - } } } @@ -143,39 +156,50 @@ namespace hemelb //if (needsHaveBeenShared == true) // return; //TODO: Fix! - // build a table of which procs needs can be achieved from which proc - std::vector > needsIHaveFromEachProc(net.Size()); - std::vector countOfNeedsIHaveFromEachProc(net.Size(), 0); - for (std::vector::iterator localNeed = neededSites.begin(); + // build a table of which sites are needed by this rank, by other rank. + IdsMap needsIHaveFromEachProc; + // This map will count the number per-rank + CountMap countOfNeedsIHaveFromEachProc; + for (IdVec::iterator localNeed = neededSites.begin(); localNeed != neededSites.end(); localNeed++) { - needsIHaveFromEachProc[ProcForSite(*localNeed)].push_back(*localNeed); - countOfNeedsIHaveFromEachProc[ProcForSite(*localNeed)]++; - + site_t needId = *localNeed; + int needRank = ProcForSite(needId); + needsIHaveFromEachProc[needRank].push_back(needId); + countOfNeedsIHaveFromEachProc[needRank]++; } - // every proc must send to all procs, how many it needs from that proc - net.RequestAllToAllSend(countOfNeedsIHaveFromEachProc); - - // every proc must receive from all procs, how many it needs to give that proc - std::vector countOfNeedsOnEachProcFromMe(net.Size(), 0); - net.RequestAllToAllReceive(countOfNeedsOnEachProcFromMe); - net.Dispatch(); - - for (proc_t other = 0; other < net.Size(); other++) + // This will store the numbers of sites other ranks need from this rank + CountMap countOfNeedsOnEachProcFromMe; + auto comms = commQ->GetComm(); + // This is collective + comm::MapAllToAll(comms, countOfNeedsIHaveFromEachProc, countOfNeedsOnEachProcFromMe, 1234); + + //comm::Request::ReqVec requestQueue; + auto requestQueue = comms->MakeRequestList(); + + // Now, for every rank, which I need something from, send the ids of those + for (CountMap::const_iterator countIt = countOfNeedsIHaveFromEachProc.begin(); + countIt != countOfNeedsIHaveFromEachProc.end(); + ++countIt) { + int other = countIt->first; + requestQueue->push_back(comms->Isend(needsIHaveFromEachProc[other], other)); + } - // now, for every proc, which I need something from,send the ids of those - net.RequestSendV(needsIHaveFromEachProc[other], other); - // and, for every proc, which needs something from me, receive those ids - needsEachProcHasFromMe[other].resize(countOfNeedsOnEachProcFromMe[other]); - net.RequestReceiveV(needsEachProcHasFromMe[other], other); - // In principle, this bit could have been implemented as a separate GatherV onto every proc - // However, in practice, we expect the needs to be basically local - // so using point-to-point will be more efficient. + // And for every rank, which needs something from me, receive those ids + for (CountMap::const_iterator countIt = countOfNeedsOnEachProcFromMe.begin(); + countIt != countOfNeedsOnEachProcFromMe.end(); + ++countIt) + { + int other = countIt->first; + int size = countIt->second; + IdVec& otherNeeds = needsEachProcHasFromMe[other]; + otherNeeds.resize(size); + requestQueue->push_back(comms->Irecv(otherNeeds, other)); } - net.Dispatch(); + requestQueue->WaitAll(); needsHaveBeenShared = true; } } diff --git a/Code/geometry/neighbouring/NeighbouringDataManager.h b/Code/geometry/neighbouring/NeighbouringDataManager.h index e5b16c3da..e3e49d0b3 100644 --- a/Code/geometry/neighbouring/NeighbouringDataManager.h +++ b/Code/geometry/neighbouring/NeighbouringDataManager.h @@ -10,8 +10,7 @@ #include "geometry/LatticeData.h" #include "geometry/neighbouring/NeighbouringLatticeData.h" #include "geometry/neighbouring/RequiredSiteInformation.h" -#include "net/net.h" -#include "net/IteratedAction.h" +#include "timestep/Actor.h" #include #include @@ -24,18 +23,22 @@ namespace hemelb namespace neighbouring { - class NeighbouringDataManager : public net::IteratedAction + class NeighbouringDataManager : public timestep::Actor { public: NeighbouringDataManager(const LatticeData & localLatticeData, NeighbouringLatticeData & neighbouringLatticeData, - net::InterfaceDelegationNet & net); + comm::Async::Ptr cq); // Initially, the required site information will not be used -- we just transfer everything. // This considerably simplifies matters. // Nevertheless, we provide the interface here in its final form void RegisterNeededSite(site_t globalId, RequiredSiteInformation requirements = RequiredSiteInformation(true)); + + // This is collective across the communicator used by the + // communication queue given to the constructor void ShareNeeds(); + std::vector &GetNeedsForProc(proc_t proc) { return needsEachProcHasFromMe[proc]; @@ -49,14 +52,33 @@ namespace hemelb // NB this is virtual so that the class can be tested. virtual proc_t ProcForSite(site_t site); protected: - void RequestComms(); + inline virtual void BeginAll() { + } + inline virtual void Begin() { + } + // Implemented + virtual void Receive(); + inline virtual void PreSend() { + } + // Implemented + virtual void Send(); + inline virtual void PreWait() { + } + inline virtual void Wait() { + } + inline virtual void End() { + } + inline virtual void EndAll() { + } private: const LatticeData & localLatticeData; NeighbouringLatticeData & neighbouringLatticeData; - net::InterfaceDelegationNet & net; - - std::vector neededSites; - std::vector > needsEachProcHasFromMe; + comm::Async::Ptr commQ; + typedef std::vector IdVec; + typedef std::map CountMap; + typedef std::map IdsMap; + IdVec neededSites; + IdsMap needsEachProcHasFromMe; bool needsHaveBeenShared; diff --git a/Code/geometry/neighbouring/RequiredSiteInformation.cc b/Code/geometry/neighbouring/RequiredSiteInformation.cc index 6601cf626..4eea16c0f 100644 --- a/Code/geometry/neighbouring/RequiredSiteInformation.cc +++ b/Code/geometry/neighbouring/RequiredSiteInformation.cc @@ -5,7 +5,7 @@ // license in the file LICENSE. #include "geometry/neighbouring/RequiredSiteInformation.h" -#include "net/mpi.h" + namespace hemelb { namespace geometry diff --git a/Code/lb/CMakeLists.txt b/Code/lb/CMakeLists.txt index 8bacdd357..4d38a949e 100644 --- a/Code/lb/CMakeLists.txt +++ b/Code/lb/CMakeLists.txt @@ -4,7 +4,7 @@ # file AUTHORS. This software is provided under the terms of the # license in the file LICENSE. add_library(hemelb_lb - iolets/BoundaryCommunicator.cc iolets/BoundaryComms.cc iolets/BoundaryValues.cc + iolets/BoundaryValues.cc iolets/InOutLet.cc iolets/InOutLetCosine.cc iolets/InOutLetFile.cc iolets/InOutLetMultiscale.cc @@ -15,5 +15,5 @@ add_library(hemelb_lb kernels/rheologyModels/AbstractRheologyModel.cc kernels/rheologyModels/CarreauYasudaRheologyModel.cc kernels/rheologyModels/CassonRheologyModel.cc kernels/rheologyModels/TruncatedPowerLawRheologyModel.cc lattices/LatticeInfo.cc lattices/D3Q15.cc lattices/D3Q19.cc lattices/D3Q27.cc lattices/D3Q15i.cc - MacroscopicPropertyCache.cc SimulationState.cc StabilityTester.cc + MacroscopicPropertyCache.cc SimulationState.cc ) diff --git a/Code/lb/EntropyTester.h b/Code/lb/EntropyTester.h index 759ad9b31..8a35a1f3f 100644 --- a/Code/lb/EntropyTester.h +++ b/Code/lb/EntropyTester.h @@ -7,7 +7,7 @@ #ifndef HEMELB_LB_ENTROPYTESTER_H #define HEMELB_LB_ENTROPYTESTER_H -#include "net/PhasedBroadcastRegular.h" +#include "timestep/CollectiveActor.h" #include "geometry/LatticeData.h" #include "lb/HFunction.h" #include "log/Logger.h" @@ -17,15 +17,15 @@ namespace hemelb namespace lb { template - class EntropyTester : public net::PhasedBroadcastRegular + class EntropyTester : public timestep::CollectiveActor { public: - EntropyTester(int* collisionTypes, - unsigned int typesTested, - const geometry::LatticeData * iLatDat, - net::Net* net, - SimulationState* simState) : - net::PhasedBroadcastRegular(net, simState, SPREADFACTOR), mLatDat(iLatDat) + EntropyTester(int* collisionTypes, unsigned int typesTested, + const geometry::LatticeData * iLatDat, comm::Communicator::ConstPtr comm, + SimulationState* simState, reporting::Timers& timings) : + timestep::CollectiveActor(comm, timings[reporting::Timers::mpiWait]), + mLatDat(iLatDat), mHPreCollision(mLatDat->GetLocalFluidSiteCount()), + workTimer(timings[reporting::Timers::monitoring]) { for (unsigned int i = 0; i < COLLISION_TYPES; i++) { @@ -36,37 +36,25 @@ namespace hemelb mCollisionTypesTested[collisionTypes[i]] = true; } - mHPreCollision = new double[mLatDat->GetLocalFluidSiteCount()]; - Reset(); } - ~EntropyTester() - { - delete[] mHPreCollision; - } - - void PreReceive() + // Run at BeginPhase + void RequestComms(void) { + // Store pre-collision values. site_t offset = 0; - double dHMax = 0.0; - - // The order of arguments in max is important - // If distributions go negative HFunc.eval() will return a NaN. Due to the - // nature of NaN and the structure of max, dH will be assigned as NaN if this is the case - // This is what we want, because the EntropyTester will fail otherwise and abort when - // it is simply sufficient to wait until StabilityTester restarts. for (unsigned int collision_type = 0; collision_type < COLLISION_TYPES; collision_type++) { if (mCollisionTypesTested[collision_type]) { - for (site_t i = offset; i < offset + mLatDat->GetMidDomainCollisionCount(collision_type); i++) + for (site_t i = offset; + i < offset + mLatDat->GetMidDomainCollisionCount(collision_type); i++) { - const geometry::Site site = mLatDat->GetSite(i); - - HFunction HFunc(site.GetFOld(), NULL); - dHMax = util::NumericalFunctions::max(dHMax, HFunc.eval() - mHPreCollision[i]); + HFunction HFuncOldOld(mLatDat->GetFNew(LatticeType::NUMVECTORS * i), + NULL); + mHPreCollision[i] = HFuncOldOld.eval(); } } @@ -77,68 +65,40 @@ namespace hemelb { if (mCollisionTypesTested[collision_type]) { - for (site_t i = offset; i < offset + mLatDat->GetDomainEdgeCollisionCount(collision_type); i++) + for (site_t i = offset; + i < offset + mLatDat->GetDomainEdgeCollisionCount(collision_type); i++) { - const geometry::Site site = mLatDat->GetSite(i); - - HFunction HFunc(site.GetFOld(), NULL); - dHMax = util::NumericalFunctions::max(dHMax, HFunc.eval() - mHPreCollision[i]); + HFunction HFuncOldOld(mLatDat->GetFNew(LatticeType::NUMVECTORS * i), + NULL); + mHPreCollision[i] = HFuncOldOld.eval(); } } offset += mLatDat->GetDomainEdgeCollisionCount(collision_type); } - - /* - * Ideally dH should never be greater than zero. However, because H is positive definite accuracy as well as - * rounding and truncation errors can make dH greater than zero in certain cases. The tolerance - * is limited by the accuracy of the simulation (including the accuracy to which alpha is calculated) - * The tolerance has to be at least as big as the accuracy to which alpha is calculated - */ - if (dHMax > 1.0E-6) - { - mUpwardsValue = DISOBEYED; - } - } - - /** - * Override the reset method in the base class, to reset the stability variables. - */ - void Reset() - { - // Re-initialise all values to indicate that the H-theorem is obeyed. - mUpwardsValue = OBEYED; - - for (unsigned int ii = 0; ii < SPREADFACTOR; ii++) - { - mChildrensValues[ii] = OBEYED; - } } - - protected: - /** - * Override the methods from the base class to propagate data from the root, and - * to send data about this node and its childrens' stabilities up towards the root. - */ - void ProgressFromChildren(unsigned long splayNumber) + void PreSend() { - ReceiveFromChildren(mChildrensValues, 1); - } - - void ProgressToParent(unsigned long splayNumber) - { - // Store pre-collision values. site_t offset = 0; + double dHMax = 0.0; + + // The order of arguments in max is important + // If distributions go negative HFunc.eval() will return a NaN. Due to the + // nature of NaN and the structure of max, dH will be assigned as NaN if this is the case + // This is what we want, because the EntropyTester will fail otherwise and abort when + // it is simply sufficient to wait until StabilityTester restarts. for (unsigned int collision_type = 0; collision_type < COLLISION_TYPES; collision_type++) { if (mCollisionTypesTested[collision_type]) { - for (site_t i = offset; i < offset + mLatDat->GetMidDomainCollisionCount(collision_type); i++) + for (site_t i = offset; + i < offset + mLatDat->GetMidDomainCollisionCount(collision_type); i++) { const geometry::Site site = mLatDat->GetSite(i); + HFunction HFunc(site.GetFOld(), NULL); - mHPreCollision[i] = HFunc.eval(); + dHMax = util::NumericalFunctions::max(dHMax, HFunc.eval() - mHPreCollision[i]); } } @@ -149,76 +109,73 @@ namespace hemelb { if (mCollisionTypesTested[collision_type]) { - for (site_t i = offset; i < offset + mLatDat->GetDomainEdgeCollisionCount(collision_type); i++) + for (site_t i = offset; + i < offset + mLatDat->GetDomainEdgeCollisionCount(collision_type); i++) { const geometry::Site site = mLatDat->GetSite(i); + HFunction HFunc(site.GetFOld(), NULL); - mHPreCollision[i] = HFunc.eval(); + dHMax = util::NumericalFunctions::max(dHMax, HFunc.eval() - mHPreCollision[i]); } } offset += mLatDat->GetDomainEdgeCollisionCount(collision_type); } - SendToParent(&mUpwardsValue, 1); + /* + * Ideally dH should never be greater than zero. However, because H is positive definite accuracy as well as + * rounding and truncation errors can make dH greater than zero in certain cases. The tolerance + * is limited by the accuracy of the simulation (including the accuracy to which alpha is calculated) + * The tolerance has to be at least as big as the accuracy to which alpha is calculated + */ + if (dHMax > 1.0E-6) + { + localHTheorem = DISOBEYED; + } + } + void Send(void) + { + collectiveReq = collectiveComm->Ireduce(localHTheorem, MPI_MAX, RootRank, globalHTheorem); } - /** * Take the combined stability information (an int, with a value of hemelb::lb::Unstable * if any child node is unstable) and start passing it back down the tree. */ - void TopNodeAction() + void PostReceive() { - if (mUpwardsValue == DISOBEYED) + if (collectiveComm->Rank() == RootRank && globalHTheorem == DISOBEYED) { log::Logger::Log("H Theorem violated."); } } - /** - * Override the method from the base class to use the data from child nodes. + * Override the reset method in the base class, to reset the stability variables. */ - void PostReceiveFromChildren(unsigned long splayNumber) + void Reset() { - // No need to test children's entropy direction if this node already disobeys H-theorem. - if (mUpwardsValue == OBEYED) - { - for (int ii = 0; ii < (int) SPREADFACTOR; ii++) - { - if (mChildrensValues[ii] == DISOBEYED) - { - mUpwardsValue = DISOBEYED; - break; - } - } - } + // Re-initialise all values to indicate that the H-theorem is obeyed. + localHTheorem = OBEYED; + globalHTheorem = OBEYED; } - private: enum HTHEOREM { - OBEYED, - DISOBEYED + OBEYED = 0, + DISOBEYED = 1 }; + static const int RootRank = 0; /** * Slightly arbitrary spread factor for the tree. */ - static const unsigned int SPREADFACTOR = 10; - const geometry::LatticeData * mLatDat; - /** - * Stability value of this node and its children to propagate upwards. - */ - int mUpwardsValue; - /** - * Array for storing the passed-up stability values from child nodes. - */ - int mChildrensValues[SPREADFACTOR]; + int localHTheorem; + int globalHTheorem; bool mCollisionTypesTested[COLLISION_TYPES]; - double* mHPreCollision; + std::vector mHPreCollision; + reporting::Timer& workTimer; }; } diff --git a/Code/lb/IncompressibilityChecker.cc b/Code/lb/IncompressibilityChecker.cc index 290b174d1..bf25381c3 100644 --- a/Code/lb/IncompressibilityChecker.cc +++ b/Code/lb/IncompressibilityChecker.cc @@ -1,17 +1,149 @@ - // This file is part of HemeLB and is Copyright (C) // the HemeLB team and/or their institutions, as detailed in the // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "lb/lattices/D3Q15.h" -#include "lb/IncompressibilityChecker.hpp" +#include "lb/IncompressibilityChecker.h" namespace hemelb { + namespace comm + { + template<> + MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() + { + MPI_Datatype dType = lb::IncompressibilityChecker::GetDensityMpiDatatype(); + HEMELB_MPI_CALL(MPI_Type_commit, (&dType)); + return dType; + } + } + namespace lb { - // Explicit instantiation - template class IncompressibilityChecker > ; + /** + * Create the MPI Datatype for sending DensityEtc structs. + */ + MPI_Datatype IncompressibilityChecker::GetDensityMpiDatatype() + { + HEMELB_MPI_TYPE_BEGIN(dType, ::hemelb::lb::DensityEtc, 3); + HEMELB_MPI_TYPE_ADD_MEMBER(min); + HEMELB_MPI_TYPE_ADD_MEMBER(max); + HEMELB_MPI_TYPE_ADD_MEMBER(maxVel); + HEMELB_MPI_TYPE_END(dType, ::hemelb::lb::DensityEtc); + return dType; + } + + void IncompressibilityChecker::UpdateDensityEtc(const DensityEtc& in, DensityEtc& inout) + { + inout.min = std::min(in.min, inout.min); + inout.max = std::max(in.max, inout.max); + inout.maxVel = std::max(in.maxVel, inout.maxVel); + } + + void IncompressibilityChecker::MpiOpUpdateFunc(void* invec, void* inoutvec, int *len, + MPI_Datatype *datatype) + { + const DensityEtc* iv = static_cast(invec); + DensityEtc* iov = static_cast(inoutvec); + for (int i = 0; i < *len; ++i) + { + UpdateDensityEtc(iv[i], iov[i]); + } + } + + IncompressibilityChecker::IncompressibilityChecker(const geometry::LatticeData * latticeData, + comm::Communicator::ConstPtr comms, + SimulationState* simState, + lb::MacroscopicPropertyCache& propertyCache, + reporting::Timers& timings, + distribn_t maximumRelativeDensityDifferenceAllowed) : + timestep::CollectiveActor(comms, timings[reporting::Timers::mpiWait]), + mLatDat(latticeData), propertyCache(propertyCache), mSimState(simState), + maximumRelativeDensityDifferenceAllowed(maximumRelativeDensityDifferenceAllowed), + workTimer(timings[reporting::Timers::monitoring]) + { + HEMELB_MPI_CALL(MPI_Op_create, (&IncompressibilityChecker::MpiOpUpdateFunc, 1, &reduction)); + localDensity.min = std::numeric_limits::max(); + localDensity.max = -std::numeric_limits::max(); + localDensity.maxVel = 0; + globalDensity = localDensity; + } + + IncompressibilityChecker::~IncompressibilityChecker() + { + HEMELB_MPI_CALL(MPI_Op_free, (&reduction)); + } + + distribn_t IncompressibilityChecker::GetGlobalSmallestDensity() const + { + return globalDensity.min; + } + + distribn_t IncompressibilityChecker::GetGlobalLargestDensity() const + { + return globalDensity.max; + } + + double IncompressibilityChecker::GetMaxRelativeDensityDifference() const + { + distribn_t maxDensityDiff = GetGlobalLargestDensity() - GetGlobalSmallestDensity(); + if (maxDensityDiff < 0.0) + return std::numeric_limits::quiet_NaN(); + + return maxDensityDiff / REFERENCE_DENSITY; + } + + double IncompressibilityChecker::GetMaxRelativeDensityDifferenceAllowed() const + { + return maximumRelativeDensityDifferenceAllowed; + } + + void IncompressibilityChecker::PreSend() + { + if (isCollectiveRunning) + return; + + for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) + { + DensityEtc etc; + etc.min = propertyCache.densityCache.Get(i); + etc.max = propertyCache.densityCache.Get(i); + etc.maxVel = propertyCache.velocityCache.Get(i).GetMagnitude(); + UpdateDensityEtc(etc, localDensity); + } + + } + /** + * Initiate the collective. + */ + void IncompressibilityChecker::Send(void) + { + // Begin collective. + collectiveReq = collectiveComm->Iallreduce(localDensity, reduction, globalDensity); + isCollectiveRunning = true; + } + + bool IncompressibilityChecker::IsDensityDiffWithinRange() const + { + return (GetMaxRelativeDensityDifference() < maximumRelativeDensityDifferenceAllowed); + } + + void IncompressibilityChecker::Report(ctemplate::TemplateDictionary& dictionary) + { + if (!IsDensityDiffWithinRange()) + { + ctemplate::TemplateDictionary *incomp = dictionary.AddSectionDictionary("DENSITIES"); + incomp->SetFormattedValue("ALLOWED", + "%.1f%%", + GetMaxRelativeDensityDifferenceAllowed() * 100); + incomp->SetFormattedValue("ACTUAL", "%.1f%%", GetMaxRelativeDensityDifference() * 100); + } + } + + double IncompressibilityChecker::GetGlobalLargestVelocityMagnitude() const + { + return globalDensity.maxVel; + } + } } diff --git a/Code/lb/IncompressibilityChecker.h b/Code/lb/IncompressibilityChecker.h index 075f0ffca..861290909 100644 --- a/Code/lb/IncompressibilityChecker.h +++ b/Code/lb/IncompressibilityChecker.h @@ -6,12 +6,11 @@ #ifndef HEMELB_LB_INCOMPRESSIBILITYCHECKER_H #define HEMELB_LB_INCOMPRESSIBILITYCHECKER_H - +#include "comm/Communicator.h" +#include "timestep/CollectiveActor.h" #include "geometry/LatticeData.h" #include "lb/MacroscopicPropertyCache.h" -#include "net/PhasedBroadcastRegular.h" #include "reporting/Reportable.h" -#include namespace hemelb { @@ -24,106 +23,25 @@ namespace hemelb */ static const distribn_t REFERENCE_DENSITY = 1.0; - template - class IncompressibilityChecker : public BroadcastPolicy, + struct DensityEtc { + double min; + double max; + double maxVel; + }; + + class IncompressibilityChecker : public timestep::CollectiveActor, public reporting::Reportable { - /** - * This class uses the phased broadcast infrastructure to keep track of the maximum density difference across the domain. - */ - public: - class DensityTracker - { - /** - * This is convenience class that encapsulates fluid magnitudes of interest being tracked across the domain. At the - * moment it tracks maximum and minimum density, it can be extended to accommodate other magnitudes. - */ - public: - /** Number of densities being tracked. Needs to be exposed for send/receive. */ - static const unsigned DENSITY_TRACKER_SIZE = 3; - - /** Identifiers of the densities being tracked. Cardinality must be kept consistent with DENSITY_TRACKER_SIZE */ - typedef enum - { - MIN_DENSITY = 0u, - MAX_DENSITY, - MAX_VELOCITY_MAGNITUDE - } DensityTrackerIndices; - - /** - * Default constructor, initialises the tracker with some large/small min/max densities. - */ - DensityTracker(); - - /** - * Constructor, initialises the tracker with density values provided. - * - * @param densityValues density values used for initialisation - */ - DensityTracker(distribn_t* const densityValues); - - /** - * Destructor - */ - ~DensityTracker(); - - /** - * Updates densities based on the values of another DensityTracker object. - * - * @param newValues density tracker object to copy from - */ - void operator=(const DensityTracker& newValues); - - /** - * Access individually each of the densities tracked. - * - * @param index index of the density of interest (see DensityTrackerIndices enum for a list) - * @return density value - */ - distribn_t& operator[](DensityTrackerIndices densityIndex) const; - - /** - * Returns a pointer to the internal array used to store densities. - * Only to be used for sending/receiving. Use [] operator, instead. - * - * @return pointer to the internal array used to store densities - */ - distribn_t* GetDensitiesArray() const; - - /** - * Updates min/max densities with values in newValue object if they are smaller/larger - * - * @param newValues new values to be considered for an update - */ - void UpdateDensityTracker(const DensityTracker& newValues); - - /** - * Updates min/max densities with value newValue if it is smaller/larger - * - * @param newDensity new density value to be considered for an update - * @param newVelocityMagnitude new velocity magnitude to be considered for an update - */ - void UpdateDensityTracker(distribn_t newDensity, distribn_t newVelocityMagnitude); - - private: - /** Array storing all the densities being tracked */ - distribn_t* densitiesArray; - - /** Keeps track on whether densitiesArray was allocated locally */ - bool allocatedHere; - }; - /** * Constructor * * @param latticeData geometry object - * @param net network interface object + * @param comms communicator object * @param simState simulation state * @param maximumRelativeDensityDifferenceAllowed maximum density difference allowed in the domain (relative to reference density, default 5%) */ - IncompressibilityChecker(const geometry::LatticeData * latticeData, - net::Net* net, + IncompressibilityChecker(const geometry::LatticeData * latticeData, comm::Communicator::ConstPtr comms, SimulationState* simState, lb::MacroscopicPropertyCache& propertyCache, reporting::Timers& timings, @@ -186,47 +104,24 @@ namespace hemelb */ double GetGlobalLargestVelocityMagnitude() const; - protected: - /** - * Override the methods from the base class to propagate data from the root, and - * to send data about this node and its childrens' density trackers up towards the root. - */ - void ProgressFromChildren(unsigned long splayNumber); - void ProgressFromParent(unsigned long splayNumber); - void ProgressToChildren(unsigned long splayNumber); - void ProgressToParent(unsigned long splayNumber); - - /** - * Take the combined density tracker information and start passing it back down the tree. - */ - void TopNodeAction(); - - /** - * Override the method from the base class to use the data from child nodes. - */ - void PostReceiveFromChildren(unsigned long splayNumber); - - /** - * Override the method from the base class for a node in the tree to update its density tracker - * - * @param splayNumber The parameter splayNumber is 0 indexed and less than splay (a template parameter of PhasedBroadcast) - */ - virtual void PostSendToParent(unsigned long splayNumber); + static MPI_Datatype GetDensityMpiDatatype(); - /** - * Use the density tracker sent by the root node. - */ - void Effect(); + protected: + + inline virtual void BeginAll() {} + inline virtual void Begin() {} + // Compute the local state + virtual void PreSend(); + // Initiate the collective + virtual void Send(); + inline virtual void PreWait() {} + inline virtual void End() {} + inline virtual void EndAll() {} private: - - /** - * Slightly arbitrary spread factor for the tree. - * - * @todo #23 This is defined in StabilityChecker as well, refactor somewhere else? - */ - static const unsigned int SPREADFACTOR = 10u; - + static void UpdateDensityEtc(const DensityEtc& in, DensityEtc& inout); + static void MpiOpUpdateFunc(void* invec, void* inoutvec, + int *len, MPI_Datatype *datatype); /** Pointer to lattice data object. */ const geometry::LatticeData * mLatDat; @@ -236,23 +131,20 @@ namespace hemelb /** Pointer to the simulation state used in the rest of the simulation. */ lb::SimulationState* mSimState; - /** Timing object. */ - reporting::Timers& timings; - /** Maximum density difference allowed in the domain (relative to reference density) */ distribn_t maximumRelativeDensityDifferenceAllowed; /** Density tracker with the densities agreed on. */ - DensityTracker* globalDensityTracker; + DensityEtc globalDensity; /** Density tracker of this node and its children to propagate upwards. */ - DensityTracker upwardsDensityTracker; + DensityEtc localDensity; - /** Density tracker as agreed at the root node, to pass downwards. */ - DensityTracker downwardsDensityTracker; + /** Custom operator for reduction/ */ + MPI_Op reduction; - /** Array for storing the passed-up densities from child nodes. */ - distribn_t childrenDensitiesSerialised[SPREADFACTOR * DensityTracker::DENSITY_TRACKER_SIZE]; + /** Time spent checking stuff */ + reporting::Timer& workTimer; }; } diff --git a/Code/lb/IncompressibilityChecker.hpp b/Code/lb/IncompressibilityChecker.hpp deleted file mode 100644 index 92572fd74..000000000 --- a/Code/lb/IncompressibilityChecker.hpp +++ /dev/null @@ -1,266 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_LB_INCOMPRESSIBILITYCHECKER_HPP -#define HEMELB_LB_INCOMPRESSIBILITYCHECKER_HPP - -#include "lb/IncompressibilityChecker.h" - -namespace hemelb -{ - namespace lb - { - template - IncompressibilityChecker::DensityTracker::DensityTracker() : - allocatedHere(true) - { - densitiesArray = new distribn_t[DENSITY_TRACKER_SIZE]; - - //! @todo #23 do we have a policy on floating point constants? - densitiesArray[MIN_DENSITY] = DBL_MAX; - densitiesArray[MAX_DENSITY] = -DBL_MAX; - densitiesArray[MAX_VELOCITY_MAGNITUDE] = 0.0; - } - - template - IncompressibilityChecker::DensityTracker::DensityTracker(distribn_t* const densityValues) : - densitiesArray(densityValues), allocatedHere(false) - { - } - - template - IncompressibilityChecker::DensityTracker::~DensityTracker() - { - if (allocatedHere) - { - delete[] densitiesArray; - } - } - - template - void IncompressibilityChecker::DensityTracker::operator=(const DensityTracker& newValues) - { - for (unsigned trackerEntry = 0; trackerEntry < DENSITY_TRACKER_SIZE; trackerEntry++) - { - densitiesArray[trackerEntry] = newValues.GetDensitiesArray()[trackerEntry]; - } - } - - template - distribn_t& IncompressibilityChecker::DensityTracker::operator[](DensityTrackerIndices densityIndex) const - { - return densitiesArray[densityIndex]; - } - - template - distribn_t* IncompressibilityChecker::DensityTracker::GetDensitiesArray() const - { - return densitiesArray; - } - - template - void IncompressibilityChecker::DensityTracker::UpdateDensityTracker(const DensityTracker& newValues) - { - if (newValues[MIN_DENSITY] < densitiesArray[MIN_DENSITY]) - { - densitiesArray[MIN_DENSITY] = newValues[MIN_DENSITY]; - } - if (newValues[MAX_DENSITY] > densitiesArray[MAX_DENSITY]) - { - densitiesArray[MAX_DENSITY] = newValues[MAX_DENSITY]; - } - if (newValues[MAX_VELOCITY_MAGNITUDE] > densitiesArray[MAX_VELOCITY_MAGNITUDE]) - { - densitiesArray[MAX_VELOCITY_MAGNITUDE] = newValues[MAX_VELOCITY_MAGNITUDE]; - } - } - - template - void IncompressibilityChecker::DensityTracker::UpdateDensityTracker(distribn_t newDensity, - distribn_t newVelocityMagnitude) - { - if (newDensity < densitiesArray[MIN_DENSITY]) - { - densitiesArray[MIN_DENSITY] = newDensity; - } - if (newDensity > densitiesArray[MAX_DENSITY]) - { - densitiesArray[MAX_DENSITY] = newDensity; - } - if (newVelocityMagnitude > densitiesArray[MAX_VELOCITY_MAGNITUDE]) - { - densitiesArray[MAX_VELOCITY_MAGNITUDE] = newVelocityMagnitude; - } - } - - template - IncompressibilityChecker::IncompressibilityChecker(const geometry::LatticeData * latticeData, - net::Net* net, - SimulationState* simState, - lb::MacroscopicPropertyCache& propertyCache, - reporting::Timers& timings, - distribn_t maximumRelativeDensityDifferenceAllowed) : - BroadcastPolicy(net, simState, SPREADFACTOR), mLatDat(latticeData), propertyCache(propertyCache), mSimState(simState), timings(timings), maximumRelativeDensityDifferenceAllowed(maximumRelativeDensityDifferenceAllowed), globalDensityTracker(NULL) - { - /* - * childrenDensitiesSerialised must be initialised to something sensible since ReceiveFromChildren won't - * fill it in completely unless the logarithm base SPREADFACTOR of the number of processes is an integer. - */ - for (unsigned leaf_index = 0; leaf_index < SPREADFACTOR; leaf_index++) - { - unsigned offset = leaf_index * DensityTracker::DENSITY_TRACKER_SIZE; - for (unsigned tracker_index = 0; tracker_index < DensityTracker::DENSITY_TRACKER_SIZE; tracker_index++) - { - switch (tracker_index) - { - case DensityTracker::MIN_DENSITY: - case DensityTracker::MAX_DENSITY: - childrenDensitiesSerialised[offset + tracker_index] = REFERENCE_DENSITY; - break; - case DensityTracker::MAX_VELOCITY_MAGNITUDE: - childrenDensitiesSerialised[offset + tracker_index] = 0.0; - break; - default: - // This should never trip. It only occurs when a new entry is added to the density tracker and - // no suitable initialisation is provided. - assert(false); - } - } - } - - } - - template - IncompressibilityChecker::~IncompressibilityChecker() - { - } - - template - distribn_t IncompressibilityChecker::GetGlobalSmallestDensity() const - { - assert(AreDensitiesAvailable()); - return (*globalDensityTracker)[DensityTracker::MIN_DENSITY]; - } - - template - distribn_t IncompressibilityChecker::GetGlobalLargestDensity() const - { - assert(AreDensitiesAvailable()); - return (*globalDensityTracker)[DensityTracker::MAX_DENSITY]; - } - - template - double IncompressibilityChecker::GetMaxRelativeDensityDifference() const - { - distribn_t maxDensityDiff = GetGlobalLargestDensity() - GetGlobalSmallestDensity(); - assert(maxDensityDiff >= 0.0); - return maxDensityDiff / REFERENCE_DENSITY; - } - - template - double IncompressibilityChecker::GetMaxRelativeDensityDifferenceAllowed() const - { - return maximumRelativeDensityDifferenceAllowed; - } - - template - void IncompressibilityChecker::PostReceiveFromChildren(unsigned long splayNumber) - { - timings[hemelb::reporting::Timers::monitoring].Start(); - - for (int childIndex = 0; childIndex < (int) SPREADFACTOR; childIndex++) - { - DensityTracker childDensities(&childrenDensitiesSerialised[childIndex * DensityTracker::DENSITY_TRACKER_SIZE]); - - upwardsDensityTracker.UpdateDensityTracker(childDensities); - } - - timings[hemelb::reporting::Timers::monitoring].Stop(); - } - - template - void IncompressibilityChecker::PostSendToParent(unsigned long splayNumber) - { - timings[hemelb::reporting::Timers::monitoring].Start(); - - for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) - { - upwardsDensityTracker.UpdateDensityTracker(propertyCache.densityCache.Get(i), - propertyCache.velocityCache.Get(i).GetMagnitude()); - } - - timings[hemelb::reporting::Timers::monitoring].Stop(); - } - - template - void IncompressibilityChecker::ProgressFromChildren(unsigned long splayNumber) - { - this->ReceiveFromChildren(childrenDensitiesSerialised, DensityTracker::DENSITY_TRACKER_SIZE); - } - - template - void IncompressibilityChecker::ProgressFromParent(unsigned long splayNumber) - { - this->ReceiveFromParent(downwardsDensityTracker.GetDensitiesArray(), DensityTracker::DENSITY_TRACKER_SIZE); - } - - template - void IncompressibilityChecker::ProgressToChildren(unsigned long splayNumber) - { - this->SendToChildren(downwardsDensityTracker.GetDensitiesArray(), DensityTracker::DENSITY_TRACKER_SIZE); - } - - template - void IncompressibilityChecker::ProgressToParent(unsigned long splayNumber) - { - this->SendToParent(upwardsDensityTracker.GetDensitiesArray(), DensityTracker::DENSITY_TRACKER_SIZE); - } - - template - void IncompressibilityChecker::TopNodeAction() - { - downwardsDensityTracker = upwardsDensityTracker; - } - - template - void IncompressibilityChecker::Effect() - { - globalDensityTracker = &downwardsDensityTracker; - } - - template - bool IncompressibilityChecker::AreDensitiesAvailable() const - { - return (globalDensityTracker != NULL); - } - - template - bool IncompressibilityChecker::IsDensityDiffWithinRange() const - { - return (GetMaxRelativeDensityDifference() < maximumRelativeDensityDifferenceAllowed); - } - - template - void IncompressibilityChecker::Report(ctemplate::TemplateDictionary& dictionary) - { - if (AreDensitiesAvailable() && !IsDensityDiffWithinRange()) - { - ctemplate::TemplateDictionary *incomp = dictionary.AddSectionDictionary("DENSITIES"); - incomp->SetFormattedValue("ALLOWED", "%.1f%%", GetMaxRelativeDensityDifferenceAllowed() * 100); - incomp->SetFormattedValue("ACTUAL", "%.1f%%", GetMaxRelativeDensityDifference() * 100); - } - } - - template - double IncompressibilityChecker::GetGlobalLargestVelocityMagnitude() const - { - assert(AreDensitiesAvailable()); - return (*globalDensityTracker)[DensityTracker::MAX_VELOCITY_MAGNITUDE]; - } - } -} - -#endif /* HEMELB_LB_INCOMPRESSIBILITYCHECKER_HPP */ diff --git a/Code/lb/SimulationState.cc b/Code/lb/SimulationState.cc index b132e1fe5..984df0f1a 100644 --- a/Code/lb/SimulationState.cc +++ b/Code/lb/SimulationState.cc @@ -15,7 +15,7 @@ namespace hemelb SimulationState::SimulationState(double timeStepLength, unsigned long totalTimeSteps) : timeStepLength(timeStepLength), timeStep(1), totalTimeSteps(totalTimeSteps), - isTerminating(false), isRendering(false), stability(Stable) + isTerminating(false), stability(Stable) { } @@ -33,10 +33,7 @@ namespace hemelb { isTerminating = value; } - void SimulationState::SetIsRendering(bool value) - { - isRendering = value; - } + void SimulationState::SetStability(Stability value) { stability = value; @@ -59,12 +56,12 @@ namespace hemelb bool SimulationState::IsTerminating() const { + if (timeStep >= totalTimeSteps) + return true; + return isTerminating; } - bool SimulationState::IsRendering() const - { - return isRendering; - } + Stability SimulationState::GetStability() const { return stability; diff --git a/Code/lb/SimulationState.h b/Code/lb/SimulationState.h index e50718b2c..0f9ca8696 100644 --- a/Code/lb/SimulationState.h +++ b/Code/lb/SimulationState.h @@ -31,14 +31,12 @@ namespace hemelb void Increment(); void Reset(); void SetIsTerminating(bool value); - void SetIsRendering(bool value); void SetStability(Stability value); LatticeTimeStep GetTimeStep() const; LatticeTimeStep Get0IndexedTimeStep() const; LatticeTimeStep GetTotalTimeSteps() const; bool IsTerminating() const; - bool IsRendering() const; Stability GetStability() const; PhysicalTime GetTime() const {return GetTimeStepLength()*Get0IndexedTimeStep();} @@ -51,7 +49,6 @@ namespace hemelb LatticeTimeStep timeStep; LatticeTimeStep totalTimeSteps; bool isTerminating; - bool isRendering; Stability stability; }; } diff --git a/Code/lb/StabilityTester.h b/Code/lb/StabilityTester.h index a4fec055c..ab560a5bd 100644 --- a/Code/lb/StabilityTester.h +++ b/Code/lb/StabilityTester.h @@ -7,7 +7,7 @@ #ifndef HEMELB_LB_STABILITYTESTER_H #define HEMELB_LB_STABILITYTESTER_H -#include "net/PhasedBroadcastRegular.h" +#include "timestep/CollectiveActor.h" #include "geometry/LatticeData.h" namespace hemelb @@ -15,138 +15,44 @@ namespace hemelb namespace lb { /** - * Class to repeatedly assess the stability of the simulation, using the PhasedBroadcast - * interface. - * - * PhasedBroadcastRegular is used because we know in advance which iterations we will - * want to communicate on. The default parameters suffice: no initial action is necessary - * because we can assess the stability just before communicating (it doesn't have to happen - * at the same time on all nodes), only one communication is needed between depths, which - * can't overlap. We go down the tree to pass the overall stability to all nodes, and we go up - * the tree to compose the local stability for all nodes to discover whether the simulation as - * a whole is stable. + * Class to repeatedly assess the stability of the simulation, using non-blocking collective */ template - class StabilityTester : public net::PhasedBroadcastRegular<> + class StabilityTester : public timestep::CollectiveActor { - public: - StabilityTester(const geometry::LatticeData * iLatDat, net::Net* net, + public: + StabilityTester(const geometry::LatticeData * iLatDat, comm::Communicator::ConstPtr comms, SimulationState* simState, reporting::Timers& timings, - const hemelb::configuration::SimConfig::MonitoringConfig* testerConfig) : - net::PhasedBroadcastRegular<>(net, simState, SPREADFACTOR), mLatDat(iLatDat), - mSimState(simState), timings(timings), testerConfig(testerConfig) - { - Reset(); - } + bool checkForConvergence, double relativeTolerance); + /** * Override the reset method in the base class, to reset the stability variables. */ - void Reset() - { - mUpwardsStability = UndefinedStability; - mDownwardsStability = UndefinedStability; - - mSimState->SetStability(UndefinedStability); - - for (unsigned int ii = 0; ii < SPREADFACTOR; ii++) - { - mChildrensStability[ii] = UndefinedStability; - } - } + void Reset(); protected: - /** - * Override the methods from the base class to propagate data from the root, and - * to send data about this node and its childrens' stabilities up towards the root. - */ - void ProgressFromChildren(unsigned long splayNumber) - { - ReceiveFromChildren(mChildrensStability, 1); - } - - void ProgressFromParent(unsigned long splayNumber) - { - ReceiveFromParent(&mDownwardsStability, 1); - } - - void ProgressToChildren(unsigned long splayNumber) - { - SendToChildren(&mDownwardsStability, 1); - } - - void ProgressToParent(unsigned long splayNumber) - { - SendToParent(&mUpwardsStability, 1); - } - - /** - * The algorithm that checks distribution function convergence must be run in this - * method rather than in ProgressToParent to make sure that the current timestep has - * finished streaming. - * - * @param splayNumber - */ - void PostSendToParent(unsigned long splayNumber) - { - timings[hemelb::reporting::Timers::monitoring].Start(); - - // No need to bother testing out local lattice points if we're going to be - // sending up a 'Unstable' value anyway. - if (mUpwardsStability != Unstable) - { - bool unconvergedSitePresent = false; - - for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) - { - for (unsigned int l = 0; l < LatticeType::NUMVECTORS; l++) - { - distribn_t value = *mLatDat->GetFNew(i * LatticeType::NUMVECTORS + l); - - // Note that by testing for value > 0.0, we also catch stray NaNs. - if (! (value > 0.0)) - { - mUpwardsStability = Unstable; - break; - } - } - ///@todo: If we refactor the previous loop out, we can get away with a single break statement - if (mUpwardsStability == Unstable) - { - break; - } - - if (testerConfig->doConvergenceCheck) - { - distribn_t relativeDifference = - ComputeRelativeDifference(mLatDat->GetFNew(i * LatticeType::NUMVECTORS), - mLatDat->GetSite(i).GetFOld()); - - if (relativeDifference > testerConfig->convergenceRelativeTolerance) - { - // The simulation is stable but hasn't converged in the whole domain yet. - unconvergedSitePresent = true; - } - } - } - - switch (mUpwardsStability) - { - case UndefinedStability: - case Stable: - case StableAndConverged: - mUpwardsStability = (testerConfig->doConvergenceCheck && !unconvergedSitePresent) ? - StableAndConverged : - Stable; - break; - case Unstable: - break; - } - } - - timings[hemelb::reporting::Timers::monitoring].Stop(); - } - + + virtual void BeginAll() {} + + virtual void Begin() {} + + // Compute the local stability/convergence state + virtual void PreSend(); + + // Initiate the collective + virtual void Send(); + + virtual void PreWait() {} + + // Wait is defined in CollectiveActor + // virtual void Wait(); + + // Apply the stability value sent by the root node to the simulation logic. + virtual void End(); + + virtual void EndAll() {} + /** * Computes the relative difference between the densities at the beginning and end of a * timestep, i.e. |(rho_new - rho_old) / (rho_old - rho_0)|. @@ -155,160 +61,34 @@ namespace hemelb * @param fOld Distribution function at the end of the previous timestep. * @return relative difference between the densities computed from fNew and fOld. */ - inline double ComputeRelativeDifference(const distribn_t* fNew, - const distribn_t* fOld) const - { - distribn_t newDensity; - distribn_t newMomentumX; - distribn_t newMomentumY; - distribn_t newMomentumZ; - LatticeType::CalculateDensityAndMomentum(fNew, - newDensity, - newMomentumX, - newMomentumY, - newMomentumZ); - - distribn_t oldDensity; - distribn_t oldMomentumX; - distribn_t oldMomentumY; - distribn_t oldMomentumZ; - LatticeType::CalculateDensityAndMomentum(fOld, - oldDensity, - oldMomentumX, - oldMomentumY, - oldMomentumZ); - - distribn_t absoluteError; - distribn_t referenceValue; - - switch (testerConfig->convergenceVariable) - { - case extraction::OutputField::Velocity: - { - distribn_t diff_vel_x = newMomentumX / newDensity - oldMomentumX / oldDensity; - distribn_t diff_vel_y = newMomentumY / newDensity - oldMomentumY / oldDensity; - distribn_t diff_vel_z = newMomentumZ / newDensity - oldMomentumZ / oldDensity; - - absoluteError = sqrt(diff_vel_x * diff_vel_x + diff_vel_y * diff_vel_y - + diff_vel_z * diff_vel_z); - referenceValue = testerConfig->convergenceReferenceValue; - break; - } - default: - // Never reached - throw Exception() - << "Convergence check based on requested variable currently not available"; - } - - return absoluteError / referenceValue; - } - - /** - * Take the combined stability information (an int, with a value of hemelb::lb::Unstable - * if any child node is unstable) and start passing it back down the tree. - */ - void TopNodeAction() - { - mDownwardsStability = mUpwardsStability; - } - - /** - * Override the method from the base class to use the data from child nodes. - */ - void PostReceiveFromChildren(unsigned long splayNumber) - { - timings[hemelb::reporting::Timers::monitoring].Start(); - - // No need to test children's stability if this node is already unstable. - if (mUpwardsStability != Unstable) - { - for (int ii = 0; ii < (int) SPREADFACTOR; ii++) - { - if (mChildrensStability[ii] == Unstable) - { - mUpwardsStability = Unstable; - break; - } - } - - // If the simulation wasn't found to be unstable and we need to check for convergence, do it now. - if ( (mUpwardsStability != Unstable) && testerConfig->doConvergenceCheck) - { - bool anyStableNotConverged = false; - bool anyConverged = false; - - // mChildrensStability will contain UndefinedStability for non-existent children - for (int ii = 0; ii < (int) SPREADFACTOR; ii++) - { - if (mChildrensStability[ii] == StableAndConverged) - { - anyConverged = true; - } - - if (mChildrensStability[ii] == Stable) - { - anyStableNotConverged = true; - } - } - - // With the current configuration the root node of the tree won't own any fluid sites. Its - // state only depends on children nodes not on local state. - if (anyConverged - && (mUpwardsStability == StableAndConverged || GetParent() == NOPARENT)) - { - mUpwardsStability = StableAndConverged; - } - - if (anyStableNotConverged) - { - mUpwardsStability = Stable; - } - } - } - - timings[hemelb::reporting::Timers::monitoring].Stop(); - } - - /** - * Apply the stability value sent by the root node to the simulation logic. - */ - void Effect() - { - mSimState->SetStability((Stability) mDownwardsStability); - } + double ComputeRelativeDifference(const distribn_t* fNew, + const distribn_t* fOld) const; private: - /** - * Slightly arbitrary spread factor for the tree. - */ - static const unsigned int SPREADFACTOR = 10; - const geometry::LatticeData * mLatDat; + const geometry::LatticeData* mLatDat; /** - * Stability value of this node and its children to propagate upwards. + * Local and global stability. */ - int mUpwardsStability; - /** - * Stability value as understood by the root node, to pass downwards. - */ - int mDownwardsStability; - /** - * Array for storing the passed-up stability values from child nodes. - */ - int mChildrensStability[SPREADFACTOR]; + int localStability; + int globalStability; + /** * Pointer to the simulation state used in the rest of the simulation. */ lb::SimulationState* mSimState; - /** Timing object. */ - reporting::Timers& timings; + /** Whether to check for steady flow simulation convergence */ + bool checkForConvergence; + + /** Relative error tolerance in convergence check */ + double relativeTolerance; - /** Object containing the user-provided configuration for this class */ - const hemelb::configuration::SimConfig::MonitoringConfig* testerConfig; + reporting::Timer& workTimer; }; } } +#include "lb/StabilityTester.hpp" #endif /* HEMELB_LB_STABILITYTESTER_H */ diff --git a/Code/lb/StabilityTester.hpp b/Code/lb/StabilityTester.hpp new file mode 100644 index 000000000..99c7f4f5e --- /dev/null +++ b/Code/lb/StabilityTester.hpp @@ -0,0 +1,163 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#ifndef HEMELB_LB_STABILITYTESTER_HPP +#define HEMELB_LB_STABILITYTESTER_HPP + +namespace hemelb +{ + namespace lb + { + /** + * Class to repeatedly assess the stability of the simulation, using non-blocking collective + */ + template + StabilityTester::StabilityTester(const geometry::LatticeData * iLatDat, + comm::Communicator::ConstPtr comms, + SimulationState* simState, + reporting::Timers& timings, + bool checkForConvergence, + double relativeTolerance) : + CollectiveActor(comms, timings[reporting::Timers::monitoring]), + mLatDat(iLatDat), mSimState(simState), checkForConvergence(checkForConvergence), + relativeTolerance(relativeTolerance), workTimer(timings[reporting::Timers::monitoring]) + { + Reset(); + } + + /** + * Override the reset method in the base class, to reset the stability variables. + */ + template + void StabilityTester::Reset() + { + localStability = UndefinedStability; + globalStability = UndefinedStability; + mSimState->SetStability(UndefinedStability); + } + + /** + * Compute the local stability/convergence state. + */ + template + void StabilityTester::PreSend(void) + { + if (isCollectiveRunning) + return; + + workTimer.Start(); + bool unconvergedSitePresent = false; + bool checkConvThisTimeStep = checkForConvergence; + localStability = Stable; + + for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) + { + for (unsigned int l = 0; l < LatticeType::NUMVECTORS; l++) + { + distribn_t value = *mLatDat->GetFNew(i * LatticeType::NUMVECTORS + l); + + // Note that by testing for value > 0.0, we also catch stray NaNs. + if (! (value > 0.0)) + { + localStability = Unstable; + break; + } + } + ///@todo: If we refactor the previous loop out, we can get away with a single break statement + if (localStability == Unstable) + break; + + if (checkConvThisTimeStep) + { + distribn_t relativeDifference = ComputeRelativeDifference(mLatDat->GetFNew(i + * LatticeType::NUMVECTORS), + mLatDat->GetSite(i).GetFOld< + LatticeType>()); + + if (relativeDifference > relativeTolerance) + { + // The simulation is stable but hasn't converged in the whole domain yet. + unconvergedSitePresent = true; + // Skip further tests for this timestep + checkConvThisTimeStep = false; + } + } + } + + if (localStability == Stable) + { + if (checkForConvergence && !unconvergedSitePresent) + { + localStability = StableAndConverged; + } + } + workTimer.Stop(); + } + + /** + * Initiate the collective. + */ + template + void StabilityTester::Send(void) + { + if (isCollectiveRunning) + return; + + // Begin collective. + collectiveReq = collectiveComm->Iallreduce(localStability, MPI_MIN, globalStability); + isCollectiveRunning = true; + } + + /** + * Computes the relative difference between the densities at the beginning and end of a + * timestep, i.e. |(rho_new - rho_old) / (rho_old - rho_0)|. + * + * @param fNew Distribution function after stream and collide, i.e. solution of the current timestep. + * @param fOld Distribution function at the end of the previous timestep. + * @return relative difference between the densities computed from fNew and fOld. + */ + template + double StabilityTester::ComputeRelativeDifference(const distribn_t* fNew, + const distribn_t* fOld) const + { + distribn_t new_density = 0.; + distribn_t old_density = 0.; + for (unsigned int l = 0; l < LatticeType::NUMVECTORS; l++) + { + new_density += fNew[l]; + old_density += fOld[l]; + } + + // This is equivalent to REFERENCE_PRESSURE_mmHg in lattice units + distribn_t ref_density = 1.0; + + if (old_density == ref_density) + { + // We want to avoid returning inf if the site is at pressure = REFERENCE_PRESSURE_mmHg + return 0.0; + } + + return fabs( (new_density - old_density) / (old_density - ref_density)); + } + + /** + * Apply the stability value sent by the root node to the simulation logic. + */ + template + void StabilityTester::End() + { + if (isCollectiveRunning) + return; + + mSimState->SetStability((Stability) globalStability); + } + } +} + +#endif /* HEMELB_LB_STABILITYTESTER_HPP */ diff --git a/Code/lb/collisions/Collisions.h b/Code/lb/collisions/Collisions.h index 5faf994ea..5d6208680 100644 --- a/Code/lb/collisions/Collisions.h +++ b/Code/lb/collisions/Collisions.h @@ -7,9 +7,6 @@ #ifndef HEMELB_LB_COLLISIONS_COLLISIONS_H #define HEMELB_LB_COLLISIONS_COLLISIONS_H -#include "lb/collisions/NonZeroVelocityEquilibriumFixedDensity.h" #include "lb/collisions/Normal.h" -#include "lb/collisions/ZeroVelocityEquilibrium.h" -#include "lb/collisions/ZeroVelocityEquilibriumFixedDensity.h" #endif /* HEMELB_LB_COLLISIONS_COLLISIONS_H */ diff --git a/Code/lb/collisions/NonZeroVelocityEquilibriumFixedDensity.h b/Code/lb/collisions/NonZeroVelocityEquilibriumFixedDensity.h deleted file mode 100644 index 98ad6415d..000000000 --- a/Code/lb/collisions/NonZeroVelocityEquilibriumFixedDensity.h +++ /dev/null @@ -1,68 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_LB_COLLISIONS_NONZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H -#define HEMELB_LB_COLLISIONS_NONZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H - -#include "lb/collisions/BaseCollision.h" -#include "lb/kernels/BaseKernel.h" - -namespace hemelb -{ - namespace lb - { - namespace collisions - { - template - class NonZeroVelocityEquilibriumFixedDensity : public BaseCollision< - NonZeroVelocityEquilibriumFixedDensity, KernelType> - { - public: - typedef KernelType CKernel; - - NonZeroVelocityEquilibriumFixedDensity(kernels::InitParams& initParams) : - kernel(initParams), boundaryObject(initParams.boundaryObject) - { - - } - - inline void DoCalculatePreCollision(kernels::HydroVars& hydroVars, - const geometry::Site& site) - { - CKernel::LatticeType::CalculateDensityAndMomentum(hydroVars.f, - hydroVars.density, - hydroVars.momentum.x, - hydroVars.momentum.y, - hydroVars.momentum.z); - - // Externally impose a density. Keep a record of the old one so we can scale the - // momentum vector. - distribn_t previousDensity = hydroVars.density; - hydroVars.density = boundaryObject->GetBoundaryDensity(site.GetIoletId()); - - hydroVars.momentum *= (hydroVars.density / previousDensity); - - kernel.CalculateFeq(hydroVars, site.GetIndex()); - } - - inline void DoCollide(const LbmParameters* lbmParams, kernels::HydroVars& iHydroVars) - { - for (Direction direction = 0; direction < CKernel::LatticeType::NUMVECTORS; ++direction) - { - iHydroVars.SetFPostCollision(direction, iHydroVars.GetFEq()[direction]); - } - } - - private: - KernelType kernel; - iolets::BoundaryValues* boundaryObject; - }; - - } - } -} - -#endif /* HEMELB_LB_COLLISIONS_NONZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H */ diff --git a/Code/lb/collisions/ZeroVelocityEquilibrium.h b/Code/lb/collisions/ZeroVelocityEquilibrium.h deleted file mode 100644 index 0ddceb830..000000000 --- a/Code/lb/collisions/ZeroVelocityEquilibrium.h +++ /dev/null @@ -1,67 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUM_H -#define HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUM_H - -#include "lb/collisions/BaseCollision.h" -#include "lb/kernels/BaseKernel.h" - -namespace hemelb -{ - namespace lb - { - namespace collisions - { - /** - * Collision operator that maintains the density, set velocity to 0 and fully relaxes - * to equilibrium. - */ - template - class ZeroVelocityEquilibrium : public BaseCollision, KernelType> - { - public: - typedef KernelType CKernel; - - ZeroVelocityEquilibrium(kernels::InitParams& initParams) : - BaseCollision, KernelType>(), kernel(initParams) - { - - } - - inline void DoCalculatePreCollision(kernels::HydroVars& hydroVars, - const geometry::Site& site) - { - hydroVars.density = 0.0; - - for (unsigned int ii = 0; ii < CKernel::LatticeType::NUMVECTORS; ++ii) - { - hydroVars.density += hydroVars.f[ii]; - } - - hydroVars.momentum = util::Vector3D::Zero(); - - kernel.CalculateFeq(hydroVars, site.GetIndex()); - } - - inline void DoCollide(const LbmParameters* lbmParams, kernels::HydroVars& iHydroVars) - { - for (Direction direction = 0; direction < CKernel::LatticeType::NUMVECTORS; ++direction) - { - iHydroVars.SetFPostCollision(direction, iHydroVars.GetFEq()[direction]); - } - } - - private: - KernelType kernel; - - }; - - } - } -} - -#endif /* HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUM_H */ diff --git a/Code/lb/collisions/ZeroVelocityEquilibriumFixedDensity.h b/Code/lb/collisions/ZeroVelocityEquilibriumFixedDensity.h deleted file mode 100644 index f6982dd29..000000000 --- a/Code/lb/collisions/ZeroVelocityEquilibriumFixedDensity.h +++ /dev/null @@ -1,63 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H -#define HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H - -#include "lb/collisions/BaseCollision.h" -#include "lb/kernels/BaseKernel.h" - -namespace hemelb -{ - namespace lb - { - namespace collisions - { - /** - * Collision operator that uses an externally imposed density, zero velocity and relaxes - * fully to equilibrium. - */ - template - class ZeroVelocityEquilibriumFixedDensity : public BaseCollision, - KernelType> - { - public: - typedef KernelType CKernel; - - ZeroVelocityEquilibriumFixedDensity(kernels::InitParams& initParams) : - BaseCollision, KernelType>(), kernel(initParams), boundaryObject(initParams.boundaryObject) - { - } - - inline void DoCalculatePreCollision(kernels::HydroVars& hydroVars, - const geometry::Site& site) - { - hydroVars.density = boundaryObject->GetBoundaryDensity(site.GetIoletId()); - - hydroVars.momentum = util::Vector3D::Zero(); - - kernel.CalculateFeq(hydroVars, site.GetIndex()); - } - - inline void DoCollide(const LbmParameters* lbmParams, kernels::HydroVars& iHydroVars) - { - for (Direction direction = 0; direction < CKernel::LatticeType::NUMVECTORS; ++direction) - { - iHydroVars.SetFPostCollision(direction, iHydroVars.GetFEq()[direction]); - } - } - - private: - KernelType kernel; - iolets::BoundaryValues* boundaryObject; - - }; - - } - } -} - -#endif /* HEMELB_LB_COLLISIONS_ZEROVELOCITYEQUILIBRIUMFIXEDDENSITY_H */ diff --git a/Code/lb/iolets/BoundaryComms.cc b/Code/lb/iolets/BoundaryComms.cc index 99826a75f..decb6441f 100644 --- a/Code/lb/iolets/BoundaryComms.cc +++ b/Code/lb/iolets/BoundaryComms.cc @@ -6,7 +6,7 @@ #include "lb/iolets/BoundaryComms.h" #include "lb/iolets/BoundaryValues.h" -#include "net/IOCommunicator.h" +#include "comm/MpiCommunicator.h" #include "util/utilityFunctions.h" namespace hemelb @@ -46,6 +46,8 @@ namespace hemelb void BoundaryComms::Wait() { + // TODO: refactor the raw MPI calls into the wrapped stuff + if (hasBoundary) { HEMELB_MPI_CALL( @@ -56,6 +58,8 @@ namespace hemelb void BoundaryComms::WaitAllComms() { + // TODO: refactor the raw MPI calls into the wrapped stuff + // Now wait for all to complete if (bcComm.IsCurrentProcTheBCProc()) { @@ -80,16 +84,19 @@ namespace hemelb // It is up to the caller to make sure only BCproc calls send void BoundaryComms::Send(distribn_t* density) { + // TODO: refactor the raw MPI calls into the wrapped stuff + auto mpi = std::dynamic_pointer_cast(bcComm.GetComm()); + MPI_Comm mpi_comm = *mpi; for (int proc = 0; proc < nProcs; proc++) { HEMELB_MPI_CALL( MPI_Isend, ( density, 1, - net::MpiDataType(*density), + comm::MpiDataType(*density), procsList[proc], 100, - bcComm, + mpi_comm, &sendRequest[proc] )); } @@ -97,16 +104,19 @@ namespace hemelb void BoundaryComms::Receive(distribn_t* density) { + // TODO: refactor the raw MPI calls into the wrapped stuff + auto mpi = std::dynamic_pointer_cast(bcComm.GetComm()); + MPI_Comm mpi_comm = *mpi; if (hasBoundary) { HEMELB_MPI_CALL( MPI_Irecv, ( density, 1, - net::MpiDataType(*density), + comm::MpiDataType(*density), bcComm.GetBCProcRank(), 100, - bcComm, + mpi_comm, &receiveRequest )); } @@ -114,6 +124,8 @@ namespace hemelb void BoundaryComms::FinishSend() { + // TODO: refactor the raw MPI calls into the wrapped stuff + // Don't move on to next step with BC proc until all messages have been sent // Precautionary measure to make sure proc doesn't overwrite, before message is sent if (bcComm.IsCurrentProcTheBCProc()) diff --git a/Code/lb/iolets/BoundaryCommunicator.cc b/Code/lb/iolets/BoundaryCommunicator.cc index 18cedaf24..1aefa207e 100644 --- a/Code/lb/iolets/BoundaryCommunicator.cc +++ b/Code/lb/iolets/BoundaryCommunicator.cc @@ -13,14 +13,19 @@ namespace hemelb namespace iolets { - BoundaryCommunicator::BoundaryCommunicator(const net::MpiCommunicator& parent) : - MpiCommunicator(parent.Duplicate()) + BoundaryCommunicator::BoundaryCommunicator(comm::Communicator::ConstPtr parent) : + comm(parent->Duplicate()) { } + comm::Communicator::ConstPtr BoundaryCommunicator::GetComm() const + { + return comm; + } + bool BoundaryCommunicator::IsCurrentProcTheBCProc() const { - return Rank() == GetBCProcRank(); + return comm->Rank() == GetBCProcRank(); } int BoundaryCommunicator::GetBCProcRank() const { diff --git a/Code/lb/iolets/BoundaryCommunicator.h b/Code/lb/iolets/BoundaryCommunicator.h index d0f596ab2..63635d499 100644 --- a/Code/lb/iolets/BoundaryCommunicator.h +++ b/Code/lb/iolets/BoundaryCommunicator.h @@ -7,7 +7,7 @@ #ifndef HEMELB_LB_IOLETS_BOUNDARYCOMMUNICATOR_H #define HEMELB_LB_IOLETS_BOUNDARYCOMMUNICATOR_H -#include "net/MpiCommunicator.h" +#include "comm/Communicator.h" namespace hemelb { @@ -15,12 +15,16 @@ namespace hemelb { namespace iolets { - class BoundaryCommunicator : public net::MpiCommunicator + class BoundaryCommunicator { - public: - BoundaryCommunicator(const net::MpiCommunicator& parent); - bool IsCurrentProcTheBCProc() const; - int GetBCProcRank() const; + public: + BoundaryCommunicator(comm::Communicator::ConstPtr parent); + bool IsCurrentProcTheBCProc() const; + int GetBCProcRank() const; + comm::Communicator::ConstPtr GetComm() const; + private: + comm::Communicator::Ptr comm; + }; } } diff --git a/Code/lb/iolets/BoundaryValues.cc b/Code/lb/iolets/BoundaryValues.cc index 9887d26b8..c2b8b7c40 100644 --- a/Code/lb/iolets/BoundaryValues.cc +++ b/Code/lb/iolets/BoundaryValues.cc @@ -5,7 +5,6 @@ // license in the file LICENSE. #include "lb/iolets/BoundaryValues.h" -#include "lb/iolets/BoundaryComms.h" #include "util/utilityFunctions.h" #include "util/fileutils.h" #include @@ -21,13 +20,12 @@ namespace hemelb geometry::LatticeData* latticeData, const std::vector &incoming_iolets, SimulationState* simulationState, - const net::MpiCommunicator& comms, + comm::Async::Ptr commQ, const util::UnitConverter& units) : - net::IteratedAction(), ioletType(ioletType), totalIoletCount(incoming_iolets.size()), localIoletCount(0), - state(simulationState), unitConverter(units), bcComms(comms) + ioletType(ioletType), totalIoletCount(incoming_iolets.size()), localIoletCount(0), + procsForIolet(incoming_iolets.size()), + state(simulationState), unitConverter(units), asyncCommsQ(commQ) { - std::vector *procsList = new std::vector[totalIoletCount]; - // Determine which iolets need comms and create them for (int ioletIndex = 0; ioletIndex < totalIoletCount; ioletIndex++) { @@ -40,28 +38,20 @@ namespace hemelb bool isIOletOnThisProc = IsIOletOnThisProc(ioletType, latticeData, ioletIndex); hemelb::log::Logger::Log("BOUNDARYVALUES.CC - isioletonthisproc? : %d", isIOletOnThisProc); - procsList[ioletIndex] = GatherProcList(isIOletOnThisProc); + procsForIolet[ioletIndex] = GatherProcList(isIOletOnThisProc); // With information on whether a proc has an IOlet and the list of procs for each IOlte // on the BC task we can create the comms - if (isIOletOnThisProc || bcComms.IsCurrentProcTheBCProc()) + if (isIOletOnThisProc || IsCurrentProcTheBCProc()) { localIoletCount++; localIoletIDs.push_back(ioletIndex); -// hemelb::log::Logger::Log("BOUNDARYVALUES.H - ioletIndex: %d", ioletIndex); - -// if (iolet->IsCommsRequired()) //DEREK: POTENTIAL MULTISCALE ISSUE (this if-statement) -// { - iolet->SetComms(new BoundaryComms(state, procsList[ioletIndex], bcComms, isIOletOnThisProc)); -// } } } // Send out initial values Reset(); - // Clear up - delete[] procsList; hemelb::log::Logger::Log("BOUNDARYVALUES.H - ioletCount: %d, first iolet ID %d", localIoletCount, localIoletIDs[0]); @@ -97,18 +87,20 @@ namespace hemelb { std::vector processorsNeedingIoletList(0); - // This is where the info about whether a proc contains the given inlet/outlet is sent - // If it does contain the given inlet/outlet it sends a true value, else it sends a false. - int isIOletOnThisProc = hasBoundary; // true if inlet i is on this proc + // This is where the info about whether a proc contains the + // given inlet/outlet is sent If it does contain the given + // inlet/outlet it sends a true value, else it sends a false. + // Ideally bools, but MPI doesn't support them :( + char isIOletOnThisProc = hasBoundary; // true if inlet i is on this proc - // These should be bool, but MPI only supports MPI_INT - // For each inlet/outlet there is an array of length equal to total number of procs. - // Each stores true/false value. True if proc of rank equal to the index contains - // the given inlet/outlet. + // For each inlet/outlet there is an array of length equal to + // total number of procs. Each stores true/false value. True + // if proc of rank equal to the index contains the given + // inlet/outlet. + + auto processorsNeedingIoletFlags = asyncCommsQ->GetComm()->Gather(isIOletOnThisProc, GetBCProcRank()); - std::vector processorsNeedingIoletFlags = bcComms.Gather(isIOletOnThisProc, bcComms.GetBCProcRank()); - - if (bcComms.IsCurrentProcTheBCProc()) + if (IsCurrentProcTheBCProc()) { // Now we have an array for each IOlet with true (1) at indices corresponding to // processes that are members of that group. We have to convert this into arrays @@ -122,60 +114,81 @@ namespace hemelb } } - return processorsNeedingIoletList; // return by copy + return processorsNeedingIoletList; // return by move } - void BoundaryValues::RequestComms() + void BoundaryValues::Begin() { - for (int i = 0; i < localIoletCount; i++) + for (int i = 0; i < localIoletCount; i++) { - HandleComms(GetLocalIolet(i)); + auto iolet = GetLocalIolet(i); + if (iolet->IsCommsRequired()) + { + iolet->Begin(this); + } } } - - void BoundaryValues::HandleComms(iolets::InOutLet* iolet) + void BoundaryValues::Receive() { - - if (iolet->IsCommsRequired()) + for (int i = 0; i < localIoletCount; i++) { - iolet->DoComms(bcComms, state->GetTimeStep()); + auto iolet = GetLocalIolet(i); + if (iolet->IsCommsRequired()) + { + iolet->Receive(this, asyncCommsQ); + } } - } - - void BoundaryValues::EndIteration() + void BoundaryValues::Send() { - for (int i = 0; i < localIoletCount; i++) + for (int i = 0; i < localIoletCount; i++) { - if (GetLocalIolet(i)->IsCommsRequired()) - { - GetLocalIolet(i)->GetComms()->FinishSend(); - } + auto iolet = GetLocalIolet(i); + if (iolet->IsCommsRequired()) + { + iolet->Send(this, asyncCommsQ); + } } } - - void BoundaryValues::FinishReceive() + // Wait done by AsyncActor + void BoundaryValues::End() { - for (int i = 0; i < localIoletCount; i++) + for (int i = 0; i < localIoletCount; i++) { - if (GetLocalIolet(i)->IsCommsRequired()) - { - GetLocalIolet(i)->GetComms()->Wait(); - } + auto iolet = GetLocalIolet(i); + if (iolet->IsCommsRequired()) + { + iolet->CommsComplete(this); + } } } + + void BoundaryValues::ForceCommunication() + { + Begin(); + { + auto tmpQ = comm::Async::New(asyncCommsQ->GetComm()); + for (int i = 0; i < localIoletCount; i++) + { + auto iolet = GetLocalIolet(i); + if (iolet->IsCommsRequired()) + { + iolet->Receive(this, tmpQ); + iolet->Send(this, tmpQ); + } + } + } + End(); + } void BoundaryValues::Reset() { for (int i = 0; i < localIoletCount; i++) { GetLocalIolet(i)->Reset(*state); - if (GetLocalIolet(i)->IsCommsRequired()) - { - GetLocalIolet(i)->GetComms()->WaitAllComms(); - - } } + // TODO: Are we sure we need to force a comms cycle? + ForceCommunication(); } // This assumes the program has already waited for comms to finish before @@ -184,16 +197,6 @@ namespace hemelb return iolets[index]->GetDensity(state->Get0IndexedTimeStep()); } - LatticeDensity BoundaryValues::GetDensityMin(int iBoundaryId) - { - return iolets[iBoundaryId]->GetDensityMin(); - } - - LatticeDensity BoundaryValues::GetDensityMax(int iBoundaryId) - { - return iolets[iBoundaryId]->GetDensityMax(); - } - } } } diff --git a/Code/lb/iolets/BoundaryValues.h b/Code/lb/iolets/BoundaryValues.h index b6639c4a3..cd4f2cc9e 100644 --- a/Code/lb/iolets/BoundaryValues.h +++ b/Code/lb/iolets/BoundaryValues.h @@ -7,11 +7,10 @@ #ifndef HEMELB_LB_IOLETS_BOUNDARYVALUES_H #define HEMELB_LB_IOLETS_BOUNDARYVALUES_H -#include "net/IOCommunicator.h" -#include "net/IteratedAction.h" +#include "comm/Communicator.h" +#include "timestep/Actor.h" #include "lb/iolets/InOutLet.h" #include "geometry/LatticeData.h" -#include "lb/iolets/BoundaryCommunicator.h" namespace hemelb { @@ -20,30 +19,47 @@ namespace hemelb namespace iolets { - class BoundaryValues : public net::IteratedAction + class BoundaryValues : public timestep::Actor { public: BoundaryValues(geometry::SiteType ioletType, geometry::LatticeData* latticeData, const std::vector &iolets, SimulationState* simulationState, - const net::MpiCommunicator& comms, + comm::Async::Ptr commQ, const util::UnitConverter& units); ~BoundaryValues(); - void RequestComms(); - void EndIteration(); + // timestep::Actor interface + virtual void BeginAll() { /* not needed */ } + virtual void Begin(); + virtual void Receive(); + virtual void PreSend() { /* not needed */ } + virtual void Send(); + virtual void PreWait() { /* not needed */ } + virtual void Wait() { /* not needed */ } + virtual void End(); + virtual void EndAll() { /* not needed */ } + + + void ForceCommunication(); + // void RequestComms(); + // void EndIteration(); void Reset(); - void FinishReceive(); + // void FinishReceive(); LatticeDensity GetBoundaryDensity(const int index); - LatticeDensity GetDensityMin(int boundaryId); - LatticeDensity GetDensityMax(int boundaryId); + static constexpr int GetBCProcRank() { + return 0; + } - static proc_t GetBCProcRank(); - iolets::InOutLet* GetLocalIolet(unsigned int index) + inline bool IsCurrentProcTheBCProc() { + return asyncCommsQ->GetComm()->Rank() == GetBCProcRank(); + } + + iolets::InOutLet* GetLocalIolet(unsigned int index) { return iolets[localIoletIDs[index]]; } @@ -60,10 +76,20 @@ namespace hemelb return ioletType; } + inline const std::vector& GetProcsForIolet(int i) const { + return procsForIolet[i]; + } + inline const std::vector& GetProcsForIolet(const InOutLet* io) const { + auto iter = std::find(iolets.begin(), iolets.end(), io); + if (iter == iolets.end()) + throw Exception() << "Can't find iolet in BoundaryValues"; + return GetProcsForIolet(iter - iolets.begin()); + } + private: bool IsIOletOnThisProc(geometry::SiteType ioletType, geometry::LatticeData* latticeData, int boundaryId); std::vector GatherProcList(bool hasBoundary); - void HandleComms(iolets::InOutLet* iolet); + // void HandleComms(iolets::InOutLet* iolet); geometry::SiteType ioletType; int totalIoletCount; // Number of IOlets and vector of their indices for communication purposes @@ -71,10 +97,11 @@ namespace hemelb std::vector localIoletIDs; // Has to be a vector of pointers for InOutLet polymorphism std::vector iolets; - + // For each iolet, lists the procs that have it (only populated on BCProc rank) + std::vector> procsForIolet; SimulationState* state; const util::UnitConverter& unitConverter; - BoundaryCommunicator bcComms; + comm::Async::Ptr asyncCommsQ; } ; } diff --git a/Code/lb/iolets/InOutLet.cc b/Code/lb/iolets/InOutLet.cc index 6add96f56..b209438d8 100644 --- a/Code/lb/iolets/InOutLet.cc +++ b/Code/lb/iolets/InOutLet.cc @@ -11,11 +11,6 @@ namespace hemelb { namespace iolets { - void InOutLet::DoComms(const BoundaryCommunicator& bcComms, const LatticeTimeStep timeStep) - { - // pass - } - namespace { unsigned SmallestMagnitudeComponent(const LatticeVector r) diff --git a/Code/lb/iolets/InOutLet.h b/Code/lb/iolets/InOutLet.h index ab29da191..01b4a1af6 100644 --- a/Code/lb/iolets/InOutLet.h +++ b/Code/lb/iolets/InOutLet.h @@ -10,6 +10,7 @@ #include "util/Vector3D.h" #include "util/UnitConverter.h" #include "lb/SimulationState.h" +#include "comm/Async.h" namespace hemelb { @@ -17,11 +18,8 @@ namespace hemelb { namespace iolets { - - //forward declare boundary comms class - class BoundaryComms; - class BoundaryCommunicator; - + class BoundaryValues; + /** * Base class for extra data needed by LB BC implementations. * Makes "Iolet coordinates" available. @@ -56,8 +54,7 @@ namespace hemelb class InOutLet { public: - InOutLet() : - comms(NULL), extraData(NULL) + InOutLet() : extraData(NULL) { } virtual ~InOutLet() @@ -80,6 +77,15 @@ namespace hemelb return false; } + virtual void Begin(BoundaryValues*) { + } + virtual void Receive(BoundaryValues*, comm::Async::Ptr) { + } + virtual void Send(BoundaryValues* , comm::Async::Ptr) { + } + virtual void CommsComplete(BoundaryValues*) { + } + /*** * This is a castable? virtual method, which is perhaps an anti-pattern * We should potentially use dynamic cast checks instead. @@ -89,20 +95,7 @@ namespace hemelb { return false; } - void SetComms(BoundaryComms * boundaryComms) - { - comms = boundaryComms; - } - BoundaryComms * GetComms() const - { - return comms; - } - /*** - * Carry out communication necessary - * @param isIoProcess Is the process the master process? - */ - virtual void DoComms(const BoundaryCommunicator& bcComms, const LatticeTimeStep timeStep); - + /*** * Set up the Iolet. * @param units a UnitConverter instance. @@ -111,36 +104,6 @@ namespace hemelb { } - /*** - * Get the minimum density, in lattice units - * @return minimum density, in lattice units - */ - virtual LatticeDensity GetDensityMin() const = 0; - - /*** - * Get the maximum density, in lattice units - * @return maximum density, in lattice units - */ - virtual LatticeDensity GetDensityMax() const = 0; - - /*** - * Get the minimum pressure, in lattice units - * @return - */ - LatticePressure GetPressureMin() const - { - return GetDensityMin() * Cs2; - } - - /*** - * Get the maximum pressure, in lattice units - * @return - */ - LatticePressure GetPressureMax() const - { - return GetDensityMax() * Cs2; - } - /// @todo: #632 This method must be moved to InOutletPressure virtual LatticeDensity GetDensity(LatticeTimeStep time_step) const = 0; @@ -194,7 +157,6 @@ namespace hemelb LatticeDensity minimumSimulationDensity; LatticePosition position; util::Vector3D normal; - BoundaryComms* comms; IoletExtraData* extraData; friend class IoletExtraData; }; diff --git a/Code/lb/iolets/InOutLetCosine.cc b/Code/lb/iolets/InOutLetCosine.cc index 356800166..fdf7a2e34 100644 --- a/Code/lb/iolets/InOutLetCosine.cc +++ b/Code/lb/iolets/InOutLetCosine.cc @@ -6,7 +6,6 @@ #include "lb/iolets/InOutLetCosine.h" #include "configuration/SimConfig.h" -#include "net/IOCommunicator.h" namespace hemelb { diff --git a/Code/lb/iolets/InOutLetCosine.h b/Code/lb/iolets/InOutLetCosine.h index ebf55cedc..ac1b5e3f2 100644 --- a/Code/lb/iolets/InOutLetCosine.h +++ b/Code/lb/iolets/InOutLetCosine.h @@ -52,15 +52,6 @@ namespace hemelb densityAmp = rho; } - LatticeDensity GetDensityMin() const - { - return (densityMean - densityAmp); - } - LatticeDensity GetDensityMax() const - { - return (densityMean + densityAmp); - } - LatticePressure GetPressureMean() const { return densityMean * Cs2; diff --git a/Code/lb/iolets/InOutLetFile.cc b/Code/lb/iolets/InOutLetFile.cc index ceac0d063..e3c214749 100644 --- a/Code/lb/iolets/InOutLetFile.cc +++ b/Code/lb/iolets/InOutLetFile.cc @@ -70,9 +70,6 @@ namespace hemelb std::vector values(0); // Must convert into vectors since LinearInterpolate works on a pair of vectors - // Determine min and max pressure on the way - PhysicalPressure pMin = timeValuePairs.begin()->second; - PhysicalPressure pMax = timeValuePairs.begin()->second; for (std::map::iterator entry = timeValuePairs.begin(); entry != timeValuePairs.end(); entry++) { @@ -87,19 +84,13 @@ namespace hemelb PhysicalSpeed final_pressure = values.back() + time_diff_ratio * pres_diff; times.push_back(totalTimeSteps*timeStepLength); - pMin = util::NumericalFunctions::min(pMin, final_pressure); - pMax = util::NumericalFunctions::max(pMax, final_pressure); values.push_back(final_pressure); break; } - pMin = util::NumericalFunctions::min(pMin, entry->second); - pMax = util::NumericalFunctions::max(pMax, entry->second); times.push_back(entry->first); values.push_back(entry->second); } - densityMin = units->ConvertPressureToLatticeUnits(pMin) / Cs2; - densityMax = units->ConvertPressureToLatticeUnits(pMax) / Cs2; // Check if last point's value matches the first if (values.back() != values.front()) diff --git a/Code/lb/iolets/InOutLetFile.h b/Code/lb/iolets/InOutLetFile.h index c94888897..a80a46c69 100644 --- a/Code/lb/iolets/InOutLetFile.h +++ b/Code/lb/iolets/InOutLetFile.h @@ -45,14 +45,6 @@ namespace hemelb pressureFilePath = path; } - LatticeDensity GetDensityMin() const - { - return densityMin; - } - LatticeDensity GetDensityMax() const - { - return densityMax; - } LatticeDensity GetDensity(LatticeTimeStep timeStep) const { return densityTable[timeStep]; @@ -61,8 +53,6 @@ namespace hemelb private: void CalculateTable(LatticeTimeStep totalTimeSteps, PhysicalTime timeStepLength); std::vector densityTable; - LatticeDensity densityMin; - LatticeDensity densityMax; std::string pressureFilePath; const util::UnitConverter* units; }; diff --git a/Code/lb/iolets/InOutLetFileVelocity.cc b/Code/lb/iolets/InOutLetFileVelocity.cc index 6875c1448..8c5e144fb 100644 --- a/Code/lb/iolets/InOutLetFileVelocity.cc +++ b/Code/lb/iolets/InOutLetFileVelocity.cc @@ -56,9 +56,6 @@ namespace hemelb std::vector values(0); // Must convert into vectors since LinearInterpolate works on a pair of vectors - // Determine min and max pressure on the way -// PhysicalPressure pMin = timeValuePairs.begin()->second; -// PhysicalPressure pMax = timeValuePairs.begin()->second; for (std::map::iterator entry = timeValuePairs.begin(); entry != timeValuePairs.end(); entry++) { @@ -81,8 +78,6 @@ namespace hemelb times.push_back(entry->first); values.push_back(entry->second); } -// densityMin = units->ConvertPressureToLatticeUnits(pMin) / Cs2; -// densityMax = units->ConvertPressureToLatticeUnits(pMax) / Cs2; /* If the time values in the input file end BEFORE the planned end of the simulation, then loop the profile afterwards (using %TimeStepsInInletVelocityProfile). */ int TimeStepsInInletVelocityProfile = times.back() / timeStepLength; diff --git a/Code/lb/iolets/InOutLetMultiscale.cc b/Code/lb/iolets/InOutLetMultiscale.cc index 3dfaec922..711868192 100644 --- a/Code/lb/iolets/InOutLetMultiscale.cc +++ b/Code/lb/iolets/InOutLetMultiscale.cc @@ -6,9 +6,8 @@ #include "lb/iolets/InOutLetMultiscale.h" #include "configuration/SimConfig.h" -#include "net/IOCommunicator.h" -#include "lb/iolets/BoundaryComms.h" #include "lb/iolets/BoundaryValues.h" +#include "comm/Async.h" namespace hemelb { @@ -71,14 +70,6 @@ namespace hemelb /* TODO: Fix pressure and GetPressure values (using PressureMax() for now). */ return units->ConvertPressureToLatticeUnits(maxPressure.GetPayload()) / Cs2; } - LatticeDensity InOutLetMultiscale::GetDensityMin() const - { - return units->ConvertPressureToLatticeUnits(minPressure.GetPayload()) / Cs2; - } - LatticeDensity InOutLetMultiscale::GetDensityMax() const - { - return units->ConvertPressureToLatticeUnits(maxPressure.GetPayload()) / Cs2; - } PhysicalVelocity InOutLetMultiscale::GetVelocity() const { return velocity; @@ -117,43 +108,29 @@ namespace hemelb commsRequired = b; } - /* Distribution of internal pressure values */ - void InOutLetMultiscale::DoComms(const BoundaryCommunicator& bcComms, LatticeTimeStep time_step) - { - bool isIoProc = bcComms.IsCurrentProcTheBCProc(); - hemelb::log::Logger::Log("DoComms in IoletMultiscale triggered: %s", - isIoProc - ? "true" - : "false"); - double pressure_array[3]; + void InOutLetMultiscale::Begin(BoundaryValues* bv) { //TODO: Change these operators on SharedValue. pressure_array[0] = pressure.GetPayload(); pressure_array[1] = minPressure.GetPayload(); pressure_array[2] = maxPressure.GetPayload(); - - net::Net commsNet(bcComms); - - const std::vector& procList = comms->GetListOfProcs(); //TODO: CHECK + IMPROVE! - - // If this proc is to do IO, send the pressure array list to all cores that require it. - if (isIoProc && procList[0] != bcComms.GetBCProcRank()) - { - for (std::vector::const_iterator it = procList.begin(); it != procList.end(); it++) - { - commsNet.RequestSend(pressure_array, 3, *it); - } - } - // Otherwise, receive the pressure array list from the core. - else if (procList[0] != bcComms.GetBCProcRank()) - { - commsNet.RequestReceive(pressure_array, 3, bcComms.GetBCProcRank()); - } - - // Perform the send / receive. - commsNet.Dispatch(); - - if (!isIoProc) - { + } + void InOutLetMultiscale::Receive(BoundaryValues* bv, comm::Async::Ptr commQ) { + // everyone receives from BC master proc + commQ->Irecv(pressure_array, 3, bv->GetBCProcRank(), 0); + } + void InOutLetMultiscale::Send(BoundaryValues* bv, comm::Async::Ptr commQ) { + if(bv->IsCurrentProcTheBCProc()) + { + const std::vector& procList = bv->GetProcsForIolet(this); //TODO: CHECK + IMPROVE! + + for (auto dest_rank: procList) + commQ->Isend(pressure_array, 3, dest_rank, 0); + } + } + + void InOutLetMultiscale::CommsComplete(BoundaryValues* bv) { + if (!bv->IsCurrentProcTheBCProc()) + { pressure.SetPayload(static_cast (pressure_array[0])); minPressure.SetPayload(static_cast (pressure_array[1])); maxPressure.SetPayload(static_cast (pressure_array[2])); @@ -161,8 +138,8 @@ namespace hemelb pressure.GetPayload(), minPressure.GetPayload(), maxPressure.GetPayload()); - } - } + } + } } } } diff --git a/Code/lb/iolets/InOutLetMultiscale.h b/Code/lb/iolets/InOutLetMultiscale.h index 7af079b11..11b9dc5d7 100644 --- a/Code/lb/iolets/InOutLetMultiscale.h +++ b/Code/lb/iolets/InOutLetMultiscale.h @@ -11,7 +11,6 @@ #include "multiscale/Intercommunicand.h" #include "multiscale/SharedValue.h" #include "log/Logger.h" -#include "lb/iolets/BoundaryCommunicator.h" namespace hemelb { @@ -55,8 +54,6 @@ namespace hemelb // field points will map one-to-one to HemeLB lattice sites initially; we will // rely on external tools to perform any interpolation tasks. LatticeDensity GetDensity(unsigned long timeStep) const; - virtual LatticeDensity GetDensityMin() const; - virtual LatticeDensity GetDensityMax() const; PhysicalVelocity GetVelocity() const; LatticePressure GetPressure() const; //std::vector minPressure; multiscale::SharedValue maxPressure; mutable multiscale::SharedValue velocity; + double pressure_array[3]; }; } } diff --git a/Code/lb/iolets/InOutLetVelocity.cc b/Code/lb/iolets/InOutLetVelocity.cc index 5fe24f79d..54a3910d0 100644 --- a/Code/lb/iolets/InOutLetVelocity.cc +++ b/Code/lb/iolets/InOutLetVelocity.cc @@ -21,14 +21,6 @@ namespace hemelb { } - LatticeDensity InOutLetVelocity::GetDensityMin() const - { - return 1.0; - } - LatticeDensity InOutLetVelocity::GetDensityMax() const - { - return 1.0; - } LatticeDensity InOutLetVelocity::GetDensity(LatticeTimeStep time_step) const { return 1.0; diff --git a/Code/lb/iolets/InOutLetVelocity.h b/Code/lb/iolets/InOutLetVelocity.h index 16270c591..b31f1adb2 100644 --- a/Code/lb/iolets/InOutLetVelocity.h +++ b/Code/lb/iolets/InOutLetVelocity.h @@ -19,8 +19,6 @@ namespace hemelb public: InOutLetVelocity(); virtual ~InOutLetVelocity(); - LatticeDensity GetDensityMin() const; - LatticeDensity GetDensityMax() const; LatticeDensity GetDensity(LatticeTimeStep time_step) const; void Reset(SimulationState &state) diff --git a/Code/lb/lb.h b/Code/lb/lb.h index 7625a105e..b234fcb7c 100644 --- a/Code/lb/lb.h +++ b/Code/lb/lb.h @@ -7,9 +7,8 @@ #ifndef HEMELB_LB_LB_H #define HEMELB_LB_LB_H -#include "net/net.h" -#include "net/IteratedAction.h" -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" +#include "timestep/Actor.h" #include "lb/SimulationState.h" #include "lb/iolets/BoundaryValues.h" #include "lb/MacroscopicPropertyCache.h" @@ -28,10 +27,10 @@ namespace hemelb { /** * Class providing core Lattice Boltzmann functionality. - * Implements the IteratedAction interface. + * Implements the timestep::Actor interface. */ template - class LBM : public net::IteratedAction + class LBM : public timestep::Actor { private: // Use the kernel specified through the build system. This will select one of the above classes. @@ -56,18 +55,31 @@ namespace hemelb * the partially initialized LBM in order to initialize the arguments to the second construction phase. */ LBM(hemelb::configuration::SimConfig *iSimulationConfig, - net::Net* net, + comm::Async::Ptr commQ, geometry::LatticeData* latDat, SimulationState* simState, reporting::Timers &atimings, geometry::neighbouring::NeighbouringDataManager *neighbouringDataManager); ~LBM(); - void RequestComms(); ///< part of IteratedAction interface. - void PreSend(); ///< part of IteratedAction interface. - void PreReceive(); ///< part of IteratedAction interface. - void PostReceive(); ///< part of IteratedAction interface. - void EndIteration(); ///< part of IteratedAction interface. + // Actor interface + inline virtual void BeginAll() { + // No-op + } + virtual void Begin() { + // No-op + } + virtual void Receive(); + virtual void PreSend(); + virtual void Send(); + virtual void PreWait(); + inline virtual void Wait() + { + // This is a no-op as waiting is delegated to the async + // comms actor + } + virtual void End(); + virtual void EndAll(); site_t TotalFluidSiteCount() const; void SetTotalFluidSiteCount(site_t); @@ -84,27 +96,18 @@ namespace hemelb * Second constructor. * */ - void Initialise(vis::Control* iControl, - iolets::BoundaryValues* iInletValues, + void Initialise(iolets::BoundaryValues* iInletValues, iolets::BoundaryValues* iOutletValues, const util::UnitConverter* iUnits); - void ReadVisParameters(); - - void CalculateMouseFlowField(const ScreenDensity densityIn, - const ScreenStress stressIn, - const LatticeDensity density_threshold_min, - const LatticeDensity density_threshold_minmax_inv, - const LatticeStress stress_threshold_max_inv, - PhysicalPressure &mouse_pressure, - PhysicalStress &mouse_stress); - hemelb::lb::LbmParameters *GetLbmParams(); lb::MacroscopicPropertyCache& GetPropertyCache(); private: void SetInitialConditions(); + LatticeDensity GetInitialDensity() const; + void InitCollisions(); // The following function pair simplify initialising the site ranges for each collider object. void InitInitParamsSiteRanges(kernels::InitParams& initParams, unsigned& state); @@ -129,40 +132,25 @@ namespace hemelb template void StreamAndCollide(Collision* collision, const site_t iFirstIndex, const site_t iSiteCount) { - if (mVisControl->IsRendering()) - { - collision->template StreamAndCollide (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); - } - else - { - collision->template StreamAndCollide (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); - } + collision->StreamAndCollide(iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); } template void PostStep(Collision* collision, const site_t iFirstIndex, const site_t iSiteCount) { - if (mVisControl->IsRendering()) - { - collision->template DoPostStep (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); - } - else - { - collision->template DoPostStep (iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); - } + collision->DoPostStep(iFirstIndex, iSiteCount, &mParams, mLatDat, propertyCache); } unsigned int inletCount; unsigned int outletCount; configuration::SimConfig *mSimConfig; - net::Net* mNet; + comm::Async::Ptr mCommQ; geometry::LatticeData* mLatDat; SimulationState* mState; iolets::BoundaryValues *mInletValues, *mOutletValues; LbmParameters mParams; - vis::Control* mVisControl; const util::UnitConverter* mUnits; diff --git a/Code/lb/lb.hpp b/Code/lb/lb.hpp index f965de44d..133ac3451 100644 --- a/Code/lb/lb.hpp +++ b/Code/lb/lb.hpp @@ -7,7 +7,6 @@ #ifndef HEMELB_LB_LB_HPP #define HEMELB_LB_LB_HPP -#include "io/writers/xdr/XdrMemWriter.h" #include "lb/lb.h" namespace hemelb @@ -29,34 +28,18 @@ namespace hemelb template LBM::LBM(configuration::SimConfig *iSimulationConfig, - net::Net* net, + comm::Async::Ptr commQ, geometry::LatticeData* latDat, SimulationState* simState, reporting::Timers &atimings, geometry::neighbouring::NeighbouringDataManager *neighbouringDataManager) : - mSimConfig(iSimulationConfig), mNet(net), mLatDat(latDat), mState(simState), + mSimConfig(iSimulationConfig), mCommQ(commQ), mLatDat(latDat), mState(simState), mParams(iSimulationConfig->GetTimeStepLength(), iSimulationConfig->GetVoxelSize()), timings(atimings), propertyCache(*simState, *latDat), neighbouringDataManager(neighbouringDataManager) { ReadParameters(); } - template - void LBM::CalculateMouseFlowField(const ScreenDensity densityIn, - const ScreenStress stressIn, - const LatticeDensity density_threshold_min, - const LatticeDensity density_threshold_minmax_inv, - const LatticeStress stress_threshold_max_inv, - PhysicalPressure &mouse_pressure, - PhysicalStress &mouse_stress) - { - LatticeDensity density = density_threshold_min + densityIn / density_threshold_minmax_inv; - LatticeStress stress = stressIn / stress_threshold_max_inv; - - mouse_pressure = mUnits->ConvertPressureToPhysicalUnits(density * Cs2); - mouse_stress = mUnits->ConvertStressToPhysicalUnits(stress); - } - template void LBM::InitInitParamsSiteRanges(kernels::InitParams& initParams, unsigned& state) { @@ -126,8 +109,7 @@ namespace hemelb } template - void LBM::Initialise(vis::Control* iControl, - iolets::BoundaryValues* iInletValues, + void LBM::Initialise(iolets::BoundaryValues* iInletValues, iolets::BoundaryValues* iOutletValues, const util::UnitConverter* iUnits) { @@ -138,42 +120,34 @@ namespace hemelb InitCollisions(); SetInitialConditions(); - - mVisControl = iControl; } template void LBM::PrepareBoundaryObjects() { - // First, iterate through all of the inlet and outlet objects, finding out the minimum density seen in the simulation. - distribn_t minDensity = std::numeric_limits::max(); - + LatticeDensity initialDensity = GetInitialDensity(); + // Now go through iolets, informing them of the minimum density. for (unsigned inlet = 0; inlet < mInletValues->GetLocalIoletCount(); ++inlet) { - minDensity = std::min(minDensity, mInletValues->GetLocalIolet(inlet)->GetDensityMin()); + mInletValues->GetLocalIolet(inlet)->SetMinimumSimulationDensity(initialDensity); } for (unsigned outlet = 0; outlet < mOutletValues->GetLocalIoletCount(); ++outlet) { - minDensity = std::min(minDensity, mOutletValues->GetLocalIolet(outlet)->GetDensityMin()); - } - - // Now go through them again, informing them of the minimum density. - for (unsigned inlet = 0; inlet < mInletValues->GetLocalIoletCount(); ++inlet) - { - mInletValues->GetLocalIolet(inlet)->SetMinimumSimulationDensity(minDensity); + mOutletValues->GetLocalIolet(outlet)->SetMinimumSimulationDensity(initialDensity); } + } - for (unsigned outlet = 0; outlet < mOutletValues->GetLocalIoletCount(); ++outlet) - { - mOutletValues->GetLocalIolet(outlet)->SetMinimumSimulationDensity(minDensity); - } + template + LatticeDensity LBM::GetInitialDensity() const + { + return mUnits->ConvertPressureToLatticeUnits(mSimConfig->GetInitialPressure()) / Cs2; } template void LBM::SetInitialConditions() { - distribn_t density = mUnits->ConvertPressureToLatticeUnits(mSimConfig->GetInitialPressure()) / Cs2; + distribn_t density = GetInitialDensity(); for (site_t i = 0; i < mLatDat->GetLocalFluidSiteCount(); i++) { @@ -192,16 +166,11 @@ namespace hemelb } template - void LBM::RequestComms() + void LBM::Receive() { timings[hemelb::reporting::Timers::lb].Start(); - - // Delegate to the lattice data object to post the asynchronous sends and receives - // (via the Net object). - // NOTE that this doesn't actually *perform* the sends and receives, it asks the Net - // to include them in the ISends and IRecvs that happen later. - mLatDat->SendAndReceive(mNet); - + // Delegate to the lattice data object + mLatDat->Receive(mCommQ); timings[hemelb::reporting::Timers::lb].Stop(); } @@ -212,10 +181,12 @@ namespace hemelb timings[hemelb::reporting::Timers::lb_calc].Start(); /** - * In the PreSend phase, we do LB on all the sites that need to have results sent to - * neighbouring ranks ('domainEdge' sites). In site id terms, this means we start at the - * end of the sites whose neighbours all lie on this rank ('midDomain'), then progress - * through the sites of each type in turn. + * In the PreSend phase, we do LB on all the sites that need to + * have results sent to neighbouring ranks ('domainEdge' + * sites). In site id terms, this means we start at the end of + * the sites whose neighbours all lie on this rank + * ('midDomain'), then progress through the sites of each type + * in turn. */ site_t offset = mLatDat->GetMidDomainSiteCount(); @@ -225,11 +196,9 @@ namespace hemelb StreamAndCollide(mWallCollision, offset, mLatDat->GetDomainEdgeCollisionCount(1)); offset += mLatDat->GetDomainEdgeCollisionCount(1); - mInletValues->FinishReceive(); StreamAndCollide(mInletCollision, offset, mLatDat->GetDomainEdgeCollisionCount(2)); offset += mLatDat->GetDomainEdgeCollisionCount(2); - mOutletValues->FinishReceive(); StreamAndCollide(mOutletCollision, offset, mLatDat->GetDomainEdgeCollisionCount(3)); offset += mLatDat->GetDomainEdgeCollisionCount(3); @@ -241,20 +210,31 @@ namespace hemelb timings[hemelb::reporting::Timers::lb_calc].Stop(); timings[hemelb::reporting::Timers::lb].Stop(); } - + template - void LBM::PreReceive() + void LBM::Send() + { + timings[hemelb::reporting::Timers::lb].Start(); + // Delegate to the lattice data + mLatDat->Send(mCommQ); + timings[hemelb::reporting::Timers::lb].Stop(); + } + + template + void LBM::PreWait() { timings[hemelb::reporting::Timers::lb].Start(); timings[hemelb::reporting::Timers::lb_calc].Start(); /** - * In the PreReceive phase, we perform LB for all the sites whose neighbours lie on this - * rank ('midDomain' rather than 'domainEdge' sites). Ideally this phase is the longest bit (maximising time for the asynchronous sends - * and receives to complete). + * In the PreWait phase, we perform LB for all the sites whose + * neighbours lie on this rank ('midDomain' rather than + * 'domainEdge' sites). Ideally this phase is the longest bit + * (maximising time for the asynchronous sends and receives to + * complete). * - * In site id terms, this means starting at the first site and progressing through the - * midDomain sites, one type at a time. + * In site id terms, this means starting at the first site and + * progressing through the midDomain sites, one type at a time. */ site_t offset = 0; @@ -280,7 +260,7 @@ namespace hemelb } template - void LBM::PostReceive() + void LBM::End() { timings[hemelb::reporting::Timers::lb].Start(); @@ -336,7 +316,7 @@ namespace hemelb } template - void LBM::EndIteration() + void LBM::EndAll() { timings[hemelb::reporting::Timers::lb].Start(); timings[hemelb::reporting::Timers::lb_calc].Start(); @@ -370,37 +350,6 @@ namespace hemelb mParams.StressType = mSimConfig->GetStressType(); } - template - void LBM::ReadVisParameters() - { - distribn_t density_min = std::numeric_limits::max(); - distribn_t density_max = std::numeric_limits::min(); - - distribn_t velocity_max = mUnits->ConvertVelocityToLatticeUnits(mSimConfig->GetMaximumVelocity()); - distribn_t stress_max = mUnits->ConvertStressToLatticeUnits(mSimConfig->GetMaximumStress()); - - for (int i = 0; i < InletCount(); i++) - { - density_min = util::NumericalFunctions::min(density_min, mInletValues->GetDensityMin(i)); - density_max = util::NumericalFunctions::max(density_max, mInletValues->GetDensityMax(i)); - } - for (int i = 0; i < OutletCount(); i++) - { - density_min = util::NumericalFunctions::min(density_min, mOutletValues->GetDensityMin(i)); - density_max = util::NumericalFunctions::max(density_max, mOutletValues->GetDensityMax(i)); - } - - distribn_t lDensity_threshold_min = density_min; - distribn_t lDensity_threshold_minmax_inv = 1.0F / (density_max - density_min); - distribn_t lVelocity_threshold_max_inv = 1.0F / velocity_max; - distribn_t lStress_threshold_max_inv = 1.0F / stress_max; - - mVisControl->SetSomeParams(mSimConfig->GetVisualisationBrightness(), - lDensity_threshold_min, - lDensity_threshold_minmax_inv, - lVelocity_threshold_max_inv, - lStress_threshold_max_inv); - } } } diff --git a/Code/lb/streamers/BaseStreamer.h b/Code/lb/streamers/BaseStreamer.h index d04a83053..0e07f61b0 100644 --- a/Code/lb/streamers/BaseStreamer.h +++ b/Code/lb/streamers/BaseStreamer.h @@ -10,7 +10,6 @@ #include #include "geometry/LatticeData.h" -#include "vis/Control.h" #include "lb/LbmParameters.h" #include "lb/kernels/BaseKernel.h" #include "lb/MacroscopicPropertyCache.h" @@ -26,20 +25,20 @@ namespace hemelb * BaseStreamer: inheritable base class for the streaming operator. The public interface * here defines the complete interface usable by external code. * - Constructor(InitParams&) - * - StreamAndCollide(const site_t, const site_t, const LbmParameters*, - * geometry::LatticeData*, hemelb::vis::Control*) - * - PostStep(const site_t, const site_t, const LbmParameters*, - * geometry::LatticeData*, hemelb::vis::Control*) + * - StreamAndCollide(const site_t, const site_t, const LbmParameters*, + * geometry::LatticeData*, lb::MacroscopicPropertyCache&) + * - PostStep(const site_t, const site_t, const LbmParameters*, + * geometry::LatticeData*, lb::MacroscopicPropertyCache&) * - Reset(kernels::InitParams* init) * * The following must be implemented by concrete streamers (which derive from this class * using the CRTP). * - typedef for CollisionType, the type of the collider operation. * - Constructor(InitParams&) - * - DoStreamAndCollide(const site_t, const site_t, const LbmParameters*, - * geometry::LatticeData*, hemelb::vis::Control*) - * - DoPostStep(const site_t, const site_t, const LbmParameters*, - * geometry::LatticeData*, hemelb::vis::Control*) + * - DoStreamAndCollide(const site_t, const site_t, const LbmParameters*, + * geometry::LatticeData*, lb::MacroscopicPropertyCache&) + * - DoPostStep(const site_t, const site_t, const LbmParameters*, + * geometry::LatticeData*, lb::MacroscopicPropertyCache&) * - DoReset(kernels::InitParams* init) * * The design is to for the streamers to be pretty dumb and for them to @@ -54,42 +53,37 @@ namespace hemelb class BaseStreamer { public: - template - inline void StreamAndCollide(const site_t firstIndex, - const site_t siteCount, + inline void StreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) { - static_cast (this)->template DoStreamAndCollide (firstIndex, - siteCount, - lbmParams, - latDat, - propertyCache); + static_cast(this)->DoStreamAndCollide(firstIndex, + siteCount, + lbmParams, + latDat, + propertyCache); } - template - inline void PostStep(const site_t firstIndex, - const site_t siteCount, - const LbmParameters* lbmParams, - geometry::LatticeData* latDat, + inline void PostStep(const site_t firstIndex, const site_t siteCount, + const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) { // The template parameter is required because we're using the CRTP to call a // metaprogrammed method of the implementation class. - static_cast (this)->template DoPostStep (firstIndex, - siteCount, - lbmParams, - latDat, - propertyCache); + static_cast(this)->DoPostStep(firstIndex, + siteCount, + lbmParams, + latDat, + propertyCache); } protected: - template - inline static void UpdateMinsAndMaxes(const geometry::Site& site, - const kernels::HydroVarsBase& hydroVars, - const LbmParameters* lbmParams, - lb::MacroscopicPropertyCache& propertyCache) + template + inline static void UpdateMinsAndMaxes( + const geometry::Site& site, + const kernels::HydroVarsBase& hydroVars, const LbmParameters* lbmParams, + lb::MacroscopicPropertyCache& propertyCache) { if (propertyCache.densityCache.RequiresRefresh()) { @@ -191,7 +185,8 @@ namespace hemelb tangentialProjectionTractionOnAPoint); } - propertyCache.tangentialProjectionTractionCache.Put(site.GetIndex(), tangentialProjectionTractionOnAPoint); + propertyCache.tangentialProjectionTractionCache.Put(site.GetIndex(), + tangentialProjectionTractionOnAPoint); } } diff --git a/Code/lb/streamers/JunkYangFactory.h b/Code/lb/streamers/JunkYangFactory.h index 8373ba240..1a5d046b6 100644 --- a/Code/lb/streamers/JunkYangFactory.h +++ b/Code/lb/streamers/JunkYangFactory.h @@ -7,6 +7,8 @@ #ifndef HEMELB_LB_STREAMERS_JUNKYANGFACTORY_H #define HEMELB_LB_STREAMERS_JUNKYANGFACTORY_H +#include + #include "lb/kernels/BaseKernel.h" #include "lb/streamers/BaseStreamer.h" #include @@ -75,7 +77,6 @@ namespace hemelb } } - template inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latticeData, @@ -143,15 +144,14 @@ namespace hemelb fOld[siteIndex](index) = site.GetFOld()[*outgoingVelocityIter]; } - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template inline void DoPostStep(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latticeData, lb::MacroscopicPropertyCache& propertyCache) @@ -322,8 +322,10 @@ namespace hemelb + LatticeType::CZ[*rowIndexIncomingVelocity] * LatticeType::CZ[*columnIndexIncomingVelocity]; - assert(site.template GetWallDistance (LatticeType::INVERSEDIRECTIONS[(Direction) *rowIndexIncomingVelocity]) >= 0); - assert(site.template GetWallDistance (LatticeType::INVERSEDIRECTIONS[(Direction) *rowIndexIncomingVelocity]) < 1); + assert(site.template GetWallDistance(LatticeType::INVERSEDIRECTIONS[(Direction ) *rowIndexIncomingVelocity]) + >= 0); + assert(site.template GetWallDistance(LatticeType::INVERSEDIRECTIONS[(Direction ) *rowIndexIncomingVelocity]) + < 1); kMatrices[contiguousSiteIndex](rowIndex, columnIndex) = (-3.0 / 2.0) @@ -364,8 +366,10 @@ namespace hemelb + LatticeType::CZ[*rowIndexIncomingVelocity] * LatticeType::CZ[*columnIndexOutgoingVelocity]; - assert(site.template GetWallDistance (LatticeType::INVERSEDIRECTIONS[(Direction) *rowIndexIncomingVelocity]) >= 0); - assert(site.template GetWallDistance (LatticeType::INVERSEDIRECTIONS[(Direction) *rowIndexIncomingVelocity]) < 1); + assert(site.template GetWallDistance(LatticeType::INVERSEDIRECTIONS[(Direction ) *rowIndexIncomingVelocity]) + >= 0); + assert(site.template GetWallDistance(LatticeType::INVERSEDIRECTIONS[(Direction ) *rowIndexIncomingVelocity]) + < 1); kMatrices[contiguousSiteIndex](rowIndex, columnIndex) = (-3.0 / 2.0) diff --git a/Code/lb/streamers/SimpleCollideAndStream.h b/Code/lb/streamers/SimpleCollideAndStream.h index 1b486daed..204d9d2ff 100644 --- a/Code/lb/streamers/SimpleCollideAndStream.h +++ b/Code/lb/streamers/SimpleCollideAndStream.h @@ -32,14 +32,12 @@ namespace hemelb public: SimpleCollideAndStream(kernels::InitParams& initParams) : - collider(initParams), bulkLinkDelegate(collider, initParams) + collider(initParams), bulkLinkDelegate(collider, initParams) { } - template - inline void DoStreamAndCollide(const site_t firstIndex, - const site_t siteCount, + inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) @@ -48,7 +46,7 @@ namespace hemelb { geometry::Site site = latDat->GetSite(siteIndex); - const distribn_t* lFOld = site.GetFOld (); + const distribn_t* lFOld = site.GetFOld(); kernels::HydroVars hydroVars(lFOld); @@ -64,18 +62,15 @@ namespace hemelb bulkLinkDelegate.StreamLink(lbmParams, latDat, site, hydroVars, ii); } - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template - inline void DoPostStep(const site_t iFirstIndex, - const site_t iSiteCount, - const LbmParameters* iLbmParams, - geometry::LatticeData* bLatDat, + inline void DoPostStep(const site_t iFirstIndex, const site_t iSiteCount, + const LbmParameters* iLbmParams, geometry::LatticeData* bLatDat, lb::MacroscopicPropertyCache& propertyCache) { diff --git a/Code/lb/streamers/StreamerTypeFactory.h b/Code/lb/streamers/StreamerTypeFactory.h index 496238356..511dec763 100644 --- a/Code/lb/streamers/StreamerTypeFactory.h +++ b/Code/lb/streamers/StreamerTypeFactory.h @@ -26,7 +26,8 @@ namespace hemelb * template on WallLinkImpl. */ template - class WallStreamerTypeFactory : public BaseStreamer > + class WallStreamerTypeFactory : public BaseStreamer< + WallStreamerTypeFactory > { public: typedef CollisionImpl CollisionType; @@ -40,14 +41,13 @@ namespace hemelb public: WallStreamerTypeFactory(kernels::InitParams& initParams) : - collider(initParams), bulkLinkDelegate(collider, initParams), wallLinkDelegate(collider, initParams) + collider(initParams), bulkLinkDelegate(collider, initParams), + wallLinkDelegate(collider, initParams) { } - template - inline void DoStreamAndCollide(const site_t firstIndex, - const site_t siteCount, + inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) @@ -56,7 +56,7 @@ namespace hemelb { geometry::Site site = latDat->GetSite(siteIndex); - const distribn_t* fOld = site.GetFOld (); + const distribn_t* fOld = site.GetFOld(); kernels::HydroVars hydroVars(fOld); @@ -80,15 +80,14 @@ namespace hemelb } //TODO: Necessary to specify sub-class? - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template - inline void DoPostStep(const site_t firstIndex, - const site_t siteCount, + + inline void DoPostStep(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParameters, geometry::LatticeData* latticeData, lb::MacroscopicPropertyCache& propertyCache) @@ -117,7 +116,8 @@ namespace hemelb * template on IoletLinkImpl. */ template - class IoletStreamerTypeFactory : public BaseStreamer > + class IoletStreamerTypeFactory : public BaseStreamer< + IoletStreamerTypeFactory > { public: typedef CollisionImpl CollisionType; @@ -131,14 +131,13 @@ namespace hemelb public: IoletStreamerTypeFactory(kernels::InitParams& initParams) : - collider(initParams), bulkLinkDelegate(collider, initParams), ioletLinkDelegate(collider, initParams) + collider(initParams), bulkLinkDelegate(collider, initParams), + ioletLinkDelegate(collider, initParams) { } - template - inline void DoStreamAndCollide(const site_t firstIndex, - const site_t siteCount, + inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) @@ -147,7 +146,7 @@ namespace hemelb { geometry::Site site = latDat->GetSite(siteIndex); - const distribn_t* fOld = site.GetFOld (); + const distribn_t* fOld = site.GetFOld(); kernels::HydroVars hydroVars(fOld); @@ -171,15 +170,14 @@ namespace hemelb } //TODO: Necessary to specify sub-class? - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template - inline void DoPostStep(const site_t firstIndex, - const site_t siteCount, + + inline void DoPostStep(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParameters, geometry::LatticeData* latticeData, lb::MacroscopicPropertyCache& propertyCache) @@ -209,8 +207,8 @@ namespace hemelb * template on WallLinkImpl and IoletLinkImpl. */ template - class WallIoletStreamerTypeFactory : public BaseStreamer > + class WallIoletStreamerTypeFactory : public BaseStreamer< + WallIoletStreamerTypeFactory > { public: typedef CollisionImpl CollisionType; @@ -224,15 +222,13 @@ namespace hemelb public: WallIoletStreamerTypeFactory(kernels::InitParams& initParams) : - collider(initParams), bulkLinkDelegate(collider, initParams), wallLinkDelegate(collider, initParams), - ioletLinkDelegate(collider, initParams) + collider(initParams), bulkLinkDelegate(collider, initParams), + wallLinkDelegate(collider, initParams), ioletLinkDelegate(collider, initParams) { } - template - inline void DoStreamAndCollide(const site_t firstIndex, - const site_t siteCount, + inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) @@ -241,7 +237,7 @@ namespace hemelb { geometry::Site site = latDat->GetSite(siteIndex); - const distribn_t* fOld = site.GetFOld (); + const distribn_t* fOld = site.GetFOld(); kernels::HydroVars hydroVars(fOld); @@ -269,18 +265,15 @@ namespace hemelb } //TODO: Necessary to specify sub-class? - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template - inline void DoPostStep(const site_t firstIndex, - const site_t siteCount, - const LbmParameters* lbmParams, - geometry::LatticeData* latticeData, + inline void DoPostStep(const site_t firstIndex, const site_t siteCount, + const LbmParameters* lbmParams, geometry::LatticeData* latticeData, lb::MacroscopicPropertyCache& propertyCache) { for (site_t siteIndex = firstIndex; siteIndex < (firstIndex + siteCount); siteIndex++) diff --git a/Code/lb/streamers/VirtualSiteIolet.h b/Code/lb/streamers/VirtualSiteIolet.h index 4e9d2bc31..8ed470330 100644 --- a/Code/lb/streamers/VirtualSiteIolet.h +++ b/Code/lb/streamers/VirtualSiteIolet.h @@ -47,22 +47,21 @@ namespace hemelb struct IoletVSiteDirection { IoletVSiteDirection(InOutLet*iolet_, VirtualSite* vsite_, Direction i_) : - iolet(iolet_), vsite(vsite_), direction(i_) + iolet(iolet_), vsite(vsite_), direction(i_) { } InOutLet* iolet; VirtualSite* vsite; Direction direction; }; - typedef typename util::FlatMultiMap::Type - VSiteByLocalIdxMultiMap; + typedef typename util::FlatMultiMap::Type VSiteByLocalIdxMultiMap; VSiteByLocalIdxMultiMap vsByLocalIdx; public: VirtualSiteIolet(kernels::InitParams& initParams) : - collider(initParams), bulkLinkDelegate(collider, initParams), - wallLinkDelegate(collider, initParams), bValues(initParams.boundaryObject), - neighbouringLatticeData(initParams.latDat->GetNeighbouringData()) + collider(initParams), bulkLinkDelegate(collider, initParams), + wallLinkDelegate(collider, initParams), bValues(initParams.boundaryObject), + neighbouringLatticeData(initParams.latDat->GetNeighbouringData()) { // Loop over the local in/outlets, creating the extra data objects. unsigned nIolets = bValues->GetLocalIoletCount(); @@ -70,7 +69,7 @@ namespace hemelb { InOutLet& iolet = *bValues->GetLocalIolet(iIolet); if (iolet.GetExtraData() == NULL) - iolet.SetExtraData(new VSExtra (iolet)); + iolet.SetExtraData(new VSExtra(iolet)); } lattices::LatticeInfo& lattice = LatticeType::GetLatticeInfo(); @@ -105,21 +104,19 @@ namespace hemelb continue; const LatticeVector neighbourLocation = siteLocation + lattice.GetVector(i); - site_t - neighbourGlobalIdx = - initParams.latDat->GetGlobalNoncontiguousSiteIdFromGlobalCoords(neighbourLocation); + site_t neighbourGlobalIdx = + initParams.latDat->GetGlobalNoncontiguousSiteIdFromGlobalCoords(neighbourLocation); typename VSiteType::Map::iterator vNeigh = extra->vSites.find(neighbourGlobalIdx); // find() returns end() if key not present if (vNeigh == extra->vSites.end()) { // Create a vSite - std::pair - inserted = - extra->vSites.insert(typename VSiteType::Map::value_type(neighbourGlobalIdx, - VSiteType(initParams, - *extra, - neighbourLocation))); + std::pair inserted = + extra->vSites.insert(typename VSiteType::Map::value_type(neighbourGlobalIdx, + VSiteType(initParams, + *extra, + neighbourLocation))); // inserted.first is an iterator, pointing to a pair vNeigh = inserted.first; } @@ -140,7 +137,6 @@ namespace hemelb * links will be done in the post-step as we must ensure that all * the data is available to construct virtual sites. */ - template inline void DoStreamAndCollide(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, @@ -150,7 +146,7 @@ namespace hemelb { geometry::Site site = latDat->GetSite(siteIndex); - const distribn_t* fOld = site.GetFOld (); + const distribn_t* fOld = site.GetFOld(); kernels::HydroVars hydroVars(fOld); @@ -192,14 +188,13 @@ namespace hemelb cachedHV.u = hydroVars.velocity; // TODO: Necessary to specify sub-class? - BaseStreamer::template UpdateMinsAndMaxes(site, - hydroVars, - lbmParams, - propertyCache); + BaseStreamer::UpdateMinsAndMaxes(site, + hydroVars, + lbmParams, + propertyCache); } } - template inline void DoPostStep(const site_t firstIndex, const site_t siteCount, const LbmParameters* lbmParams, geometry::LatticeData* latDat, lb::MacroscopicPropertyCache& propertyCache) @@ -209,8 +204,8 @@ namespace hemelb vsByLocalIdx.lower_bound(firstIndex), endVSites = vsByLocalIdx.lower_bound(firstIndex + siteCount); - for (typename VSiteByLocalIdxMultiMap::iterator vSiteIt = beginVSites; vSiteIt - != endVSites; ++vSiteIt) + for (typename VSiteByLocalIdxMultiMap::iterator vSiteIt = beginVSites; + vSiteIt != endVSites; ++vSiteIt) { site_t siteIdx = vSiteIt->first; // vSiteIt->second == (Iolet*, VirtualSite*, Direction) @@ -238,8 +233,8 @@ namespace hemelb std::ofstream hvCache("hvCache"); hvCache << "# local global x y z" << std::endl; - for (RSHV::Map::const_iterator hvIt = extra->hydroVarsCache.begin(); hvIt - != extra->hydroVarsCache.end(); ++hvIt) + for (RSHV::Map::const_iterator hvIt = extra->hydroVarsCache.begin(); + hvIt != extra->hydroVarsCache.end(); ++hvIt) { site_t global = hvIt->first; LatticeVector pos; @@ -272,7 +267,8 @@ namespace hemelb std::ofstream outletMap("outletMap"); outletMap << "# local global x y z vSitePtr direction" << std::endl; for (typename VSiteByLocalIdxMultiMap::const_iterator entry = - ioletStreamer->vsByLocalIdx.begin(); entry != ioletStreamer->vsByLocalIdx.end(); ++entry) + ioletStreamer->vsByLocalIdx.begin(); entry != ioletStreamer->vsByLocalIdx.end(); + ++entry) { site_t local = entry->first; geometry::Site site = latDat->GetSite(local); @@ -286,8 +282,8 @@ namespace hemelb std::ofstream outletWallMap("outletWallMap"); outletWallMap << "# local global x y z vSitePtr direction" << std::endl; for (typename VSiteByLocalIdxMultiMap::const_iterator entry = - ioletWallStreamer->vsByLocalIdx.begin(); entry - != ioletWallStreamer->vsByLocalIdx.end(); ++entry) + ioletWallStreamer->vsByLocalIdx.begin(); + entry != ioletWallStreamer->vsByLocalIdx.end(); ++entry) { site_t local = entry->first; geometry::Site site = latDat->GetSite(local); @@ -303,7 +299,7 @@ namespace hemelb static VSExtra* GetExtra(InOutLet* iolet) { // Get the extra data for this iolet - VSExtra* ans = dynamic_cast*> (iolet->GetExtraData()); + VSExtra* ans = dynamic_cast*>(iolet->GetExtraData()); if (ans == NULL) { // panic @@ -351,7 +347,8 @@ namespace hemelb LatticeDensity CalculateVirtualSiteDensity(const geometry::LatticeData& latDat, const InOutLet& iolet, RSHV::Map& hydroVarsCache, - const VSiteType& vSite, const LatticeTimeStep t) + const VSiteType& vSite, + const LatticeTimeStep t) { LatticeDensity rho = 0.; LatticeDensity rho_iolet = iolet.GetDensity(t); @@ -384,7 +381,8 @@ namespace hemelb LatticeVelocity CalculateVirtualSiteVelocity(const geometry::LatticeData& latDat, const InOutLet& iolet, RSHV::Map& hydroVarsCache, - const VSiteType& vSite, const LatticeTimeStep t) + const VSiteType& vSite, + const LatticeTimeStep t) { /* * Anstaz is u(x,y) = Ax + By + C @@ -437,8 +435,8 @@ namespace hemelb coeffs[i] += vSite.velocityMatrixInv[i][j] * sums[j]; // Compute the magnitude of the velocity. - LatticeSpeed ansNorm = coeffs[0] * vSite.hv.posIolet.x + coeffs[1] - * vSite.hv.posIolet.y + coeffs[2]; + LatticeSpeed ansNorm = coeffs[0] * vSite.hv.posIolet.x + coeffs[1] * vSite.hv.posIolet.y + + coeffs[2]; // multiply by the iolet normal and we're done! return iolet.GetNormal() * ansNorm; @@ -459,7 +457,7 @@ namespace hemelb geometry::neighbouring::ConstNeighbouringSite neigh = latDat.GetNeighbouringData().GetSite(globalIdx); - const distribn_t* fOld = neigh.GetFOld (); + const distribn_t* fOld = neigh.GetFOld(); LatticeType::CalculateDensityAndMomentum(fOld, ans.rho, ans.u.x, ans.u.y, ans.u.z); if (LatticeType::IsLatticeCompressible()) { diff --git a/Code/log/Logger.cc b/Code/log/Logger.cc index 21744132f..da105c0d5 100644 --- a/Code/log/Logger.cc +++ b/Code/log/Logger.cc @@ -13,7 +13,7 @@ #include #include "util/utilityFunctions.h" -#include "net/mpi.h" +#include "comm/MpiEnvironment.h" #include "log/Logger.h" namespace hemelb @@ -31,9 +31,9 @@ namespace hemelb if (thisRank < 0) { // Check that MPI is ready - if (net::MpiEnvironment::Initialized()) + if (comm::MpiEnvironment::Initialized()) { - thisRank = net::MpiCommunicator::World().Rank(); + thisRank = comm::MpiEnvironment::World()->Rank(); } startTime = util::myClock(); } diff --git a/Code/main.cc b/Code/main.cc index b2bdc0bdb..ad677fa13 100644 --- a/Code/main.cc +++ b/Code/main.cc @@ -4,8 +4,7 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/mpi.h" -#include "net/IOCommunicator.h" +#include "comm/MpiEnvironment.h" #include "configuration/CommandLine.h" #include "SimulationMaster.h" @@ -16,13 +15,12 @@ int main(int argc, char *argv[]) // standard output // Bring up MPI - hemelb::net::MpiEnvironment mpi(argc, argv); + hemelb::comm::MpiEnvironment mpi(argc, argv); hemelb::log::Logger::Init(); try { - hemelb::net::MpiCommunicator commWorld = hemelb::net::MpiCommunicator::World(); - - hemelb::net::IOCommunicator hemelbCommunicator(commWorld); + auto commWorld = mpi.World(); + try { // Parse command line @@ -32,7 +30,7 @@ int main(int argc, char *argv[]) hemelb::debug::Debugger::Init(options.GetDebug(), argv[0], commWorld); // Prepare main simulation object... - SimulationMaster master = SimulationMaster(options, hemelbCommunicator); + SimulationMaster master = SimulationMaster(options, commWorld); // ..and run it. master.RunSimulation(); diff --git a/Code/multiscale/MultiscaleSimulationMaster.h b/Code/multiscale/MultiscaleSimulationMaster.h index fd9b9ee22..06d196e52 100644 --- a/Code/multiscale/MultiscaleSimulationMaster.h +++ b/Code/multiscale/MultiscaleSimulationMaster.h @@ -23,7 +23,7 @@ namespace hemelb { public: MultiscaleSimulationMaster(hemelb::configuration::CommandLine &options, - const net::IOCommunicator& ioComm, + comm::Communicator::ConstPtr ioComm, Intercommunicator & aintercomms) : SimulationMaster(options, ioComm), intercomms(aintercomms), multiscaleIoletType("inoutlet") @@ -35,13 +35,9 @@ namespace hemelb inletValues->GetLocalIoletCount(), outletValues->GetLocalIoletCount()); hemelb::log::Logger::Log("inlets: %d", - inletValues->GetLocalIolet(0)->IsCommsRequired(), - inletValues->GetLocalIolet(0)->GetDensityMax(), - inletValues->GetLocalIolet(0)->GetPressureMax()); + inletValues->GetLocalIolet(0)->IsCommsRequired()); hemelb::log::Logger::Log("outlets: %d", - outletValues->GetLocalIolet(0)->IsCommsRequired(), - outletValues->GetLocalIolet(0)->GetDensityMax(), - outletValues->GetLocalIolet(0)->GetPressureMax()); + outletValues->GetLocalIolet(0)->IsCommsRequired()); // we only want to register those iolets which are needed on this process. // Fortunately, the BoundaryValues instance has worked this out for us. @@ -70,7 +66,7 @@ namespace hemelb std::vector GlobalIoletCount; GlobalIoletCount.push_back(inletValues->GetLocalIoletCount()); GlobalIoletCount.push_back(outletValues->GetLocalIoletCount()); - ioComms.Broadcast(GlobalIoletCount, 0); + ioComms->Broadcast(GlobalIoletCount, 0); std::vector > invertedInletBoundaryList(GlobalIoletCount[0]); std::vector > invertedOutletBoundaryList(GlobalIoletCount[1]); @@ -229,39 +225,40 @@ namespace hemelb * (it's hard enough to get the physics right with a consistent * state ;)). */ + // TODO: Derek to please comment on this. + + // With the rejigged BoundarValues, the above comment is + // probably defunct. Hence I have removed the RequestComms + // calls below. (They may have in fact not quite worked as + // intented as they was no call to the net->Dispatch + // method to actually cause communication) + hemelb::log::Logger::Log("inlet and outlet count: %d and %d", inletValues->GetLocalIoletCount(), outletValues->GetLocalIoletCount()); hemelb::log::Logger::Log("inlets: %d", - inletValues->GetLocalIolet(0)->IsCommsRequired(), - inletValues->GetLocalIolet(0)->GetDensityMax(), - inletValues->GetLocalIolet(0)->GetPressureMax()); + inletValues->GetLocalIolet(0)->IsCommsRequired()); hemelb::log::Logger::Log("outlets: %d", - outletValues->GetLocalIolet(0)->IsCommsRequired(), - outletValues->GetLocalIolet(0)->GetDensityMax(), - outletValues->GetLocalIolet(0)->GetPressureMax()); + outletValues->GetLocalIolet(0)->IsCommsRequired()); + // SetCommsRequired(inletValues, true); + // SetCommsRequired(outletValues, true); - SetCommsRequired(inletValues, true); - SetCommsRequired(outletValues, true); - - inletValues->RequestComms(); - outletValues->RequestComms(); - SetCommsRequired(inletValues, false); - SetCommsRequired(outletValues, false); + // inletValues->RequestComms(); + // outletValues->RequestComms(); + // SetCommsRequired(inletValues, false); + // SetCommsRequired(outletValues, false); for (unsigned int i = 0; i < inletValues->GetLocalIoletCount(); i++) { - hemelb::log::Logger::Log("Inlet[%i]: Measured Density is %f. Pressure is %f.", + hemelb::log::Logger::Log("Inlet[%i]: Measured Density is %f.", i, - inletValues->GetLocalIolet(i)->GetDensity(GetState()->GetTimeStep()), - inletValues->GetLocalIolet(i)->GetPressureMax()); + inletValues->GetLocalIolet(i)->GetDensity(GetState()->GetTimeStep())); } for (unsigned int i = 0; i < outletValues->GetLocalIoletCount(); i++) { - hemelb::log::Logger::Log("Outlet[%i]: Measured Density is %f. Pressure is %f.", + hemelb::log::Logger::Log("Outlet[%i]: Measured Density is %f.", i, - outletValues->GetLocalIolet(i)->GetDensity(GetState()->GetTimeStep()), - outletValues->GetLocalIolet(i)->GetPressureMax()); + outletValues->GetLocalIolet(i)->GetDensity(GetState()->GetTimeStep())); } /* Temporary Orchestration hardcode for testing 1/100 step ratio @@ -342,49 +339,18 @@ namespace hemelb std::vector > ExchangeAndCompleteInverseBoundaryList( std::vector > inList) { - std::vector > outList; - int *recvSizes = new int[ioComms.Size()]; - int *recvDispls = new int[ioComms.Size()]; + std::vector > outList(inList.size()); /* TODO: ASSUMPTION: * inList.size() is equal everywhere. This is not necessarily the case. * Use an AllReduce MAX and resize inList accordingly to make the remnant * of the code work here for unequal inList sizes. */ - for (unsigned int i = 0; i < inList.size(); i++) { - int sendSize = ((int) inList[i].size()); - site_t *sendList = new site_t[inList[i].size()]; - for (unsigned int j = 0; j < inList[i].size(); j++) - { - sendList[j] = inList[i][j]; - } - HEMELB_MPI_CALL( MPI_Allgather, - ( &sendSize, 1, MPI_INT, recvSizes, 1, MPI_INT, ioComms )); - - int64_t totalSize = 0; - - int np = ioComms.Size(); - int64_t offset = 0; - - for (int j = 0; j < np; j++) - { - totalSize += recvSizes[j]; - recvDispls[j] = offset; - offset += recvSizes[j]; - } - - site_t *recvList = new site_t[totalSize]; //inList[i].size() - - HEMELB_MPI_CALL( MPI_Allgatherv, - ( sendList, inList[i].size(), MPI_LONG_LONG, recvList, recvSizes, recvDispls, MPI_LONG_LONG, ioComms )); - - std::vector subList; - for (int j = 0; j < totalSize; j++) - { - subList.push_back(recvList[j]); - } - outList.push_back(subList); + int sendSize = inList[i].size(); + auto recvSizes = ioComms->AllGather(sendSize); + auto recvList = ioComms->AllGatherV(inList[i], recvSizes); + outList.emplace_back(std::move(recvList)); } return outList; diff --git a/Code/net/BaseNet.cc b/Code/net/BaseNet.cc deleted file mode 100644 index e6c438fc0..000000000 --- a/Code/net/BaseNet.cc +++ /dev/null @@ -1,83 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -/*! \file net.cc - \brief In this file the functions useful to discover the topology used and - to create and delete the domain decomposition and the various - buffers are defined. - */ - -#include -#include -#include - -#include "net/BaseNet.h" -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" -#include "net/IOCommunicator.h" -namespace hemelb -{ - namespace net - { - - void BaseNet::Dispatch() - { - Send(); - Receive(); - Wait(); - } - - BaseNet::BaseNet(const MpiCommunicator &commObject) : - BytesSent(0), SyncPointsCounted(0), communicator(commObject) - { - } - - void BaseNet::Receive() - { - ReceiveGathers(); - ReceiveGatherVs(); - ReceiveAllToAll(); - // Ensure collectives are called before point-to-point, as some implementing mixins implement collectives via point-to-point - ReceivePointToPoint(); - } - - void BaseNet::Send() - { - - SendGathers(); - SendGatherVs(); - SendAllToAll(); - // Ensure collectives are called before point-to-point, as some implementing mixins implement collectives via point-to-point - SendPointToPoint(); - } - - void BaseNet::Wait() - { - SyncPointsCounted++; //DTMP: counter for monitoring purposes. - - WaitGathers(); - WaitGatherVs(); - WaitPointToPoint(); - WaitAllToAll(); - - displacementsBuffer.clear(); - countsBuffer.clear(); - } - - std::vector & BaseNet::GetDisplacementsBuffer() - { - displacementsBuffer.push_back(std::vector()); - return displacementsBuffer.back(); - } - - std::vector & BaseNet::GetCountsBuffer() - { - countsBuffer.push_back(std::vector()); - return countsBuffer.back(); - } - - } -} diff --git a/Code/net/BaseNet.h b/Code/net/BaseNet.h deleted file mode 100644 index e5320269f..000000000 --- a/Code/net/BaseNet.h +++ /dev/null @@ -1,109 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_BASENET_H -#define HEMELB_NET_BASENET_H - -#include -#include -#include -#include - -#include "constants.h" -#include "net/mpi.h" -#include "net/MpiCommunicator.h" - -namespace hemelb -{ - namespace net - { - - class BaseNet - { - public: - BaseNet(const MpiCommunicator &communicator); - - virtual ~BaseNet() - { - } - ; - - //DTMP: monitoring variables - long long int BytesSent; - long long int SyncPointsCounted; - - void Receive(); - void Send(); - virtual void Wait(); - - /*** - * Carry out a complete send-receive-wait - */ - void Dispatch(); - - inline const MpiCommunicator &GetCommunicator() const - { - return communicator; - } - - inline int Rank() const - { - return communicator.Rank(); - } - inline int Size() const - { - return communicator.Size(); - } - protected: - virtual void SendPointToPoint()=0; - virtual void SendGathers()=0; - virtual void SendGatherVs()=0; - virtual void SendAllToAll()=0; - - virtual void ReceiveGathers()=0; - virtual void ReceiveGatherVs()=0; - virtual void ReceivePointToPoint()=0; - virtual void ReceiveAllToAll()=0; - - virtual void WaitPointToPoint()=0; - virtual void WaitGathers()=0; - virtual void WaitGatherVs()=0; - virtual void WaitAllToAll()=0; - - // Interfaces exposing MPI_Datatype, not intended for client class use - virtual void RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type)=0; - virtual void RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type)=0; - - - /* - * Blocking gathers are implemented in MPI as a single call for both send/receive - * But, here we separate send and receive parts, since this interface may one day be used for - * nonblocking collectives. - */ - virtual void RequestGatherVSendImpl(void* buffer, int count, proc_t toRank, MPI_Datatype type)=0; - virtual void RequestGatherReceiveImpl(void* buffer, MPI_Datatype type)=0; - virtual void RequestGatherSendImpl(void* buffer, proc_t toRank, MPI_Datatype type)=0; - virtual void RequestGatherVReceiveImpl(void* buffer, int * displacements, int *counts, MPI_Datatype type)=0; - - - virtual void RequestAllToAllReceiveImpl(void * buffer,int count,MPI_Datatype type)=0; - virtual void RequestAllToAllSendImpl(void * buffer,int count,MPI_Datatype type)=0; - - std::vector & GetDisplacementsBuffer(); - std::vector & GetCountsBuffer(); - - const MpiCommunicator &communicator; - private: - /*** - * Buffers which can be used to store displacements and counts for cleaning up interfaces - * These will be cleaned up following a Wait/Dispatch - */ - std::vector > displacementsBuffer; - std::vector > countsBuffer; - }; - } -} -#endif // HEMELB_NET_NET_H diff --git a/Code/net/BuildInfo.h.in b/Code/net/BuildInfo.h.in deleted file mode 100644 index a4f2f295e..000000000 --- a/Code/net/BuildInfo.h.in +++ /dev/null @@ -1,24 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_BUILDINFO_H_IN -#define HEMELB_NET_BUILDINFO_H_IN -namespace hemelb -{ - namespace net - { - typedef @HEMELB_POINTPOINT_IMPLEMENTATION@PointPoint PointPointImpl ; - typedef @HEMELB_GATHERS_IMPLEMENTATION@Gathers GathersImpl ; - typedef @HEMELB_ALLTOALL_IMPLEMENTATION@AllToAll AllToAllImpl ; - #cmakedefine HEMELB_SEPARATE_CONCERNS - #ifdef HEMELB_SEPARATE_CONCERNS - static const bool separate_communications = true; - #else - static const bool separate_communications = false; - #endif - } -} -#endif // HEMELB_NET_BUILDINFO_H_IN \ No newline at end of file diff --git a/Code/net/CMakeLists.txt b/Code/net/CMakeLists.txt deleted file mode 100644 index 123dd9523..000000000 --- a/Code/net/CMakeLists.txt +++ /dev/null @@ -1,27 +0,0 @@ - -# This file is part of HemeLB and is Copyright (C) -# the HemeLB team and/or their institutions, as detailed in the -# file AUTHORS. This software is provided under the terms of the -# license in the file LICENSE. -add_library(hemelb_net - MpiDataType.cc MpiEnvironment.cc MpiError.cc - MpiCommunicator.cc MpiGroup.cc MpiFile.cc - IteratedAction.cc BaseNet.cc -IOCommunicator.cc -mixins/pointpoint/CoalescePointPoint.cc -mixins/pointpoint/SeparatedPointPoint.cc -mixins/pointpoint/ImmediatePointPoint.cc -mixins/gathers/SeparatedGathers.cc -mixins/gathers/ViaPointPointGathers.cc -mixins/alltoall/SeparatedAllToAll.cc -mixins/alltoall/ViaPointPointAllToAll.cc -mixins/StoringNet.cc ProcComms.cc -phased/StepManager.cc) -configure_file ( - "${PROJECT_SOURCE_DIR}/net/BuildInfo.h.in" - "${PROJECT_BINARY_DIR}/net/BuildInfo.h" - ) -configure_file ( - "${PROJECT_SOURCE_DIR}/net/MpiConstness.h.in" - "${PROJECT_BINARY_DIR}/net/MpiConstness.h" - ) \ No newline at end of file diff --git a/Code/net/IOCommunicator.cc b/Code/net/IOCommunicator.cc deleted file mode 100644 index a2f88c97f..000000000 --- a/Code/net/IOCommunicator.cc +++ /dev/null @@ -1,29 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/IOCommunicator.h" -#include "net/mpi.h" - -namespace hemelb -{ - namespace net - { - IOCommunicator::IOCommunicator(const MpiCommunicator& comm) : MpiCommunicator(comm) - { - } - - bool IOCommunicator::OnIORank() const - { - return Rank() == GetIORank(); - } - - int IOCommunicator::GetIORank() const - { - return 0; - } - - } -} diff --git a/Code/net/IOCommunicator.h b/Code/net/IOCommunicator.h deleted file mode 100644 index 2d0e57590..000000000 --- a/Code/net/IOCommunicator.h +++ /dev/null @@ -1,33 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_IOCOMMUNICATOR_H -#define HEMELB_NET_IOCOMMUNICATOR_H - -//#include -//#include -// -//#include "constants.h" -#include "net/MpiCommunicator.h" - -namespace hemelb -{ - namespace net - { - /** - * An MPI communicator with a special I/O rank. - */ - class IOCommunicator : public MpiCommunicator - { - public: - IOCommunicator(const MpiCommunicator& comm); - bool OnIORank() const; - int GetIORank() const; - }; - } -} - -#endif /* HEMELB_NET_IOCOMMUNICATOR_H */ diff --git a/Code/net/IteratedAction.cc b/Code/net/IteratedAction.cc deleted file mode 100644 index 33204099f..000000000 --- a/Code/net/IteratedAction.cc +++ /dev/null @@ -1,68 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/IteratedAction.h" -#include "net/phased/StepManager.h" - -namespace hemelb -{ - namespace net - { - IteratedAction::~IteratedAction() - { - - } - - bool IteratedAction::CallAction(int action) - { - switch (static_cast(action)) - { - case phased::steps::BeginPhase: - RequestComms(); - return true; - case phased::steps::PreSend: - PreSend(); - return true; - case phased::steps::PreWait: - PreReceive(); - return true; - case phased::steps::EndPhase: - PostReceive(); - return true; - case phased::steps::EndAll: - EndIteration(); - return true; - default: - return false; - } - } - - void IteratedAction::RequestComms() - { - - } - - void IteratedAction::PreSend() - { - - } - - void IteratedAction::PreReceive() - { - - } - - void IteratedAction::PostReceive() - { - - } - - void IteratedAction::EndIteration() - { - - } - } -} diff --git a/Code/net/IteratedAction.h b/Code/net/IteratedAction.h deleted file mode 100644 index 44c1e137c..000000000 --- a/Code/net/IteratedAction.h +++ /dev/null @@ -1,34 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_ITERATEDACTION_H -#define HEMELB_NET_ITERATEDACTION_H -#include "net/phased/Concern.h" -namespace hemelb -{ - namespace net - { - class IteratedAction : public phased::Concern - { - public: - /*** - * When an iterated actor is called through the phased::StepManager mechanism, - * this method dispatches to the individual step methods - * @param action Enumeration indicating the step - * @return True if an action was successfully called for the step - */ - bool CallAction(int action); - virtual ~IteratedAction(); - virtual void RequestComms(); - virtual void PreSend(); - virtual void PreReceive(); - virtual void PostReceive(); - virtual void EndIteration(); - }; - } -} - -#endif /* HEMELB_NET_ITERATEDACTION_H */ diff --git a/Code/net/MpiCommunicator.cc b/Code/net/MpiCommunicator.cc deleted file mode 100644 index 5405366df..000000000 --- a/Code/net/MpiCommunicator.cc +++ /dev/null @@ -1,115 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/MpiCommunicator.h" -#include "net/MpiGroup.h" - -namespace hemelb -{ - namespace net - { - namespace - { - void Deleter(MPI_Comm* comm) - { - int finalized; - HEMELB_MPI_CALL(MPI_Finalized, (&finalized)); - if (!finalized) - HEMELB_MPI_CALL(MPI_Comm_free, (comm)); - delete comm; - } - } - - MpiCommunicator MpiCommunicator::World() - { - return MpiCommunicator(MPI_COMM_WORLD, false); - } - - MpiCommunicator::MpiCommunicator() : commPtr() - { - } - - MpiCommunicator::MpiCommunicator(MPI_Comm communicator, bool owner) : commPtr() - { - if (communicator == MPI_COMM_NULL) - return; - - if (owner) - { - commPtr.reset(new MPI_Comm(communicator), Deleter); - } - else - { - commPtr.reset(new MPI_Comm(communicator)); - } - } - - MpiCommunicator::~MpiCommunicator() - { - } - - bool operator==(const MpiCommunicator& comm1, const MpiCommunicator& comm2) - { - if (comm1) - { - if (comm2) - { - int result; - HEMELB_MPI_CALL(MPI_Comm_compare, - (comm1, comm2, &result)); - return result == MPI_IDENT; - } - return false; - } - return (!comm2); - } - - bool operator!=(const MpiCommunicator& comm1, const MpiCommunicator& comm2) - { - return ! (comm1 == comm2); - } - - int MpiCommunicator::Rank() const - { - int rank; - HEMELB_MPI_CALL(MPI_Comm_rank, (*commPtr, &rank)); - return rank; - } - - int MpiCommunicator::Size() const - { - int size; - HEMELB_MPI_CALL(MPI_Comm_size, (*commPtr, &size)); - return size; - } - - MpiGroup MpiCommunicator::Group() const - { - MPI_Group grp; - HEMELB_MPI_CALL(MPI_Comm_group, (*commPtr, &grp)); - return MpiGroup(grp, true); - } - - MpiCommunicator MpiCommunicator::Create(const MpiGroup& grp) const - { - MPI_Comm newComm; - HEMELB_MPI_CALL(MPI_Comm_create, (*commPtr, grp, &newComm)); - return MpiCommunicator(newComm, true); - } - - void MpiCommunicator::Abort(int errCode) const - { - HEMELB_MPI_CALL(MPI_Abort, (*commPtr, errCode)); - } - - MpiCommunicator MpiCommunicator::Duplicate() const - { - MPI_Comm newComm; - HEMELB_MPI_CALL(MPI_Comm_dup, (*commPtr, &newComm)); - return MpiCommunicator(newComm, true); - } - } -} diff --git a/Code/net/MpiCommunicator.h b/Code/net/MpiCommunicator.h deleted file mode 100644 index 4ec4595be..000000000 --- a/Code/net/MpiCommunicator.h +++ /dev/null @@ -1,143 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MPICOMMUNICATOR_H -#define HEMELB_NET_MPICOMMUNICATOR_H - -//#include "units.h" -//#include "net/mpi.h" -#include -#include "net/MpiError.h" -#include - -namespace hemelb -{ - namespace net - { - class MpiGroup; - - class MpiCommunicator - { - public: - static MpiCommunicator World(); - - /** - * Constructor for an uninitialised communicator, equivalent to - * MPI_COMM_NULL - * @param communicator - */ - MpiCommunicator(); - - /** - * Class has virtual methods so should have virtual d'tor. - */ - virtual ~MpiCommunicator(); - - /** - * Returns the local rank on the communicator - * @return - */ - virtual int Rank() const; - - /** - * Returns the size of the communicator (i.e. total number of procs involved). - * @return - */ - virtual int Size() const; - - /** - * Creates a new communicator - see MPI_COMM_GROUP - * @param Group which is a subset of the group of this communicator. - * @return New communicator. - */ - MpiCommunicator Create(const MpiGroup& grp) const; - - /** - * Allow implicit casts to MPI_Comm - * @return The underlying MPI communicator. - */ - operator MPI_Comm() const - { - return *commPtr; - } - /** - * Is this communicator valid? I.e. not equal to MPI_COMM_NULL. - */ - operator bool() const - { - return (bool)commPtr; - } - /** - * Returns the MPI group being used. - * @return - */ - MpiGroup Group() const; - - /** - * Abort - should try to bring down all tasks, but no guarantees - * @param errCode - */ - void Abort(int errCode) const; - - /** - * Duplicate the communicator - see MPI_COMM_DUP - * @return - */ - MpiCommunicator Duplicate() const; - - template - void Broadcast(T& val, const int root) const; - template - void Broadcast(std::vector& vals, const int root) const; - - template - T AllReduce(const T& val, const MPI_Op& op) const; - template - std::vector AllReduce(const std::vector& vals, const MPI_Op& op) const; - - template - T Reduce(const T& val, const MPI_Op& op, const int root) const; - template - std::vector Reduce(const std::vector& vals, const MPI_Op& op, const int root) const; - - template - std::vector Gather(const T& val, const int root) const; - - template - std::vector AllGather(const T& val) const; - - template - std::vector AllToAll(const std::vector& vals) const; - - template - void Send(const T& val, int dest, int tag=0) const; - template - void Send(const std::vector& val, int dest, int tag=0) const; - - template - void Receive(T& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; - template - void Receive(std::vector& val, int src, int tag=0, MPI_Status* stat=MPI_STATUS_IGNORE) const; - - protected: - /** - * Constructor to get data needed from an MPI communicator - * @param communicator - */ - MpiCommunicator(MPI_Comm communicator, bool willOwn); - - boost::shared_ptr commPtr; - }; - - bool operator==(const MpiCommunicator& comm1, const MpiCommunicator& comm2); - bool operator!=(const MpiCommunicator& comm1, const MpiCommunicator& comm2); - - } -} - -#include "net/MpiCommunicator.hpp" - -#endif /* HEMELB_NET_MPICOMMUNICATOR_H */ diff --git a/Code/net/MpiCommunicator.hpp b/Code/net/MpiCommunicator.hpp deleted file mode 100644 index 2aac317bb..000000000 --- a/Code/net/MpiCommunicator.hpp +++ /dev/null @@ -1,172 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. -#ifndef HEMELB_NET_MPICOMMUNICATOR_HPP -#define HEMELB_NET_MPICOMMUNICATOR_HPP - -#include "net/MpiDataType.h" -#include "net/MpiConstness.h" - -namespace hemelb -{ - namespace net - { - template - void MpiCommunicator::Broadcast(T& val, const int root) const - { - HEMELB_MPI_CALL( - MPI_Bcast, - (&val, 1, MpiDataType(), root, *this) - ); - } - template - void MpiCommunicator::Broadcast(std::vector& vals, const int root) const - { - HEMELB_MPI_CALL( - MPI_Bcast, - (&vals[0], vals.size(), MpiDataType(), root, *this) - ); - } - - template - T MpiCommunicator::AllReduce(const T& val, const MPI_Op& op) const - { - T ans; - HEMELB_MPI_CALL( - MPI_Allreduce, - (MpiConstCast(&val), &ans, 1, MpiDataType(), op, *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::AllReduce(const std::vector& vals, const MPI_Op& op) const - { - std::vector ans(vals.size()); - HEMELB_MPI_CALL( - MPI_Allreduce, - (MpiConstCast(&vals[0]), &ans[0], vals.size(), MpiDataType(), op, *this) - ); - return ans; - } - - template - T MpiCommunicator::Reduce(const T& val, const MPI_Op& op, const int root) const - { - T ans; - HEMELB_MPI_CALL( - MPI_Reduce, - (MpiConstCast(&val), &ans, 1, MpiDataType(), op, root, *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::Reduce(const std::vector& vals, const MPI_Op& op, - const int root) const - { - std::vector ans; - T* recvbuf = NULL; - - if (Rank() == root) - { - // Standard says the address of receive buffer only matters at the root. - ans.resize(vals.size()); - recvbuf = &ans[0]; - } - - HEMELB_MPI_CALL( - MPI_Reduce, - (MpiConstCast(&vals[0]), recvbuf, vals.size(), MpiDataType(), op, root, *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::Gather(const T& val, const int root) const - { - std::vector ans; - T* recvbuf = NULL; - - if (Rank() == root) - { - // Standard says the address of receive buffer only matters at the root. - ans.resize(Size()); - recvbuf = &ans[0]; - } - HEMELB_MPI_CALL( - MPI_Gather, - (MpiConstCast(&val), 1, MpiDataType(), - recvbuf, 1, MpiDataType(), - root, *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::AllGather(const T& val) const - { - std::vector ans(Size()); - T* recvbuf = &ans[0]; - - HEMELB_MPI_CALL( - MPI_Allgather, - (MpiConstCast(&val), 1, MpiDataType(), - recvbuf, 1, MpiDataType(), - *this) - ); - return ans; - } - - template - std::vector MpiCommunicator::AllToAll(const std::vector& vals) const - { - std::vector ans(vals.size()); - HEMELB_MPI_CALL( - MPI_Alltoall, - (MpiConstCast(&vals[0]), 1, MpiDataType(), - &ans[0], 1, MpiDataType(), - *this) - ); - return ans; - } - - template - void MpiCommunicator::Send(const T& val, int dest, int tag) const - { - HEMELB_MPI_CALL( - MPI_Send, - (MpiConstCast(&val), 1, MpiDataType(), dest, tag, *this) - ); - } - template - void MpiCommunicator::Send(const std::vector& vals, int dest, int tag) const - { - HEMELB_MPI_CALL( - MPI_Send, - (MpiConstCast(&vals[0]), vals.size(), MpiDataType(), dest, tag, *this) - ); - } - - template - void MpiCommunicator::Receive(T& val, int src, int tag, MPI_Status* stat) const - { - HEMELB_MPI_CALL( - MPI_Recv, - (&val, 1, MpiDataType(), src, tag, *this, stat) - ); - } - template - void MpiCommunicator::Receive(std::vector& vals, int src, int tag, MPI_Status* stat) const - { - HEMELB_MPI_CALL( - MPI_Recv, - (&vals, vals.size(), MpiDataType(), src, tag, *this, stat) - ); - } - } -} - -#endif // HEMELB_NET_MPICOMMUNICATOR_HPP diff --git a/Code/net/MpiConstness.h.in b/Code/net/MpiConstness.h.in deleted file mode 100644 index ff1a237a1..000000000 --- a/Code/net/MpiConstness.h.in +++ /dev/null @@ -1,30 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MPICONSTNESS_H_IN -#define HEMELB_NET_MPICONSTNESS_H_IN -namespace hemelb -{ - namespace net - { -#cmakedefine HAVE_CONSTCORRECTMPI - -#ifdef HAVE_CONSTCORRECTMPI - template - const T* MpiConstCast(const T* ptr) - { - return ptr; - } -#else - template - T* MpiConstCast(const T* ptr) - { - return const_cast(ptr); - } -#endif - } -} -#endif // HEMELB_NET_MPICONSTNESS_H_IN \ No newline at end of file diff --git a/Code/net/PhasedBroadcast.h b/Code/net/PhasedBroadcast.h deleted file mode 100644 index cd4bed7c9..000000000 --- a/Code/net/PhasedBroadcast.h +++ /dev/null @@ -1,407 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASEDBROADCAST_H -#define HEMELB_NET_PHASEDBROADCAST_H - -#include - -#include "debug/Debugger.h" -#include "net/IteratedAction.h" -#include "net/net.h" -#include "lb/SimulationState.h" -#include "net/IOCommunicator.h" - -namespace hemelb -{ - namespace net - { - /* - * PROGRAMME - * --------- - * - * For 0-indexed cycle counts and 0-indexed depths, tree depth = N. - * - * Iterations 0 to (N-1) are nodes at depth (it) passing down to nodes at depth (it + 1). - * Action happens on all nodes at the end of iteration (N-1). - * Iterations N to (2N-1) are nodes at depth (2N - it) passing up to nodes at depth (2N - it - 1). - */ - - /** - * PhasedBroadcast - a class to control the general process of communication between - * all nodes over multiple iterations. By using a common interface, we can ensure that - * communication happens asynchronously at a single point in each iteration - only one - * communication from any node to any other node will be performed, giving efficient - * performance. - * - * Communication uses a tree structure with a single top-level node, and a constant number of - * children per node down the tree. All nodes at the same depth communicate on the same - * iteration with nodes either above or below them (depending on the position through the - * communication programme defined above). The class supports actions that have to be performed - * before any communication begins, followed by potentially multiple, overlapping communication - * stages between related nodes in each consecutive pair of depths in the tree. Communication - * can be up the tree towards the topmost node, down the tree (beginning at the topmost node) - * or both (down the tree then up). An action to be performed on all nodes, when communication - * from top to bottom of the tree has been completed, is also supported. - * - * The class is called via the IteratedAction interface. Classes that use this interface should - * derive from either PhasedBroadcastRegular (for communication that is to happen at regular - * intervals) or PhasedBroadcastIrregular (for communication that is to happen at irregular - * intervals). The derived class should override one or more virtual functions in the chosen - * base class, and should perform communication using the templated - * [SendTo|ReceiveFrom][Children|Parent] functions defined in this class. - * - * This class is made general using template parameters: - * - * initialAction = if true, an extra iteration occurs at the start of each broadcast cycle - * splay = the number of consecutive iterations communication between a pair of nodes needs to - * go on for. Useful if the passed data is an array of variable length; one node can spend an - * iteration telling the other how many elements will be passed then the next iteration - * sending them. - * ovrlp = the number of iterations where a node both receives and sends. overlap <= splay. - * down = true if parents communicate to their child nodes in the pattern. - * up = true if children communicate to their parent nodes in the pattern. - */ - template - class PhasedBroadcast : public IteratedAction - { - public: - PhasedBroadcast(Net * iNet, - const lb::SimulationState * iSimState, - unsigned int spreadFactor) : - mSimState(iSimState), mMyDepth(0), mTreeDepth(0), mNet(iNet) - { - // Calculate the correct values for the depth variables. - proc_t noSeenToThisDepth = 1; - proc_t noAtCurrentDepth = 1; - - const MpiCommunicator& netTop = mNet->GetCommunicator(); - - while (noSeenToThisDepth < netTop.Size()) - { - // Go down a level. I.e. increase the depth of the tree, to a new level which has M times - // as many nodes on it. - ++mTreeDepth; - noAtCurrentDepth *= spreadFactor; - noSeenToThisDepth += noAtCurrentDepth; - - // If this node is at the current depth, it must have a rank lower than or equal to the highest - // rank at the current depth but greater than the highest rank at the previous depth. - if (noSeenToThisDepth > netTop.Rank() && ( (noSeenToThisDepth - - noAtCurrentDepth) <= netTop.Rank())) - { - mMyDepth = mTreeDepth; - } - } - - // In a M-tree, with a root of 0, each node N's parent has rank floor((N-1) / M) - if (netTop.Rank() == 0) - { - mParent = NOPARENT; - } - else - { - mParent = (netTop.Rank() - 1) / spreadFactor; - } - - // The children of a node N in a M-tree with root 0 are those in the range (M*N)+1,...,(M*N) + M - for (unsigned int child = (spreadFactor * netTop.Rank()) + 1; child - <= spreadFactor * (1 + netTop.Rank()); ++child) - { - if (child < (unsigned int) netTop.Size()) - { - mChildren.push_back(child); - } - } - } - - protected: - - const lb::SimulationState * mSimState; - - /** - * Helper function for sending data to child nodes. - */ - template - void SendToChildren(T* data, int count) - { - for (std::vector::iterator it = mChildren.begin(); it != mChildren.end(); ++it) - { - mNet->RequestSend (data, count, *it); - } - } - - /** - * Helper function for receiving data from parent nodes. - */ - template - void ReceiveFromParent(T* data, int count) - { - if (mParent != NOPARENT) - { - mNet ->RequestReceive (data, count, mParent); - } - } - - /** - * Receives data from each child. This is a set length per child, and each child's - * data is inserted contiguously into the provided array. - * - * The user must handle the case where the number of children is smaller than the - * spread factor and the array pointed by dataStart is not completely filled in. - * - * @param dataStart Pointer to the start of the array. - * @param countPerChild Number of elements to receive per child. - */ - template - void ReceiveFromChildren(T* dataStart, int countPerChild) - { - T* data = dataStart; - for (std::vector::iterator it = mChildren.begin(); it != mChildren.end(); ++it) - { - mNet->RequestReceive (data, countPerChild, *it); - data += countPerChild; - } - } - - /** - * Receives data from each child. This is a variable length per child, and each child's - * data is inserted into the appropriate array. - * - * The user must handle the case where the number of children is smaller than the - * spread factor and the array pointed by dataStart is not completely filled in. - * - * @param dataStart Pointer to the start of the array. - * @param countPerChild Number of elements to receive per child. - */ - template - void ReceiveFromChildren(T** dataStart, unsigned int* countPerChild) - { - unsigned int childNum = 0; - for (std::vector::iterator it = mChildren.begin(); it != mChildren.end(); ++it) - { - mNet->RequestReceive (dataStart[childNum], countPerChild[childNum], *it); - ++childNum; - } - } - - /** - * Helper function for sending data to parent nodes. - */ - template - void SendToParent(T* data, int count) - { - if (mParent != NOPARENT) - { - mNet->RequestSend (data, count, mParent); - } - } - - /** - * Returns the total number of iterations spent doing a complete traversal -- all the way - * down the tree and back up again. - * - * @return - */ - unsigned long GetRoundTripLength() const - { - unsigned long delayTime = initialAction - ? 1 - : 0; - - unsigned long multiplier = (down - ? 1 - : 0) + (up - ? 1 - : 0); - - return delayTime + multiplier * GetTraverseTime(); - } - - /** - * Get the index of the iteration when we first start descending the tree. - * - * @return - */ - unsigned long GetFirstDescending() const - { - return (initialAction - ? 1 - : 0); - } - - /** - * Get the index of the iteration when we first start ascending the tree. - * - * @return - */ - unsigned long GetFirstAscending() const - { - return GetFirstDescending() + (down - ? GetTraverseTime() - : 0); - } - - /** - * Get the 0-indexed depth of this rank. - * - * @return - */ - unsigned long GetMyDepth() const - { - return mMyDepth; - } - - /** - * Get the 0-indexed depth of the whole tree. - * - * @return - */ - unsigned long GetTreeDepth() const - { - return mTreeDepth; - } - - /** - * Get the number of iterations required for a half-cycle (messages going either all the - * way up the tree or all the way down) - * - * @return - */ - unsigned long GetTraverseTime() const - { - return mTreeDepth * (splay - ovrlp) + ovrlp; - } - - /** - * Gets the overlap value for a send to a parent node given the number of iterations - * through the 'upwards' phase. Returns true if this node should send to is parent. - */ - bool GetSendParentOverlap(unsigned long subCycleNumber, unsigned long* sendOverlap) - { - // The first cycle we're sending to parents. - unsigned long firstSendCycle = (mTreeDepth - mMyDepth) * (splay - ovrlp); - - // If we're either not far enough or too far through the cycle, don't send. - if (subCycleNumber < firstSendCycle || subCycleNumber >= (firstSendCycle + splay)) - { - return false; - } - // Otherwise calculate the overlap value and return true. - else - { - *sendOverlap = subCycleNumber - firstSendCycle; - return true; - } - } - - /** - * Gets the overlap value for a send to child nodes given the number of iterations - * through the 'downwards' phase. Returns true if this node should send to is children. - */ - bool GetSendChildrenOverlap(unsigned long subCycleNumber, unsigned long* sendOverlap) - { - // The first cycle we're sending to children. - unsigned long firstSendCycle = mMyDepth * (splay - ovrlp); - - // If we're either not far enough or too far through the cycle, don't send. - if (subCycleNumber < firstSendCycle || subCycleNumber >= (firstSendCycle + splay)) - { - return false; - } - // Otherwise calculate the overlap value and return true. - else - { - *sendOverlap = subCycleNumber - firstSendCycle; - return true; - } - } - - /** - * Gets the overlap value for a receive from a parent node given the number of iterations - * through the 'downwards' phase. Returns true if this node should receive from its parent. - */ - bool GetReceiveParentOverlap(unsigned long subCycleNumber, unsigned long* receiveOverlap) - { - // The first cycle we're receiving from parents. - unsigned long firstReceiveCycle = (mMyDepth - 1) * (splay - ovrlp); - - // If we're either not far enough or too far through the cycle, don't receive. - if (subCycleNumber < firstReceiveCycle || subCycleNumber >= (firstReceiveCycle + splay)) - { - return false; - } - // Otherwise calculate the overlap value and return true. - else - { - *receiveOverlap = subCycleNumber - firstReceiveCycle; - return true; - } - } - - /** - * Gets the overlap value for a receive from child node given the number of iterations - * through the 'upwards' phase. Returns true if this node should receive from its children. - */ - bool GetReceiveChildrenOverlap(unsigned long subCycleNumber, unsigned long* receiveOverlap) - { - // The first cycle we're receiving from parents. - unsigned long firstReceiveCycle = (mTreeDepth - (mMyDepth + 1)) * (splay - ovrlp); - - // If we're either not far enough or too far through the cycle, don't receive. - if (subCycleNumber < firstReceiveCycle || subCycleNumber >= (firstReceiveCycle + splay)) - { - return false; - } - // Otherwise calculate the overlap value and return true. - else - { - *receiveOverlap = subCycleNumber - firstReceiveCycle; - return true; - } - } - - /** - * Returns the number of the parent node. - * @return - */ - int GetParent() const - { - return mParent; - } - - const std::vector& GetChildren() const - { - return mChildren; - } - - protected: - static const int NOPARENT = -1; - - private: - /** - * Note that depths are 0-indexed. I.e. a tree with a single node has - * a depth of zero, and the node itself is considered to be at depth 0. - */ - unsigned int mMyDepth; - unsigned int mTreeDepth; - - /** - * This node's parent rank. - */ - int mParent; - /** - * This node's child ranks. - */ - std::vector mChildren; - protected: - Net * mNet; - }; - } -} - -#endif /* HEMELB_NET_PHASEDBROADCAST_H */ diff --git a/Code/net/PhasedBroadcastIrregular.h b/Code/net/PhasedBroadcastIrregular.h deleted file mode 100644 index 26c2093cf..000000000 --- a/Code/net/PhasedBroadcastIrregular.h +++ /dev/null @@ -1,393 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASEDBROADCASTIRREGULAR_H -#define HEMELB_NET_PHASEDBROADCASTIRREGULAR_H - -#include - -#include "net/PhasedBroadcast.h" - -namespace hemelb -{ - namespace net - { - /** - * PhasedBroadcastIrregular - a class for performing phased broadcasts that are not restricted - * to starting at regular intervals. A longer description is given in PhasedBroadcast.h. - */ - template - class PhasedBroadcastIrregular : public PhasedBroadcast - { - // Typedef for the base class's type - typedef PhasedBroadcast base; - typedef std::list storeType; - - public: - /** - * Constructor that initialises the base class. - * - * @param iNet - * @param iSimState - * @param spreadFactor - * @return - */ - PhasedBroadcastIrregular(Net * iNet, - const lb::SimulationState * iSimState, - unsigned int spreadFactor) : - base(iNet, iSimState, spreadFactor) - { - performInstantBroadcast = false; - } - - /** - * Request the broadcasting to start on this iteration. Returns the number of the iteration - * on which it will complete. - * - * @return - */ - unsigned long Start() - { - unsigned long currentTimeStep = base::mSimState->GetTimeStep(); - unsigned long finishTime = currentTimeStep + base::GetRoundTripLength() - 1; - - // If it's going to take longer than is available for the simulation, assume the - // implementing class will do its thing instantaneously this iteration. - if (finishTime > base::mSimState->GetTotalTimeSteps()) - { - performInstantBroadcast = true; - return currentTimeStep; - } - else - { - if (startIterations.empty() || startIterations.back() != currentTimeStep) - { - startIterations.push_back(currentTimeStep); - } - - return finishTime; - } - } - - /** - * Function that requests all the communications from the Net object. - */ - void RequestComms() - { - const unsigned long currentIt = base::mSimState->GetTimeStep(); - const unsigned long firstAscent = base::GetFirstAscending(); - const unsigned long firstDescent = base::GetFirstDescending(); - - for (storeType::const_iterator it = startIterations.begin(); it != startIterations.end(); it++) - { - unsigned long progress = currentIt - *it; - - // Next, deal with the case of a cycle with an initial pass down the tree. - if (down) - { - if (progress >= firstDescent && progress < firstAscent) - { - unsigned long sendOverlap; - unsigned long receiveOverlap; - - if (base::GetSendChildrenOverlap(progress - firstDescent, &sendOverlap) - && !base::GetChildren().empty()) - { - ProgressToChildren(*it, sendOverlap); - } - - if (base::GetReceiveParentOverlap(progress - firstDescent, &receiveOverlap) - && base::GetParent() >= 0) - { - ProgressFromParent(*it, receiveOverlap); - } - } - } - - // Now deal with the case of a pass up the tree. - if (up) - { - if (progress >= firstAscent) - { - unsigned long sendOverlap; - unsigned long receiveOverlap; - - if (base::GetSendParentOverlap(progress - firstAscent, &sendOverlap) - && base::GetParent() >= 0) - { - ProgressToParent(*it, sendOverlap); - } - - if (base::GetReceiveChildrenOverlap(progress - firstAscent, &receiveOverlap) - && !base::GetChildren().empty()) - { - ProgressFromChildren(*it, receiveOverlap); - } - } - } - } - - unsigned long searchValue = currentIt - (base::GetRoundTripLength() + 1); - - // Use this time to clear out the array. We do this once every iteration so it - // suffices to get rid of one value. - for (storeType::iterator it = startIterations.begin(); it != startIterations.end(); it++) - { - if (*it == searchValue) - { - ClearOut(searchValue); - startIterations.erase(it); - break; - } - } - } - - /** - * Function that acts while waiting for data to be received. This performs the initial - * action on the first iteration only. - */ - void PreReceive() - { - // The only thing to do while waiting is the initial action. - if (initAction) - { - if (IsInitialAction()) - { - InitialAction(base::mSimState->GetTimeStep()); - } - } - } - - /** - * Function to be called after the Receives have completed, where the - * data is used. - */ - void PostReceive() - { - const unsigned long firstAscent = base::GetFirstAscending(); - const unsigned long traversalLength = base::GetTraverseTime(); - const unsigned long cycleLength = base::GetRoundTripLength(); - const unsigned long currentIt = base::mSimState->GetTimeStep(); - - for (storeType::const_iterator it = startIterations.begin(); it != startIterations.end(); it++) - { - const unsigned long progress = currentIt - *it; - - // Deal with the case of a cycle with an initial pass down the tree. - if (down) - { - const unsigned long firstDescent = base::GetFirstDescending(); - - if (progress >= firstDescent && progress < firstAscent) - { - unsigned long receiveOverlap; - - if (base::GetReceiveParentOverlap(progress - firstDescent, &receiveOverlap)) - { - PostReceiveFromParent(*it, receiveOverlap); - } - - // If we're halfway through the programme, all top-down changes have occurred and - // can be applied on all nodes at once safely. - if (progress == (traversalLength - 1)) - { - Effect(currentIt); - } - } - } - - // Deal with the case of a pass up the tree. - if (up) - { - if (progress >= firstAscent && progress < cycleLength) - { - unsigned long receiveOverlap; - - if (base::GetReceiveChildrenOverlap(progress - firstAscent, &receiveOverlap) - && !base::GetChildren().empty()) - { - PostReceiveFromChildren(*it, receiveOverlap); - } - - unsigned long sendOverlap; - - if (base::GetSendParentOverlap(progress - firstAscent, &sendOverlap) - && base::GetParent() >= 0) - { - PostSendToParent(*it, sendOverlap); - } - } - } - - // If this node is the root of the tree and we've just finished the upwards half, it - // must act. - if (progress == (base::GetRoundTripLength() - 1) - && this->mNet->Rank() == 0) - { - TopNodeAction(*it); - } - } - - if (performInstantBroadcast) - { - InstantBroadcast(currentIt); - performInstantBroadcast = false; - } - } - - protected: - /** - * Returns true if we are performing an instant broadcast on the current iteration. - * - * @return - */ - bool IsInstantBroadcast() const - { - return performInstantBroadcast; - } - - /** - * Returns true if we are performing the initial action for a phased broadcast on - * the current iteration. - * - * @return - */ - bool IsInitialAction() const - { - for (storeType::const_iterator it = startIterations.begin(); it != startIterations.end(); it++) - { - if (*it == base::mSimState->GetTimeStep()) - { - return true; - } - } - - return false; - } - - /** - * Overridable function for the initial action performed by a node at the beginning of the - * cycle. Only has an effect if the template paramter initialAction is true. - */ - virtual void InitialAction(unsigned long startIteration) - { - - } - - /** - * Overridable function for when a node has to receive from its children in the tree. - * - * Use ReceiveFromChildren to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressFromChildren(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to receive from its parent in the tree. - * - * Use ReceiveFromParent to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressFromParent(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to send to its children in the tree. - * - * Use SendToChildren to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressToChildren(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to send to its parent in the tree. - * - * Use SendToParent to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressToParent(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Overridable function, called by a node after data has been received from its children. - * The parameter splayNumber is 0 indexed and less than splay. - */ - virtual void PostReceiveFromChildren(unsigned long startIteration, - unsigned long splayNumber) - { - - } - - /** - * Overridable function, called by a node after data has been received from its parent. The - * parameter splayNumber is 0 indexed and less than splay. - */ - virtual void PostReceiveFromParent(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Overridable function, called by a node after data has been sent to its parent. - * @param startIteration The iteration on which this phased send began. - * @param splayNumber The number of steps we have passed through this phase of the sending. - */ - virtual void PostSendToParent(unsigned long startIteration, unsigned long splayNumber) - { - - } - - /** - * Action taken when upwards-travelling data reaches the top node. - */ - virtual void TopNodeAction(unsigned long startIteration) - { - - } - - /** - * Action taken by all nodes when downwards-travelling data has been sent to every node. - */ - virtual void Effect(unsigned long startIteration) - { - - } - - /** - * For the case where we don't have enough iterations for a phased broadcast to complete. - */ - virtual void InstantBroadcast(unsigned long startIteration) - { - - } - - /** - * For clearing out the results of an iteration after completion. - */ - virtual void ClearOut(unsigned long startIteration) - { - - } - - private: - storeType startIterations; - bool performInstantBroadcast; - }; - } -} - -#endif /* HEMELB_NET_PHASEDBROADCASTIRREGULAR_H */ diff --git a/Code/net/PhasedBroadcastRegular.h b/Code/net/PhasedBroadcastRegular.h deleted file mode 100644 index 4cd603541..000000000 --- a/Code/net/PhasedBroadcastRegular.h +++ /dev/null @@ -1,293 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASEDBROADCASTREGULAR_H -#define HEMELB_NET_PHASEDBROADCASTREGULAR_H - -#include "net/PhasedBroadcast.h" - -namespace hemelb -{ - namespace net - { - /** - * PhasedBroadcastRegular - a class for performing phased broadcasts starting at regular - * intervals. A longer description is given in PhasedBroadcast.h. - */ - template - class PhasedBroadcastRegular : public PhasedBroadcast - { - public: - /** - * Constructor that calls the base class's constructor. - * - * @param iNet - * @param iSimState - * @param spreadFactor - * @return - */ - PhasedBroadcastRegular(Net * iNet, const lb::SimulationState * iSimState, unsigned int spreadFactor) : - base(iNet, iSimState, spreadFactor) - { - - } - - /** - * Function that requests all the communications from the Net object. - */ - void RequestComms() - { - const unsigned long iCycleNumber = Get0IndexedIterationNumber(); - const unsigned long firstAscent = base::GetFirstAscending(); - const unsigned long firstDescent = base::GetFirstDescending(); - - // Nothing to do for initial action case. - - // Next, deal with the case of a cycle with an initial pass down the tree. - if (goDown) - { - if (iCycleNumber >= firstDescent && iCycleNumber < firstAscent) - { - unsigned long sendOverlap; - unsigned long receiveOverlap; - - if (base::GetSendChildrenOverlap(iCycleNumber - firstDescent, &sendOverlap)) - { - ProgressToChildren(sendOverlap); - } - - if (base::GetReceiveParentOverlap(iCycleNumber - firstDescent, &receiveOverlap)) - { - ProgressFromParent(receiveOverlap); - } - } - } - - // And deal with the case of a cycle with a pass up the tree. - if (goUp) - { - if (iCycleNumber >= firstAscent) - { - unsigned long sendOverlap; - unsigned long receiveOverlap; - - if (base::GetSendParentOverlap(iCycleNumber - firstAscent, &sendOverlap)) - { - ProgressToParent(sendOverlap); - } - - if (base::GetReceiveChildrenOverlap(iCycleNumber - firstAscent, &receiveOverlap)) - { - ProgressFromChildren(receiveOverlap); - } - } - } - } - - /** - * Function called after send begin but before receives are known to be completed. The - * action performed is limited to the InitialAction, if used. - */ - void PreReceive() - { - // The only thing to do while waiting is the initial action. - if (initialAction) - { - if (Get0IndexedIterationNumber() == 0) - { - InitialAction(); - } - } - } - - /** - * Function to be called after the Receives have completed, where the - * data is used. - */ - void PostReceive() - { - const unsigned long iCycleNumber = Get0IndexedIterationNumber(); - const unsigned long firstAscent = - PhasedBroadcast::GetFirstAscending(); - const unsigned long traversalLength = - PhasedBroadcast::GetTraverseTime(); - - // Deal with the case of a cycle with an initial pass down the tree. - if (goDown) - { - const unsigned long firstDescent = - PhasedBroadcast::GetFirstDescending(); - - if (iCycleNumber >= firstDescent && iCycleNumber < firstAscent) - { - unsigned long receiveOverlap; - - if (base::GetReceiveParentOverlap(iCycleNumber - firstDescent, &receiveOverlap)) - { - PostReceiveFromParent(receiveOverlap); - } - - // If we're halfway through the programme, all top-down changes have occurred and - // can be applied on all nodes at once safely. - if ( (iCycleNumber - firstDescent) == (traversalLength - 1)) - { - Effect(); - } - } - } - - // Deal with the case of a cycle with a pass back up the tree. - if (goUp) - { - if (iCycleNumber >= firstAscent) - { - unsigned long receiveOverlap, sendOverlap; - - if (base::GetReceiveChildrenOverlap(iCycleNumber - firstAscent, &receiveOverlap)) - { - PostReceiveFromChildren(receiveOverlap); - } - - if (base::GetSendParentOverlap(iCycleNumber - firstAscent, &sendOverlap)) - { - PostSendToParent(sendOverlap); - } - - } - } - - // If this node is the root of the tree and we've just finished the upwards half, it - // must act. - if (iCycleNumber == (base::GetRoundTripLength() - 1) - && this->mNet->GetCommunicator().Rank() == 0) - { - TopNodeAction(); - } - } - - /** - * Returns the number of the iteration, as an integer between inclusive-0 and - * exclusive-2 * (the tree depth) - */ - unsigned long Get0IndexedIterationNumber() const - { - if (base::GetTreeDepth() > 0) - { - unsigned long stepsPassed = base::mSimState->Get0IndexedTimeStep(); - - return stepsPassed % base::GetRoundTripLength(); - } - else - { - return 0; - } - } - - protected: - - // Typedef for the base class type, for convenience. - typedef PhasedBroadcast base; - - /** - * Overridable function for the initial action performed by a node at the beginning of the - * cycle. Only has an effect if the template paramter initialAction is true. - */ - virtual void InitialAction() - { - - } - - /** - * Overridable function for when a node has to receive from its children in the tree. - * - * Use ReceiveFromChildren to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressFromChildren(unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to receive from its parent in the tree. - * - * Use ReceiveFromParent to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressFromParent(unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to send to its children in the tree. - * - * Use SendToChildren to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressToChildren(unsigned long splayNumber) - { - - } - - /** - * Overridable function for when a node has to send to its parent in the tree. - * - * Use SendToParent to do this. The parameter splayNumber is 0 indexed and less - * than splay. - */ - virtual void ProgressToParent(unsigned long splayNumber) - { - - } - - /** - * Overridable function, called by a node after data has been received from its children. - * The parameter splayNumber is 0 indexed and less than splay. - */ - virtual void PostReceiveFromChildren(unsigned long splayNumber) - { - - } - - /** - * Overridable function, called by a node after data has been sent to its parent. - * - * @param splayNumber The parameter splayNumber is 0 indexed and less than splay. - */ - virtual void PostSendToParent(unsigned long splayNumber) - { - } - - /** - * Overridable function, called by a node after data has been received from its parent. The - * parameter splayNumber is 0 indexed and less than splay. - */ - virtual void PostReceiveFromParent(unsigned long splayNumber) - { - - } - - /** - * Action taken when upwards-travelling data reaches the top node. - */ - virtual void TopNodeAction() - { - - } - - /** - * Action taken by all nodes when downwards-travelling data has been sent to every node. - */ - virtual void Effect() - { - - } - }; - } -} - -#endif /* HEMELB_NET_PHASEDBROADCASTREGULAR_H */ diff --git a/Code/net/ProcComms.cc b/Code/net/ProcComms.cc deleted file mode 100644 index 63639ff1d..000000000 --- a/Code/net/ProcComms.cc +++ /dev/null @@ -1,37 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/ProcComms.h" -namespace hemelb -{ - namespace net - { - void ProcComms::CreateMPIType() - { - std::vector displacements(size()); - std::vector lengths; - std::vector types; - - int location = 0; - - MPI_Aint offset; - MPI_Get_address(front().Pointer, &offset); - - for (iterator it = begin(); it != end(); ++it) - { - MPI_Get_address(it->Pointer, &displacements[location]); - displacements[location] -= offset; - - ++location; - lengths.push_back(it->Count); - types.push_back(it->Type); - } - // Create the type and commit it. - MPI_Type_create_struct(this->size(), &lengths.front(), &displacements.front(), &types.front(), &Type); - MPI_Type_commit(&Type); - } - } -} diff --git a/Code/net/ProcComms.h b/Code/net/ProcComms.h deleted file mode 100644 index 1dc197d2c..000000000 --- a/Code/net/ProcComms.h +++ /dev/null @@ -1,49 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PROCCOMMS_H -#define HEMELB_NET_PROCCOMMS_H -#include "constants.h" -#include "net/mpi.h" -#include "net/StoredRequest.h" -#include -#include - -namespace hemelb -{ - namespace net - { - template - class BaseProcComms : public std::deque - { - public: - MPI_Datatype Type; - }; - - class ProcComms : public BaseProcComms - { - public: - void CreateMPIType(); - }; - - class GatherProcComms : public BaseProcComms - { - - }; - - class AllToAllProcComms : public BaseProcComms - // Rank of request is not used - is all-to-all - { - - }; - - class GatherVReceiveProcComms : public BaseProcComms - { - - }; - } -} -#endif diff --git a/Code/net/StoredRequest.h b/Code/net/StoredRequest.h deleted file mode 100644 index 6336a3450..000000000 --- a/Code/net/StoredRequest.h +++ /dev/null @@ -1,52 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_STOREDREQUEST_H -#define HEMELB_NET_STOREDREQUEST_H -#include -#include "constants.h" -#include "net/mpi.h" -namespace hemelb -{ - namespace net - { - - class SimpleRequest - { - public: - void * Pointer; - int Count; - MPI_Datatype Type; - proc_t Rank; - SimpleRequest(void *pointer, int count, MPI_Datatype type, proc_t rank) : - Pointer(pointer), Count(count), Type(type), Rank(rank) - { - } - }; - - class ScalarRequest : public SimpleRequest - { - public: - ScalarRequest(void *pointer, MPI_Datatype type, proc_t rank) : - SimpleRequest(pointer, 1, type, rank) - { - } - }; - - class GatherVReceiveRequest : public SimpleRequest - { - public: - int * Counts; - int * Displacements; - GatherVReceiveRequest(void *pointer, int *displacements, int *counts, MPI_Datatype type) : - SimpleRequest(pointer, 0, type, 0), Counts(counts), Displacements(displacements) - { - } - }; - } -} - -#endif diff --git a/Code/net/mixins/InterfaceDelegationNet.h b/Code/net/mixins/InterfaceDelegationNet.h deleted file mode 100644 index 301c916ef..000000000 --- a/Code/net/mixins/InterfaceDelegationNet.h +++ /dev/null @@ -1,185 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_INTERFACEDELEGATIONNET_H -#define HEMELB_NET_MIXINS_INTERFACEDELEGATIONNET_H -namespace hemelb -{ - namespace net - { - /*** - * Define the external template-based interface to be used by client classes. - * We define a series of templates with nice C++ style interfaces which delegate to C-style template interfaces - * And then, delegate the templated C-style interfaces to nontemplated interfaces taking an MPI datatype. - */ - class InterfaceDelegationNet : public virtual BaseNet - { - public: - InterfaceDelegationNet(const MpiCommunicator& comms) : - BaseNet(comms) - { - } - - template - void RequestSendV(std::vector &payload, proc_t toRank) - { - RequestSend(&payload[0], payload.size(), toRank); - } - - template - void RequestSendR(T& value, proc_t toRank) - { - RequestSend(&value, 1, toRank); - } - - template - void RequestReceiveR(T& value, proc_t fromRank) - { - RequestReceive(&value, 1, fromRank); - } - - template - void RequestReceiveV(std::vector &payload, proc_t toRank) - { - RequestReceive(&payload[0], payload.size(), toRank); - } - - template - void RequestGatherVReceive(std::vector& bigBuffer, const std::vector& countsIn) - { - // It may seem inefficient to go through the list twice (it is), but it avoids any nastiness - // in 32 / 64 bit problems, because we can take our displacement as simply the difference - // between two addresses (by allocating the bigBuffer before the address calculation). - - int totalCount = 0; - for (std::vector::const_iterator count_iterator = countsIn.begin(); - count_iterator != countsIn.end(); count_iterator++) - { - totalCount += *count_iterator; - } - - // Allocate the large buffer we'll receive all data into. - bigBuffer.resize(totalCount); - bigBuffer.reserve(1); - - std::vector & displacements = this->GetDisplacementsBuffer(); - std::vector &counts = this->GetCountsBuffer(); - - // Now store the displacement and count for each sending proc. - int countSoFar = 0; - - for (std::vector::const_iterator count_iterator = countsIn.begin(); - count_iterator != countsIn.end(); count_iterator++) - { - int nextCount = *count_iterator; - - counts.push_back(nextCount); - displacements.push_back(&bigBuffer[countSoFar] - &bigBuffer[0]); - - countSoFar += nextCount; - } - - // And store the pointer and culminating arrays. - RequestGatherVReceive(&bigBuffer.front(), &displacements.front(), &counts.front()); - } - - template - void RequestGatherReceive(std::vector &buffer) - { - // Ensure vector has some underlying array, even if it's unused. - buffer.reserve(1); - RequestGatherReceive(&buffer.front()); - } - - template - void RequestGatherSend(T& value, proc_t toRank) - { - RequestGatherSend(&value, toRank); - } - - template - void RequestGatherVSend(std::vector &payload, proc_t toRank) - { - RequestGatherVSend(&payload.front(), payload.size(), toRank); - } - - /*** - * This is for a scalar all to all - * @param buffer vector with length same as communicator size - */ - template - void RequestAllToAllSend(std::vector &buffer) - { - RequestAllToAllSend(&buffer.front(), 1); - } - /*** - * This is for a scalar all to all - * @param buffer vector with length same as communicator size - */ - template - void RequestAllToAllReceive(std::vector &buffer) - { - RequestAllToAllReceive(&buffer.front(), 1); - } - - template - void RequestSend(T* pointer, int count, proc_t rank) - { - RequestSendImpl(pointer, count, rank, MpiDataType()); - } - - template - void RequestReceive(T* pointer, int count, proc_t rank) - { - RequestReceiveImpl(pointer, count, rank, MpiDataType()); - } - - /* - * Blocking gathers are implemented in MPI as a single call for both send/receive - * But, here we separate send and receive parts, since this interface may one day be used for - * nonblocking collectives. - */ - - template - void RequestGatherVSend(T* buffer, int count, proc_t toRank) - { - RequestGatherVSendImpl(buffer, count, toRank, MpiDataType()); - } - - template - void RequestGatherReceive(T* buffer) - { - RequestGatherReceiveImpl(buffer, MpiDataType()); - } - - template - void RequestGatherSend(T* buffer, proc_t toRank) - { - RequestGatherSendImpl(buffer, toRank, MpiDataType()); - } - - template - void RequestGatherVReceive(T* buffer, int * displacements, int *counts) - { - RequestGatherVReceiveImpl(buffer, displacements, counts, MpiDataType()); - } - - template - void RequestAllToAllSend(T* buffer, int count) - { - RequestAllToAllSendImpl(buffer, count, MpiDataType()); - } - template - void RequestAllToAllReceive(T* buffer, int count) - { - RequestAllToAllReceiveImpl(buffer, count, MpiDataType()); - } - - } - ; - } -} -#endif diff --git a/Code/net/mixins/StoringNet.cc b/Code/net/mixins/StoringNet.cc deleted file mode 100644 index b60c8bc2a..000000000 --- a/Code/net/mixins/StoringNet.cc +++ /dev/null @@ -1,76 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/StoringNet.h" -namespace hemelb -{ - namespace net - { - StoringNet::StoringNet(const MpiCommunicator& comms) : - BaseNet(comms) - { - } - void StoringNet::RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type) - { - if (count > 0) - { - sendProcessorComms[rank].push_back(SimpleRequest(pointer, count, type, rank)); - } - } - - void StoringNet::RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type) - { - if (count > 0) - { - receiveProcessorComms[rank].push_back(SimpleRequest(pointer, count, type, rank)); - } - } - - /* - * Blocking gathers are implemented in MPI as a single call for both send/receive - * But, here we separate send and receive parts, since this interface may one day be used for - * nonblocking collectives. - */ - - void StoringNet::RequestGatherVSendImpl(void* buffer, int count, proc_t toRank, - MPI_Datatype type) - { - gatherVSendProcessorComms[toRank].push_back(SimpleRequest(buffer, count, type, toRank)); - } - - void StoringNet::RequestGatherReceiveImpl(void* buffer, MPI_Datatype type) - { - /* - * Dummy rank to ScalarRequest of zero -- gathers always receive to the core where the receive request is made. - * Avoids defining another type of request. - */ - gatherReceiveProcessorComms.push_back(ScalarRequest(buffer, type, 0)); - } - - void StoringNet::RequestGatherSendImpl(void* buffer, proc_t toRank, MPI_Datatype type) - { - gatherSendProcessorComms[toRank].push_back(ScalarRequest(buffer, type, toRank)); - } - - void StoringNet::RequestGatherVReceiveImpl(void* buffer, int * displacements, int *counts, - MPI_Datatype type) - { - gatherVReceiveProcessorComms.push_back(GatherVReceiveRequest(buffer, - displacements, - counts, - type)); - } - - void StoringNet::RequestAllToAllReceiveImpl(void * buffer, int count, MPI_Datatype type) - { - allToAllReceiveProcComms.push_back(SimpleRequest(buffer, count, type, 0)); - } - void StoringNet::RequestAllToAllSendImpl(void * buffer, int count, MPI_Datatype type) - { - allToAllSendProcComms.push_back(SimpleRequest(buffer, count, type, 0)); - } - } -} diff --git a/Code/net/mixins/StoringNet.h b/Code/net/mixins/StoringNet.h deleted file mode 100644 index 1781ff0ab..000000000 --- a/Code/net/mixins/StoringNet.h +++ /dev/null @@ -1,53 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_STORINGNET_H -#define HEMELB_NET_MIXINS_STORINGNET_H -#include "net/BaseNet.h" -#include "log/Logger.h" -#include "net/ProcComms.h" -namespace hemelb -{ - namespace net - { - class StoringNet : public virtual BaseNet - { - public: - StoringNet(const MpiCommunicator& comms); - - virtual void RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); - virtual void RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); - - void RequestGatherVSendImpl(void* buffer, int count, proc_t toRank, MPI_Datatype type); - void RequestGatherReceiveImpl(void* buffer, MPI_Datatype type); - - void RequestGatherSendImpl(void* buffer, proc_t toRank, MPI_Datatype type); - void RequestGatherVReceiveImpl(void* buffer, int * displacements, int *counts, MPI_Datatype type); - - virtual void RequestAllToAllReceiveImpl(void * buffer, int count, MPI_Datatype type); - virtual void RequestAllToAllSendImpl(void * buffer, int count, MPI_Datatype type); - - protected: - /** - * Struct representing all that's needed to successfully communicate with another processor. - */ - - std::map sendProcessorComms; - std::map receiveProcessorComms; - - std::map gatherVSendProcessorComms; - GatherVReceiveProcComms gatherVReceiveProcessorComms; - - std::map gatherSendProcessorComms; - GatherProcComms gatherReceiveProcessorComms; - - AllToAllProcComms allToAllReceiveProcComms; - AllToAllProcComms allToAllSendProcComms; - - }; - } -} -#endif diff --git a/Code/net/mixins/alltoall/SeparatedAllToAll.cc b/Code/net/mixins/alltoall/SeparatedAllToAll.cc deleted file mode 100644 index 1e8f2308f..000000000 --- a/Code/net/mixins/alltoall/SeparatedAllToAll.cc +++ /dev/null @@ -1,34 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/alltoall/SeparatedAllToAll.h" -#include -namespace hemelb -{ - namespace net - { - SeparatedAllToAll::SeparatedAllToAll(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms) - { - } - - void SeparatedAllToAll::WaitAllToAll() - { - assert(allToAllReceiveProcComms.size() == allToAllSendProcComms.size()); - for (unsigned int i = 0; i < allToAllReceiveProcComms.size(); i++) - { - SimpleRequest & sendreq = allToAllSendProcComms[i]; - SimpleRequest & receivereq = allToAllReceiveProcComms[i]; - HEMELB_MPI_CALL(MPI_Alltoall, - (sendreq.Pointer, sendreq.Count, sendreq.Type, receivereq.Pointer, receivereq.Count, receivereq.Type, communicator)); - } - - allToAllReceiveProcComms.clear(); - allToAllSendProcComms.clear(); - } - } -} - diff --git a/Code/net/mixins/alltoall/SeparatedAllToAll.h b/Code/net/mixins/alltoall/SeparatedAllToAll.h deleted file mode 100644 index 06abebe64..000000000 --- a/Code/net/mixins/alltoall/SeparatedAllToAll.h +++ /dev/null @@ -1,26 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_ALLTOALL_SEPARATEDALLTOALL_H -#define HEMELB_NET_MIXINS_ALLTOALL_SEPARATEDALLTOALL_H - -#include "net/mixins/StoringNet.h" - -namespace hemelb{ - namespace net{ - - class SeparatedAllToAll : public virtual StoringNet - { - public: - SeparatedAllToAll(const MpiCommunicator& comms); - private: - void ReceiveAllToAll(){} - void SendAllToAll(){} - void WaitAllToAll(); - }; - } -} -#endif diff --git a/Code/net/mixins/alltoall/ViaPointPointAllToAll.cc b/Code/net/mixins/alltoall/ViaPointPointAllToAll.cc deleted file mode 100644 index 648be4980..000000000 --- a/Code/net/mixins/alltoall/ViaPointPointAllToAll.cc +++ /dev/null @@ -1,67 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/alltoall/ViaPointPointAllToAll.h" -namespace hemelb -{ - namespace net - { - ViaPointPointAllToAll::ViaPointPointAllToAll(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms) - { - } - void ViaPointPointAllToAll::ReceiveAllToAll() - { - - for (AllToAllProcComms::iterator receivereq = allToAllReceiveProcComms.begin(); - receivereq != allToAllReceiveProcComms.end(); receivereq++) - { - - int size; - MPI_Type_size(receivereq->Type, &size); - - for (int source_rank = 0; source_rank < communicator.Size(); source_rank++) - { - - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - RequestReceiveImpl(static_cast(receivereq->Pointer) + size * source_rank, - 1, - source_rank, - receivereq->Type); - } - - } - - allToAllReceiveProcComms.clear(); - } - - void ViaPointPointAllToAll::SendAllToAll() - { - - for (AllToAllProcComms::iterator sendreq = allToAllSendProcComms.begin(); - sendreq != allToAllSendProcComms.end(); sendreq++) - { - - int size; - MPI_Type_size(sendreq->Type, &size); - - for (int dest_rank = 0; dest_rank < communicator.Size(); dest_rank++) - { - - RequestSendImpl(static_cast(sendreq->Pointer) + size * dest_rank, - 1, - dest_rank, - sendreq->Type); - } - - } - - allToAllSendProcComms.clear(); - } - - } -} diff --git a/Code/net/mixins/alltoall/ViaPointPointAllToAll.h b/Code/net/mixins/alltoall/ViaPointPointAllToAll.h deleted file mode 100644 index 7a424dade..000000000 --- a/Code/net/mixins/alltoall/ViaPointPointAllToAll.h +++ /dev/null @@ -1,26 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_ALLTOALL_VIAPOINTPOINTALLTOALL_H -#define HEMELB_NET_MIXINS_ALLTOALL_VIAPOINTPOINTALLTOALL_H - -#include "net/mixins/StoringNet.h" - -namespace hemelb{ - namespace net{ - - class ViaPointPointAllToAll : public virtual StoringNet - { - public: - ViaPointPointAllToAll(const MpiCommunicator& comms); - private: - void ReceiveAllToAll(); - void SendAllToAll(); - void WaitAllToAll(){} - }; - } -} -#endif diff --git a/Code/net/mixins/gathers/SeparatedGathers.cc b/Code/net/mixins/gathers/SeparatedGathers.cc deleted file mode 100644 index 2b0ed7ac6..000000000 --- a/Code/net/mixins/gathers/SeparatedGathers.cc +++ /dev/null @@ -1,114 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/gathers/SeparatedGathers.h" -namespace hemelb -{ - namespace net - { - SeparatedGathers::SeparatedGathers(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms) - { - } - - void SeparatedGathers::WaitGathers() - { - /* - * For each of the stored gather send requests... - * Each of which, is all the requests to gather to a certain root... - */ - for (std::map::iterator send_it = gatherSendProcessorComms.begin(); - send_it != gatherSendProcessorComms.end(); ++send_it) - { - /* - * send_it->first is the rank to which the gather must be sent - * send_it->second is a VECTOR of ScalarRequests, each a stored request defining Type and Pointer. - * There is one entry, for each request to gather to that root. - */ - /* - * If I am sending to MYSELF, then I must assemble a gather request for sending and receiving myself - */ - if (send_it->first == communicator.Rank()) - { - /* - * I may be the ROOT of several gathers. - * If I am, then we hope, the sends and receives were defined in the same order. - * So, we assume the first of the receive requests, matches the first of the send-to-self requests - */ - int gather_index = 0; - for (GatherProcComms::iterator receive_it = gatherReceiveProcessorComms.begin(); - receive_it != gatherReceiveProcessorComms.end(); ++receive_it) - { - ScalarRequest toself = send_it->second[gather_index]; - - /* - * So now, I can send/receive to myself, for this request - */HEMELB_MPI_CALL(MPI_Gather, - (toself.Pointer, 1, toself.Type, receive_it->Pointer, 1, receive_it->Type, communicator.Rank(), communicator)); - ++gather_index; - } - } - else - /* - * I am not sending to myself, so I just need to send each of the gathers to the given root - * No attempt is made to coalesce several gathers to the same root. - */ - { - for (GatherProcComms::iterator req = send_it->second.begin(); - req != send_it->second.end(); req++) - { - HEMELB_MPI_CALL(MPI_Gather, - (req->Pointer, 1, req->Type, NULL, 1, req->Type, send_it->first, communicator)); - } - } - } - - gatherSendProcessorComms.clear(); - gatherReceiveProcessorComms.clear(); - } - - void SeparatedGathers::WaitGatherVs() - { - - for (std::map::iterator send_it = gatherVSendProcessorComms.begin(); - send_it != gatherVSendProcessorComms.end(); ++send_it) - { - - if (send_it->first == communicator.Rank()) - { - int gather_index = 0; - - for (GatherVReceiveProcComms::iterator receive_it = gatherVReceiveProcessorComms.begin(); - receive_it != gatherVReceiveProcessorComms.end(); ++receive_it) - { - /*** - * Again, as for WaitGather, we assume that send/receive requests were placed in parallel ordering... - */ - SimpleRequest toself = send_it->second[gather_index]; - - HEMELB_MPI_CALL(MPI_Gatherv, - ( toself.Pointer, toself.Count, toself.Type, receive_it->Pointer, receive_it->Counts, receive_it->Displacements, receive_it->Type, communicator.Rank(), communicator)); - ++gather_index; - } - } - else - { - - for (ProcComms::iterator req = send_it->second.begin(); req != send_it->second.end(); - req++) - { - HEMELB_MPI_CALL(MPI_Gatherv, - ( req->Pointer, req->Count, req->Type, NULL, NULL, NULL, req->Type, send_it->first, communicator)); - } - - } - } - gatherVSendProcessorComms.clear(); - gatherVReceiveProcessorComms.clear(); - } - } -} - diff --git a/Code/net/mixins/gathers/SeparatedGathers.h b/Code/net/mixins/gathers/SeparatedGathers.h deleted file mode 100644 index 91423e73b..000000000 --- a/Code/net/mixins/gathers/SeparatedGathers.h +++ /dev/null @@ -1,30 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_GATHERS_SEPARATEDGATHERS_H -#define HEMELB_NET_MIXINS_GATHERS_SEPARATEDGATHERS_H - -#include "net/mixins/StoringNet.h" - -namespace hemelb{ - namespace net{ - - class SeparatedGathers : public virtual StoringNet - { - public: - SeparatedGathers(const MpiCommunicator& comms); - private: - void ReceiveGathers(){} - void SendGathers(){} - void ReceiveGatherVs(){} - void SendGatherVs(){} - void WaitGathers(); - void WaitGatherVs(); - }; - - } -} -#endif diff --git a/Code/net/mixins/gathers/ViaPointPointGathers.cc b/Code/net/mixins/gathers/ViaPointPointGathers.cc deleted file mode 100644 index 933917978..000000000 --- a/Code/net/mixins/gathers/ViaPointPointGathers.cc +++ /dev/null @@ -1,98 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/gathers/ViaPointPointGathers.h" -namespace hemelb -{ - namespace net - { - ViaPointPointGathers::ViaPointPointGathers(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms) - { - } - - void ViaPointPointGathers::ReceiveGathers() - { - for (GatherProcComms::iterator receive_it = gatherReceiveProcessorComms.begin(); - receive_it != gatherReceiveProcessorComms.end(); ++receive_it) - { - - int size; - MPI_Type_size(receive_it->Type, &size); - - for (int source_rank = 0; source_rank < communicator.Size(); source_rank++) - { - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - RequestReceiveImpl(static_cast(receive_it->Pointer) + size * source_rank, - 1, - source_rank, - receive_it->Type); - } - } - - gatherReceiveProcessorComms.clear(); - } - void ViaPointPointGathers::SendGathers() - { - for (std::map::iterator send_it = gatherSendProcessorComms.begin(); - send_it != gatherSendProcessorComms.end(); ++send_it) - { - - for (GatherProcComms::iterator req = send_it->second.begin(); req != send_it->second.end(); - req++) - { - RequestSendImpl(req->Pointer, 1, send_it->first, req->Type); - } - - } - - gatherSendProcessorComms.clear(); - } - void ViaPointPointGathers::ReceiveGatherVs() - { - for (GatherVReceiveProcComms::iterator receive_it = gatherVReceiveProcessorComms.begin(); - receive_it != gatherVReceiveProcessorComms.end(); ++receive_it) - { - - int size; - MPI_Type_size(receive_it->Type, &size); - - for (int source_rank = 0; source_rank < communicator.Size(); source_rank++) - { - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - // Note that MPI Displacements are given in the arithmetic appropriate to the MPI_Datatype, not void*, i.e. in units of the size - // It will also potentially fail, if the MPI_Datatype used, is a sparse (strided) type. - // This class is intended for timing and testing use, not production use. - RequestReceiveImpl(static_cast(receive_it->Pointer) - + receive_it->Displacements[source_rank] * size, - receive_it->Counts[source_rank], - source_rank, - receive_it->Type); - } - } - - gatherVReceiveProcessorComms.clear(); - } - void ViaPointPointGathers::SendGatherVs() - { - for (std::map::iterator send_it = gatherVSendProcessorComms.begin(); - send_it != gatherVSendProcessorComms.end(); ++send_it) - { - - for (ProcComms::iterator req = send_it->second.begin(); req != send_it->second.end(); req++) - { - RequestSendImpl(req->Pointer, req->Count, send_it->first, req->Type); - } - - } - - gatherVSendProcessorComms.clear(); - } - } -} - diff --git a/Code/net/mixins/gathers/ViaPointPointGathers.h b/Code/net/mixins/gathers/ViaPointPointGathers.h deleted file mode 100644 index eb4445c38..000000000 --- a/Code/net/mixins/gathers/ViaPointPointGathers.h +++ /dev/null @@ -1,33 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_GATHERS_VIAPOINTPOINTGATHERS_H -#define HEMELB_NET_MIXINS_GATHERS_VIAPOINTPOINTGATHERS_H - -#include "net/mixins/StoringNet.h" - -namespace hemelb{ - namespace net{ - /*** - * Reimplement gathers via point-point calls - * This code is not robust for working with gathers of complex defined datatypes. - * It is intended for testing and performance measurement use, and should not be used in production. - */ - class ViaPointPointGathers : public virtual StoringNet - { - public: - ViaPointPointGathers(const MpiCommunicator& comms); - private: - void ReceiveGathers(); - void SendGathers(); - void ReceiveGatherVs(); - void SendGatherVs(); - void WaitGathers(){} - void WaitGatherVs(){} - }; - } -} -#endif diff --git a/Code/net/mixins/mixins.h b/Code/net/mixins/mixins.h deleted file mode 100644 index 570e780f8..000000000 --- a/Code/net/mixins/mixins.h +++ /dev/null @@ -1,19 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_MIXINS_H -#define HEMELB_NET_MIXINS_MIXINS_H - -#include "net/mixins/pointpoint/CoalescePointPoint.h" -#include "net/mixins/pointpoint/ImmediatePointPoint.h" -#include "net/mixins/pointpoint/SeparatedPointPoint.h" -#include "net/mixins/StoringNet.h" -#include "net/mixins/gathers/SeparatedGathers.h" -#include "net/mixins/InterfaceDelegationNet.h" -#include "net/mixins/gathers/ViaPointPointGathers.h" -#include "net/mixins/alltoall/SeparatedAllToAll.h" -#include "net/mixins/alltoall/ViaPointPointAllToAll.h" -#endif diff --git a/Code/net/mixins/pointpoint/CoalescePointPoint.cc b/Code/net/mixins/pointpoint/CoalescePointPoint.cc deleted file mode 100644 index 77e62cdf0..000000000 --- a/Code/net/mixins/pointpoint/CoalescePointPoint.cc +++ /dev/null @@ -1,145 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/pointpoint/CoalescePointPoint.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace net - { - - void CoalescePointPoint::EnsureEnoughRequests(size_t count) - { - if (requests.size() < count) - { - requests.resize(count, MPI_Request()); - statuses.resize(count, MPI_Status()); - } - } - - void CoalescePointPoint::ReceivePointToPoint() - { - - // Make sure the MPI datatypes have been created. - EnsurePreparedToSendReceive(); - proc_t m = 0; - - for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); - ++it) - { - - MPI_Irecv(it->second.front().Pointer, - 1, - it->second.Type, - it->first, - 10, - communicator, - &requests[m]); - ++m; - } - - - //if(m>1) { - // hemelb::log::Logger::Log("RecvPointToPoint() Neighbouring proc count: %i", m); - //} - - } - - // Makes sure the MPI_Datatypes for sending and receiving have been created for every neighbour. - void CoalescePointPoint::EnsurePreparedToSendReceive() - { - if (sendReceivePrepped) - { - return; - } - - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); ++it) - { - it->second.CreateMPIType(); - } - - for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); - ++it) - { - it->second.CreateMPIType(); - } - - EnsureEnoughRequests(receiveProcessorComms.size() + sendProcessorComms.size()); - - sendReceivePrepped = true; - } - - void CoalescePointPoint::SendPointToPoint() - { - - // Make sure the MPI datatypes have been created. - EnsurePreparedToSendReceive(); - proc_t m = 0; - - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); ++it) - { - - int TypeSizeStorage = 0; //DTMP:byte size tracking - MPI_Type_size(it->second.Type, &TypeSizeStorage); //DTMP: - BytesSent += TypeSizeStorage; //DTMP: - - MPI_Isend(it->second.front().Pointer, - 1, - it->second.Type, - it->first, - 10, - communicator, - &requests[receiveProcessorComms.size() + m]); - - ++m; - } - } - - /*! - Free the allocated data. - */ - CoalescePointPoint::~CoalescePointPoint() - { - if (sendReceivePrepped) - { - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); - ++it) - { - MPI_Type_free(&it->second.Type); - } - - for (std::map::iterator it = receiveProcessorComms.begin(); - it != receiveProcessorComms.end(); ++it) - { - MPI_Type_free(&it->second.Type); - } - - } - } - - void CoalescePointPoint::WaitPointToPoint() - { - - MPI_Waitall((int) (sendProcessorComms.size() + receiveProcessorComms.size()), &requests[0], &statuses[0]); - - for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); - ++it) - { - MPI_Type_free(&it->second.Type); - } - receiveProcessorComms.clear(); - - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); ++it) - { - MPI_Type_free(&it->second.Type); - } - sendProcessorComms.clear(); - sendReceivePrepped = false; - - } - } -} diff --git a/Code/net/mixins/pointpoint/CoalescePointPoint.h b/Code/net/mixins/pointpoint/CoalescePointPoint.h deleted file mode 100644 index 69844b6be..000000000 --- a/Code/net/mixins/pointpoint/CoalescePointPoint.h +++ /dev/null @@ -1,46 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_POINTPOINT_COALESCEPOINTPOINT_H -#define HEMELB_NET_MIXINS_POINTPOINT_COALESCEPOINTPOINT_H -#include "net/BaseNet.h" -#include "net/mixins/StoringNet.h" -namespace hemelb -{ - namespace net - { - class CoalescePointPoint : public virtual StoringNet - { - - public: - CoalescePointPoint(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms), sendReceivePrepped(false) - { - } - ~CoalescePointPoint(); - - void WaitPointToPoint(); - - protected: - void ReceivePointToPoint(); - void SendPointToPoint(); - - private: - void EnsureEnoughRequests(size_t count); - void EnsurePreparedToSendReceive(); - bool sendReceivePrepped; - - // Requests and statuses available for general communication within the Net object (both - // initialisation and during each iteration). Code using these must make sure - // there are enough available. We do this in a way to minimise the number created - // on each core, but also to minimise creation / deletion overheads. - std::vector requests; - std::vector statuses; - }; - } -} - -#endif diff --git a/Code/net/mixins/pointpoint/ImmediatePointPoint.cc b/Code/net/mixins/pointpoint/ImmediatePointPoint.cc deleted file mode 100644 index 9fa1f7ad6..000000000 --- a/Code/net/mixins/pointpoint/ImmediatePointPoint.cc +++ /dev/null @@ -1,51 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/pointpoint/ImmediatePointPoint.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace net - { - void ImmediatePointPoint::RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type) - { - HEMELB_MPI_CALL( - MPI_Ssend, - (pointer, count, type, rank, 10, communicator) - ); - } - void ImmediatePointPoint::RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type) - { - HEMELB_MPI_CALL( - MPI_Recv, - (pointer, count, type, rank, 10, communicator, MPI_STATUS_IGNORE) - ); - } - - void ImmediatePointPoint::ReceivePointToPoint() - { - - } - - void ImmediatePointPoint::SendPointToPoint() - { - - } - - /*! - Free the allocated data. - */ - ImmediatePointPoint::~ImmediatePointPoint() - { - } - - void ImmediatePointPoint::WaitPointToPoint() - { - - } - } -} diff --git a/Code/net/mixins/pointpoint/ImmediatePointPoint.h b/Code/net/mixins/pointpoint/ImmediatePointPoint.h deleted file mode 100644 index 967b56126..000000000 --- a/Code/net/mixins/pointpoint/ImmediatePointPoint.h +++ /dev/null @@ -1,40 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_POINTPOINT_IMMEDIATEPOINTPOINT_H -#define HEMELB_NET_MIXINS_POINTPOINT_IMMEDIATEPOINTPOINT_H -#include "net/BaseNet.h" -#include "net/mixins/StoringNet.h" -namespace hemelb -{ - namespace net - { - // although ImmediatePointPoint does not use the StoringNet capabilities at all - // it needs to inherit it - // so that it becomes the unique final overrider - class ImmediatePointPoint : public virtual StoringNet - { - - public: - ImmediatePointPoint(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms) - { - } - ~ImmediatePointPoint(); - - void WaitPointToPoint(); - // we will *NOT* store the requests, so we must provide RequestSendImpl ourselves. - virtual void RequestSendImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); - virtual void RequestReceiveImpl(void* pointer, int count, proc_t rank, MPI_Datatype type); - protected: - void ReceivePointToPoint(); //PASS - void SendPointToPoint(); //PASS - - }; - } -} - -#endif diff --git a/Code/net/mixins/pointpoint/SeparatedPointPoint.cc b/Code/net/mixins/pointpoint/SeparatedPointPoint.cc deleted file mode 100644 index 280699204..000000000 --- a/Code/net/mixins/pointpoint/SeparatedPointPoint.cc +++ /dev/null @@ -1,114 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mixins/pointpoint/SeparatedPointPoint.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace net - { - - void SeparatedPointPoint::EnsureEnoughRequests(size_t count) - { - if (requests.size() < count) - { - requests.resize(count, MPI_Request()); - statuses.resize(count, MPI_Status()); - } - } - - void SeparatedPointPoint::ReceivePointToPoint() - { - EnsurePreparedToSendReceive(); - proc_t m = 0; - - for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); - ++it) - { - for (ProcComms::iterator request = it->second.begin(); request != it->second.end(); request++) - { - - MPI_Irecv(request->Pointer, - request->Count, - request->Type, - it->first, - 10, - communicator, - &requests[m]); - ++m; - } - } - - } - - // Makes sure the MPI_Datatypes for sending and receiving have been created for every neighbour. - void SeparatedPointPoint::EnsurePreparedToSendReceive() - { - if (sendReceivePrepped) - { - return; - } - - count_sends=0; - count_receives=0; - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); ++it) - { - count_sends += it->second.size(); - } - - for (std::map::iterator it = receiveProcessorComms.begin(); it != receiveProcessorComms.end(); - ++it) - { - count_receives += it->second.size(); - } - - EnsureEnoughRequests(count_sends+count_receives); - - sendReceivePrepped = true; - } - - void SeparatedPointPoint::SendPointToPoint() - { - - // Make sure the MPI datatypes have been created. - EnsurePreparedToSendReceive(); - proc_t m = 0; - - for (std::map::iterator it = sendProcessorComms.begin(); it != sendProcessorComms.end(); ++it) - { - for (ProcComms::iterator request = it->second.begin(); request != it->second.end(); request++) - { - MPI_Isend(request->Pointer, - request->Count, - request->Type, - it->first, - 10, - communicator, - &requests[count_receives+m]); - ++m; - } - } - } - - /*! - Free the allocated data. - */ - SeparatedPointPoint::~SeparatedPointPoint() - { - } - - void SeparatedPointPoint::WaitPointToPoint() - { - MPI_Waitall(static_cast(count_sends+count_receives), &requests[0], &statuses[0]); - - receiveProcessorComms.clear(); - sendProcessorComms.clear(); - sendReceivePrepped = false; - - } - } -} diff --git a/Code/net/mixins/pointpoint/SeparatedPointPoint.h b/Code/net/mixins/pointpoint/SeparatedPointPoint.h deleted file mode 100644 index 239e76e2a..000000000 --- a/Code/net/mixins/pointpoint/SeparatedPointPoint.h +++ /dev/null @@ -1,49 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MIXINS_POINTPOINT_SEPARATEDPOINTPOINT_H -#define HEMELB_NET_MIXINS_POINTPOINT_SEPARATEDPOINTPOINT_H -#include "net/BaseNet.h" -#include "net/mixins/StoringNet.h" -namespace hemelb -{ - namespace net - { - class SeparatedPointPoint : public virtual StoringNet - { - - public: - SeparatedPointPoint(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms), sendReceivePrepped(false), count_sends(0), - count_receives(0) - { - } - ~SeparatedPointPoint(); - - void WaitPointToPoint(); - - protected: - void ReceivePointToPoint(); - void SendPointToPoint(); - - private: - void EnsureEnoughRequests(size_t count); - void EnsurePreparedToSendReceive(); - bool sendReceivePrepped; - - // Requests and statuses available for general communication within the Net object (both - // initialisation and during each iteration). Code using these must make sure - // there are enough available. We do this in a way to minimise the number created - // on each core, but also to minimise creation / deletion overheads. - std::vector requests; - std::vector statuses; - unsigned int count_sends; - unsigned int count_receives; - }; - } -} - -#endif diff --git a/Code/net/mpi.h b/Code/net/mpi.h deleted file mode 100644 index 82ddfa6fa..000000000 --- a/Code/net/mpi.h +++ /dev/null @@ -1,17 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_MPI_H -#define HEMELB_NET_MPI_H - -#include -#include "net/MpiDataType.h" -#include "net/MpiEnvironment.h" -#include "net/MpiCommunicator.h" -#include "net/MpiGroup.h" -#include "net/MpiError.h" - -#endif // HEMELB_NET_MPI_H diff --git a/Code/net/net.h b/Code/net/net.h deleted file mode 100644 index 954cf9ea9..000000000 --- a/Code/net/net.h +++ /dev/null @@ -1,33 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_NET_H -#define HEMELB_NET_NET_H - -#include "net/BaseNet.h" -#include "net/mixins/mixins.h" -#include "net/BuildInfo.h" -namespace hemelb -{ - namespace net - { - class Net : public PointPointImpl, - public InterfaceDelegationNet, - public AllToAllImpl, - public GathersImpl - { - public: - Net(const MpiCommunicator &communicator) : - BaseNet(communicator), StoringNet(communicator), PointPointImpl(communicator), - InterfaceDelegationNet(communicator), AllToAllImpl(communicator), - GathersImpl(communicator) - { - } - }; - } -} -#endif - diff --git a/Code/net/phased/Concern.cc b/Code/net/phased/Concern.cc deleted file mode 100644 index ac58e5a88..000000000 --- a/Code/net/phased/Concern.cc +++ /dev/null @@ -1,6 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - diff --git a/Code/net/phased/Concern.h b/Code/net/phased/Concern.h deleted file mode 100644 index 2ab9c8c96..000000000 --- a/Code/net/phased/Concern.h +++ /dev/null @@ -1,21 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASED_CONCERN_H -#define HEMELB_NET_PHASED_CONCERN_H -namespace hemelb{ - namespace net{ - namespace phased{ - class Concern{ - public: - Concern(){} - virtual ~Concern(){} - virtual bool CallAction(int action)=0; - }; - } - } -} -#endif //ONCE diff --git a/Code/net/phased/NetConcern.h b/Code/net/phased/NetConcern.h deleted file mode 100644 index 2cfc38765..000000000 --- a/Code/net/phased/NetConcern.h +++ /dev/null @@ -1,51 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASED_NETCONCERN_H -#define HEMELB_NET_PHASED_NETCONCERN_H - -#include "net/phased/Concern.h" -#include "net/phased/steps.h" - -namespace hemelb -{ - namespace net - { - namespace phased - { - class NetConcern : public Concern - { - public: - NetConcern(net::BaseNet & net) : - net(net) - { - } - bool CallAction(int action) - { - switch (static_cast(action)) - { - case phased::steps::Send: - net.Send(); - return true; - case phased::steps::Receive: - net.Receive(); - return true; - case phased::steps::Wait: - net.Wait(); - return true; - - default: - return false; - } - } - private: - net::BaseNet & net; - }; - } - } -} - -#endif diff --git a/Code/net/phased/StepManager.cc b/Code/net/phased/StepManager.cc deleted file mode 100644 index 340b10728..000000000 --- a/Code/net/phased/StepManager.cc +++ /dev/null @@ -1,210 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/phased/StepManager.h" -#include - -namespace hemelb -{ - namespace net - { - namespace phased - { - - StepManager::StepManager(Phase phases, reporting::Timers *timers, bool separate_concerns) : - registry(phases), concerns(), timers(timers), separate_concerns(separate_concerns) - { - } - - void StepManager::Register(Phase phase, steps::Step step, Concern & concern, MethodLabel method) - { - if (step == steps::BeginAll || step == steps::EndAll) - { - phase = 0; // special actions are always recorded in the phase zero registry - } - registry[phase][step].push_back(Action(concern, method)); - if (std::find(concerns.begin(),concerns.end(),&concern)==concerns.end()){ - concerns.push_back(&concern); - } - } - - void StepManager::RegisterIteratedActorSteps(Concern &concern, Phase phase) - { - for (int step = steps::BeginPhase; step <= steps::EndAll; step++) - { - if (step == steps::Receive || step == steps::Send || step == steps::Wait) - { - continue; - } - // C++ makes it completely annoying to iterate over values in an enum - Register(phase, static_cast(step), concern, step); - } - } - - void StepManager::RegisterCommsSteps(Concern &concern, Phase phase) - { - Register(phase, steps::Send, concern, steps::Send); - Register(phase, steps::Receive, concern, steps::Receive); - Register(phase, steps::Wait, concern, steps::Wait); - } - - void StepManager::RegisterCommsForAllPhases(Concern &concern) - { - for (Phase phase = 0; phase < registry.size(); phase++) - { - RegisterCommsSteps(concern, phase); - } - } - - unsigned int StepManager::ConcernCount() const - { - return concerns.size(); - } - - unsigned int StepManager::ActionCount() const - { - unsigned int total = 0; - for (std::vector::const_iterator phase = registry.begin(); phase < registry.end(); phase++) - { - for (Registry::const_iterator step = phase->begin(); step != phase->end(); step++) - { - for (std::vector::const_iterator action = step->second.begin(); action != step->second.end(); - action++) - { - total++; - } - } - } - return total; - } - - void StepManager::CallActionsForPhase(Phase phase) - { - // It is assumed, that in the step enum, begin phase begins, and end phase ends, the steps which - // must be called for a given phase. - for (int step = steps::BeginPhase; step <= steps::EndPhase; step++) - { - CallActionsForStep(static_cast(step), phase); - } - } - - void StepManager::CallActionsForPhaseSeparatedConcerns(Phase phase) - { - for (std::vector::iterator concern = concerns.begin(); concern != concerns.end(); concern++) - { - for (int step = steps::BeginPhase; step <= steps::EndPhase; step++) - { - if (step == steps::Receive || step == steps::Send || step == steps::Wait) - { - // Call ALL comms actions for all concerns - // Because, these concerns are net::Net objects, that do the actual send/receive/wait MPI calls - /** - * E.g: - * if A is a concern, B is a concern, C is a concern - * A is an it actor, B is an it actor,C is a net::Net - * You want to go: A C A C A C B C B C B C - */ - CallActionsForStep(static_cast(step), phase); - } - else - { - // Call the actions only for THIS concern - CallActionsForStepForConcern(static_cast(step), *concern, phase); - } - } - } - } - - void StepManager::CallSpecialAction(steps::Step step) - { - // special actions are always recorded in the phase zero registry - CallActionsForStep(static_cast(step), 0); - } - - void StepManager::CallActionsSeparatedConcerns() - { - CallSpecialAction(steps::BeginAll); - for (Phase phase = 0; phase < registry.size(); phase++) - { - CallActionsForPhaseSeparatedConcerns(phase); - } - CallSpecialAction(steps::EndAll); - } - - void StepManager::CallActions() - { - if (separate_concerns) - { - CallActionsSeparatedConcerns(); - return; - } - CallSpecialAction(steps::BeginAll); - for (Phase phase = 0; phase < registry.size(); phase++) - { - CallActionsForPhase(phase); - } - CallSpecialAction(steps::EndAll); - } - - void StepManager::CallActionsForStep(steps::Step step, Phase phase) - { - StartTimer(step); - std::vector &actionsForStep = registry[phase][step]; - for (std::vector::iterator action = actionsForStep.begin(); action != actionsForStep.end(); action++) - { - action->Call(); - } - StopTimer(step); - } - - void StepManager::CallActionsForStepForConcern(steps::Step step, Concern * concern, Phase phase) - { - StartTimer(step); - std::vector &actionsForStep = registry[phase][step]; - for (std::vector::iterator action = actionsForStep.begin(); action != actionsForStep.end(); action++) - { - if (action->concern == concern) - { - action->Call(); - } - } - StopTimer(step); - } - - void StepManager::StartTimer(steps::Step step) - { - if (!timers) - { - return; - } - if (step == steps::Wait) - { - (*timers)[hemelb::reporting::Timers::mpiWait].Start(); - } - if (step == steps::Send) - { - (*timers)[hemelb::reporting::Timers::mpiSend].Start(); - } - } - - void StepManager::StopTimer(steps::Step step) - { - if (!timers) - { - return; - } - if (step == steps::Wait) - { - (*timers)[hemelb::reporting::Timers::mpiWait].Stop(); - } - if (step == steps::Send) - { - (*timers)[hemelb::reporting::Timers::mpiSend].Stop(); - } - } - } - } -} diff --git a/Code/net/phased/StepManager.h b/Code/net/phased/StepManager.h deleted file mode 100644 index fe4584361..000000000 --- a/Code/net/phased/StepManager.h +++ /dev/null @@ -1,175 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASED_STEPMANAGER_H -#define HEMELB_NET_PHASED_STEPMANAGER_H -#include -#include -#include -#include -#include "reporting/Timers.h" -#include "net/IteratedAction.h" -#include "net/phased/Concern.h" -#include "net/phased/steps.h" - -#include "log/Logger.h" -namespace hemelb -{ - namespace net - { - namespace phased - { - /*** - * The step manager provides ability to call, in sequence, methods on several - * interested objects (Called "concerns") for each of several Steps, - * in each of several Phases. - * This provides a mechanism to allow concerns to schedule there activities - * during and between each of the steps of an asynchronous communication pattern - * There may be several sequences of asynchronous communication during a single - * iteration of the SimulationMaster, by registering several phases, each with - * it's own pre/post send-receive steps, etc. - */ - class StepManager - { - public: - - typedef unsigned int Phase; - typedef int MethodLabel; - - /*** - * Class abstracting methods which can be called on concerns - * In principle, we would like to use function pointer classes to do this - * But to keep things simpler, we are assuming indirection through a 'CallAction' method - * on concerns, which handles dispatch to the appropriate method. - */ - class Action - { - public: - Concern * concern; - MethodLabel method; - std::string name; - Action(Concern &concern, MethodLabel method) : - concern(&concern), method(method) - { - } - Action(const Action & action) : - concern(action.concern), method(action.method) - { - } - bool Call() - { - return concern->CallAction(method); - } - }; - - typedef std::map > Registry; - - /*** - * Construct a step manager - * @param The number of phases, default 1. - * @param timers, Record the times for the steps to this timers object, if given - */ - StepManager(Phase phases = 1, reporting::Timers * timers = NULL, bool separate_concerns = false); - - /*** - * Register an action of a concern - * @param phase Phase where an action should be called - * @param step Step where an action should be called - * @param concern Concern on which the method should be called - * @param method label, indicating which method of the concern should be registered - */ - void Register(Phase phase, steps::Step step, Concern & concern, MethodLabel method); - - /*** - * Register a concern for all of the steps typically used by an action, - * syntactic sugar for the individual registrations - * @param concern Concern which should be called, typically an IteratedActor - * @param phase Phase for which this IteratedActor should be called. - */ - void RegisterIteratedActorSteps(Concern &concern, Phase phase = 0); - - /*** - * Register a concern for all of the steps typically used to send/receive communications - * syntactic sugar for the individual registrations (to Send, Receive, Wait) - * @param concern Concern which should be called, typically a net::Net - * @param phase Phase for which this concern should be used for comms - */ - void RegisterCommsSteps(Concern &concern, Phase phase = 0); - - /*** - * Register a concern for all comms steps in all phases, typically the main net::Net - * @param concern Concern to register, typically a net::Net - */ - void RegisterCommsForAllPhases(Concern &concern); - - /*** - * Call the actions, concern by concern, for the given phase - * @param phase - */ - void CallActionsForPhaseSeparatedConcerns(Phase phase = 0); - - /*** - * Call the actions for the given phase - * @param phase - */ - void CallActionsForPhase(Phase phase = 0); - - /*** - * Call all registered actions for a given phase in a given step - * @param step - * @param phase (Default the first phase) - */ - void CallActionsForStepForConcern(steps::Step step,Concern * concern, Phase phase = 0); - - /*** - * Call all registered actions for a given phase in a given step - * @param step - * @param phase (Default the first phase) - */ - void CallActionsForStep(steps::Step step, Phase phase = 0); - - /*** - * Call all actions for all phases - */ - void CallActions(); - - /*** - * Call actions in a different order: do all steps for each concerns separately - */ - void CallActionsSeparatedConcerns(); - - /*** - * Call actions which do not belong to phases: - * the special actions which occur before all phases and after all phases - * @param step - */ - void CallSpecialAction(steps::Step step); - - /*** - * Get the total number of registered concerns - * @return total number of registered concerns - */ - unsigned int ConcernCount() const; - - /*** - * Get the total number of registered actions - * @return total number of registered actions - */ - unsigned int ActionCount() const; - - private: - std::vector registry; // one registry for each phase - std::vector concerns; // can't be a set as must be order-stable - reporting::Timers *timers; - void StartTimer(steps::Step step); - void StopTimer(steps::Step step); - const bool separate_concerns; - - }; - } - } -} -#endif //ONCE diff --git a/Code/net/phased/steps.h b/Code/net/phased/steps.h deleted file mode 100644 index df6befe45..000000000 --- a/Code/net/phased/steps.h +++ /dev/null @@ -1,36 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_NET_PHASED_STEPS_H -#define HEMELB_NET_PHASED_STEPS_H - -namespace hemelb -{ - namespace net - { - namespace phased - { - namespace steps - { - enum Step - { - // Order significant here - // BeginPhase must begin and EndPhase must end, those steps which should be called for a given phase. - BeginAll = -1, // Called only before first phase - BeginPhase = 0, - Receive = 1, - PreSend = 2, - Send = 3, - PreWait = 4, - Wait = 5, - EndPhase = 6, - EndAll = 7, // Called only after final phase - }; - } - } - } -} -#endif diff --git a/Code/reporting/BuildInfo.h.in b/Code/reporting/BuildInfo.h.in index c4a19e155..7c433d120 100644 --- a/Code/reporting/BuildInfo.h.in +++ b/Code/reporting/BuildInfo.h.in @@ -12,8 +12,6 @@ namespace hemelb namespace reporting { static const std::string mercurial_revision_number="@HEMELB_REVISION_NUMBER@"; - static const std::string steering_lib="@HEMELB_STEERING_LIB@"; - static const std::string streaklines_on="@HEMELB_USE_STREAKLINES@"; static const std::string build_type="@CMAKE_BUILD_TYPE@"; static const std::string optimisation="@HEMELB_OPTIMISATION@"; static const std::string use_sse3="@HEMELB_USE_SSE3@"; @@ -27,8 +25,6 @@ namespace hemelb static const std::string wall_inlet_boundary_condition="@HEMELB_WALL_INLET_BOUNDARY@"; static const std::string wall_outlet_boundary_condition="@HEMELB_WALL_OUTLET_BOUNDARY@"; static const std::string point_point_impl="@HEMELB_POINTPOINT_IMPLEMENTATION@"; - static const std::string gathers_impl="@HEMELB_GATHERS_IMPLEMENTATION@"; - static const std::string alltoall_impl="@HEMELB_ALLTOALL_IMPLEMENTATION@"; static const std::string separate_concerns="@HEMELB_SEPARATE_CONCERNS@"; @@ -36,8 +32,6 @@ namespace hemelb void Report(ctemplate::TemplateDictionary& dictionary){ ctemplate::TemplateDictionary *build = dictionary.AddSectionDictionary("BUILD"); build->SetValue("REVISION", mercurial_revision_number); - build->SetValue("STEERING", steering_lib); - build->SetValue("STREAKLINES", streaklines_on); build->SetValue("TYPE", build_type); build->SetValue("OPTIMISATION", optimisation); build->SetValue("USE_SSE3", use_sse3); @@ -51,11 +45,9 @@ namespace hemelb build->SetValue("WALL_INLET_BOUNDARY_CONDITION", wall_inlet_boundary_condition); build->SetValue("WALL_OUTLET_BOUNDARY_CONDITION", wall_outlet_boundary_condition); build->SetValue("SEPARATE_CONCERNS",separate_concerns); - build->SetValue("ALLTOALL_IMPLEMENTATION",alltoall_impl); - build->SetValue("GATHERS_IMPLEMENTATION",gathers_impl); build->SetValue("POINTPOINT_IMPLEMENTATION",point_point_impl); } }; } } -#endif // HEMELB_REPORTING_BUILDINFO_H_IN \ No newline at end of file +#endif // HEMELB_REPORTING_BUILDINFO_H_IN diff --git a/Code/reporting/Policies.h b/Code/reporting/Policies.h index b9fbdc37d..0858303db 100644 --- a/Code/reporting/Policies.h +++ b/Code/reporting/Policies.h @@ -15,8 +15,8 @@ */ #include -#include "net/mpi.h" -#include "net/IOCommunicator.h" +//#include "net/mpi.h" +//#include "net/IOCommunicator.h" #include "util/utilityFunctions.h" namespace hemelb @@ -24,49 +24,49 @@ namespace hemelb namespace reporting { - /** - * Abstraction of interaction with MPI system. - * Mocked by hemelb::unittests::reporting::MPICommsMock - */ - class MPICommsPolicy - { - public: - /** - * Stores a pointer to the MPI Network topology singleton. - */ - MPICommsPolicy(const net::IOCommunicator& inst_) : - instance(inst_) - { - } - protected: - /** - * Wrap the MPI reduce call. - * @param sendbuf Pointer to start of send buffer - * @param recvbuf Pointer to start of receive buffer - * @param count Number of data to send/receive - * @param datatype MPI Datatype - * @param op Operator to reduce under - * @param root Target processor where the reduction is reduced to. - * @param comm MPI Communicator to reduce over - * @return - */ - int Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root) - { - return MPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, instance); - } + // /** + // * Abstraction of interaction with MPI system. + // * Mocked by hemelb::unittests::reporting::MPICommsMock + // */ + // class MPICommsPolicy + // { + // public: + // /** + // * Stores a pointer to the MPI Network topology singleton. + // */ + // MPICommsPolicy(const comm::Communicator* inst_) : + // instance(inst_) + // { + // } + // protected: + // /** + // * Wrap the MPI reduce call. + // * @param sendbuf Pointer to start of send buffer + // * @param recvbuf Pointer to start of receive buffer + // * @param count Number of data to send/receive + // * @param datatype MPI Datatype + // * @param op Operator to reduce under + // * @param root Target processor where the reduction is reduced to. + // * @param comm MPI Communicator to reduce over + // * @return + // */ + // int Reduce(void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, int root) + // { + // return MPI_Reduce(sendbuf, recvbuf, count, datatype, op, root, instance); + // } - /** - * Total number of MPI nodes in the communicator. - * @return Total number of processors, some of which may be shared on a single machine. - */ - proc_t GetProcessorCount() const - { - return instance.Size(); - } + // /** + // * Total number of MPI nodes in the communicator. + // * @return Total number of processors, some of which may be shared on a single machine. + // */ + // proc_t GetProcessorCount() const + // { + // return instance->Size(); + // } - private: - const net::IOCommunicator& instance; //! Reference to the singleton instance of the MPI topology - }; + // private: + // const comm::Communicator* instance; //! Reference to the singleton instance of the MPI topology + // }; /** * A way to get the time. diff --git a/Code/reporting/Reporter.cc b/Code/reporting/Reporter.cc index c9a370be6..dfdfd58d3 100644 --- a/Code/reporting/Reporter.cc +++ b/Code/reporting/Reporter.cc @@ -11,16 +11,11 @@ namespace hemelb namespace reporting { Reporter::Reporter(const std::string &apath, const std::string &inputFile) : - path(apath), imageCount(0), dictionary("Reporting dictionary") + path(apath), dictionary("Reporting dictionary") { dictionary.SetValue("CONFIG", inputFile); } - void Reporter::Image() - { - imageCount++; - } - void Reporter::AddReportable(Reportable* reportable) { reportableObjects.push_back(reportable); @@ -39,8 +34,6 @@ namespace hemelb void Reporter::FillDictionary() { - dictionary.SetIntValue("IMAGES", imageCount); - for (std::vector::iterator reporters = reportableObjects.begin(); reporters != reportableObjects.end(); reporters++) { diff --git a/Code/reporting/Reporter.h b/Code/reporting/Reporter.h index 1f25ae8fa..6761f5d45 100644 --- a/Code/reporting/Reporter.h +++ b/Code/reporting/Reporter.h @@ -40,7 +40,6 @@ namespace hemelb * @param aState Reference to state of ongoing simulation. */ Reporter(const std::string &path, const std::string &inputFile); - void Image(); //! Inform the reporter that an image has been saved. void AddReportable(Reportable* reportable); @@ -65,7 +64,6 @@ namespace hemelb private: const std::string &path; void Write(const std::string &ctemplate, const std::string &as); //! Write the report to disk, (or wherever the WriterPolicy decides.) - unsigned int imageCount; //! Number of images written. ctemplate::TemplateDictionary dictionary; std::vector reportableObjects; }; diff --git a/Code/reporting/Timers.cc b/Code/reporting/Timers.cc index 267ccf07b..916679040 100644 --- a/Code/reporting/Timers.cc +++ b/Code/reporting/Timers.cc @@ -9,6 +9,6 @@ namespace hemelb { namespace reporting { - template class TimersBase; // explicit instantiate + template class TimersBase; // explicit instantiate } } diff --git a/Code/reporting/Timers.h b/Code/reporting/Timers.h index ee22d3591..ed3dbcc41 100644 --- a/Code/reporting/Timers.h +++ b/Code/reporting/Timers.h @@ -11,6 +11,8 @@ #include "reporting/Reportable.h" #include "util/utilityFunctions.h" #include "reporting/Policies.h" +#include "comm/Communicator.h" + namespace hemelb { namespace reporting @@ -69,10 +71,9 @@ namespace hemelb /** * Class which manages a set of timers timing aspects of a HemeLB run * @tparam ClockPolicy How to get the current time - * @tparam CommsPolicy How to share information between processes */ - template - class TimersBase : public CommsPolicy, public Reportable + template + class TimersBase : public Reportable { public: typedef TimerBase Timer; @@ -92,7 +93,6 @@ namespace hemelb latDatInitialise, //!< Time spent initialising the lattice data lb, //!< Time spent doing the core lattice boltzman simulation lb_calc, //!< Time spent doing calculations in the core lattice boltzmann simulation - visualisation, //!< Time spent on visualisation monitoring, //!< Time spent monitoring for stability, compressibility, etc. mpiSend, //!< Time spent sending MPI data mpiWait, //!< Time spent waiting for MPI @@ -102,7 +102,6 @@ namespace hemelb readBlock, readBlocksPrelim, readBlocksAll, - steeringWait, //!< Time spent waiting for a steering client in wait on connect mode. moveForcingNumbers, moveForcingData, blockRequirements, @@ -127,8 +126,7 @@ namespace hemelb */ static const std::string timerNames[TimersBase::numberOfTimers]; - TimersBase(const net::IOCommunicator& comms) : - CommsPolicy(comms), + TimersBase(comm::Communicator::ConstPtr commPtr) : comms(commPtr), timers(numberOfTimers), maxes(numberOfTimers), mins(numberOfTimers), means(numberOfTimers) { } @@ -203,21 +201,22 @@ namespace hemelb void Report(ctemplate::TemplateDictionary& dictionary); private: + comm::Communicator::ConstPtr comms; std::vector timers; //! The set of timers std::vector maxes; //! Max across processes std::vector mins; //! Min across processes std::vector means; //! Average across processes }; typedef TimerBase Timer; - typedef TimersBase Timers; + typedef TimersBase Timers; - template - const std::string TimersBase::timerNames[TimersBase::numberOfTimers] = + template + const std::string TimersBase::timerNames[TimersBase::numberOfTimers] = { "Total", "Seed Decomposition", "Domain Decomposition", "File Read", "Re Read", "Unzip", "Moves", "Parmetis", - "Lattice Data initialisation", "Lattice Boltzmann", "LB calc only", "Visualisation", "Monitoring", "MPI Send", + "Lattice Data initialisation", "Lattice Boltzmann", "LB calc only", "Monitoring", "MPI Send", "MPI Wait", "Simulation total", "Reading communications", "Parsing", "Read IO", "Read Blocks prelim", - "Read blocks all", "Steering Client Wait", "Move Forcing Counts", "Move Forcing Data", "Block Requirements", + "Read blocks all", "Move Forcing Counts", "Move Forcing Data", "Block Requirements", "Move Counts Sending", "Move Data Sending", "Populating moves list for decomposition optimisation", "Initial geometry reading", "Colloid initialisation", "Colloid position communication", "Colloid velocity communication", "Colloid force calculations", "Colloid calculations for updating", diff --git a/Code/reporting/Timers.hpp b/Code/reporting/Timers.hpp index 18f574867..ff0c89aa2 100644 --- a/Code/reporting/Timers.hpp +++ b/Code/reporting/Timers.hpp @@ -8,48 +8,33 @@ #define HEMELB_REPORTING_TIMERS_HPP #include "reporting/Timers.h" +#include namespace hemelb { namespace reporting { - template - void TimersBase::Reduce() + template + void TimersBase::Reduce() { - double timings[numberOfTimers]; + std::vector timings(numberOfTimers); for (unsigned int ii = 0; ii < numberOfTimers; ii++) { timings[ii] = timers[ii].Get(); } - CommsPolicy::Reduce(timings, - &maxes[0], - numberOfTimers, - net::MpiDataType(), - MPI_MAX, - 0); - CommsPolicy::Reduce(timings, - &means[0], - numberOfTimers, - net::MpiDataType(), - MPI_SUM, - 0); - CommsPolicy::Reduce(timings, - &mins[0], - numberOfTimers, - net::MpiDataType(), - MPI_MIN, - 0); - for (unsigned int ii = 0; ii < numberOfTimers; ii++) - { - means[ii] /= double(CommsPolicy::GetProcessorCount()); - } + maxes = comms->Reduce(timings, MPI_MAX, 0); + means = comms->Reduce(timings, MPI_SUM, 0); + mins = comms->Reduce(timings, MPI_MIN, 0); + const int np = comms->Size(); + std::transform(means.begin(), means.end(), means.begin(), + [&](double x) { return x / np; }); } - template - void TimersBase::Report(ctemplate::TemplateDictionary& dictionary) + template + void TimersBase::Report(ctemplate::TemplateDictionary& dictionary) { - dictionary.SetIntValue("THREADS", CommsPolicy::GetProcessorCount()); + dictionary.SetIntValue("THREADS", comms->Size()); for (unsigned int ii = 0; ii < numberOfTimers; ii++) { diff --git a/Code/resources/report.txt.ctp b/Code/resources/report.txt.ctp index 8390d6ea0..a5c53bc7f 100644 --- a/Code/resources/report.txt.ctp +++ b/Code/resources/report.txt.ctp @@ -27,8 +27,6 @@ Name Local Min Mean Max {{#BUILD}} Revision number:{{REVISION}} -Steering mode: {{STEERING}} -Streaklines: {{STREAKLINES}} Build type: {{TYPE}} Optimisation level: {{OPTIMISATION}} Use SSE3: {{USE_SSE3}} @@ -42,7 +40,5 @@ Wall/iolet boundary condition: {{WALL_IOLET_BOUNDARY_CONDITION}} Communications options: Point to point implementation: {{POINTPOINT_IMPLEMENTATION}} -All to all implementation: {{ALLTOALL_IMPLEMENTATION}} -Gathers implementation: {{GATHERS_IMPLEMENTATION}} Separated concerns: {{SEPARATE_CONCERNS}} {{/BUILD}} \ No newline at end of file diff --git a/Code/resources/report.xml.ctp b/Code/resources/report.xml.ctp index 17cc7e052..635ecf1ac 100644 --- a/Code/resources/report.xml.ctp +++ b/Code/resources/report.xml.ctp @@ -10,8 +10,6 @@ {{#BUILD}} {{REVISION}} - {{STEERING}} - {{STREAKLINES}} {{TYPE}} {{OPTIMISATION}} {{USE_SSE3}} @@ -27,8 +25,6 @@ {{POINTPOINT_IMPLEMENTATION}} - {{ALLTOALL_IMPLEMENTATION}} - {{GATHERS_IMPLEMENTATION}} {{SEPARATE_CONCERNS}} @@ -79,4 +75,4 @@ {{/TIMER}} - \ No newline at end of file + diff --git a/Code/steering/CMakeLists.txt b/Code/steering/CMakeLists.txt deleted file mode 100644 index 2f7d26bc7..000000000 --- a/Code/steering/CMakeLists.txt +++ /dev/null @@ -1,50 +0,0 @@ - -# This file is part of HemeLB and is Copyright (C) -# the HemeLB team and/or their institutions, as detailed in the -# file AUTHORS. This software is provided under the terms of the -# license in the file LICENSE. -set(HEMELB_STEERING_HOST "CCS" CACHE STRING "Use a default host suffix for steering? (CCS, NGS2Leeds, NGS2Manchester, LONI, NCSA or blank)") -if(HEMELB_STEERING_HOST) - add_definitions("-D${HEMELB_STEERING_HOST}") -endif() - -if(HEMELB_STEERING_LIB MATCHES [Bb]asic) - find_package (Threads) - set(steerers - basic/ClientConnection.cc - basic/HttpPost.cc - basic/ImageSendComponent.cc - basic/Network.cc - basic/SimulationParameters.cc - basic/SteeringComponentB.cc - ) - add_definitions(-DHEMELB_STEERING_LIB=basic) -elseif(HEMELB_STEERING_LIB MATCHES [Nn]one) - set(steerers - none/ClientConnection.cc - none/ImageSendComponent.cc - none/Network.cc - none/SteeringComponentN.cc - ) - add_definitions(-DHEMELB_STEERING_LIB=none) -else() - message("Unrecognised steering mode, using basic") - set(steerers - basic/ClientConnection.cc - basic/HttpPost.cc - basic/ImageSendComponent.cc - basic/Network.cc - basic/SimulationParameters.cc - basic/SteeringComponentB.cc - ) - add_definitions(-DHEMELB_STEERING_LIB=basic) -endif() - -add_library(hemelb_steering - #common/Steerer.cc # Not used in old nrmake build either -- TODO find out why - common/SteeringComponentC.cc - #common/Tags.cc - ${steerers} -) - -target_link_libraries(hemelb_steering ${CMAKE_THREAD_LIBS_INIT}) diff --git a/Code/steering/ClientConnection.h b/Code/steering/ClientConnection.h deleted file mode 100644 index 308d2f7d1..000000000 --- a/Code/steering/ClientConnection.h +++ /dev/null @@ -1,45 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_CLIENTCONNECTION_H -#define HEMELB_STEERING_CLIENTCONNECTION_H - -#include -#include -#include "reporting/Timers.h" - -namespace hemelb -{ - namespace steering - { - class ClientConnection - { - public: - ClientConnection(int iSteeringSessionId, reporting::Timers & timings); - ~ClientConnection(); - - int GetWorkingSocket(); - - void ReportBroken(int iSocketNum); - - private: - static const in_port_t MYPORT = 65250; - static const unsigned int CONNECTION_BACKLOG = 10; - - int mCurrentSocket; - int mListeningSocket; - bool mIsBroken; - // Use a semaphore to make sure that we don't create two new connections - // when a broken one is reported simultaneously by two separate threads - // (for example). - std::mutex mIsBusy; - reporting::Timers & timers; - - }; - } -} - -#endif /* HEMELB_STEERING_CLIENTCONNECTION_H */ diff --git a/Code/steering/ImageSendComponent.h b/Code/steering/ImageSendComponent.h deleted file mode 100644 index 73d395a61..000000000 --- a/Code/steering/ImageSendComponent.h +++ /dev/null @@ -1,66 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_IMAGESENDCOMPONENT_H -#define HEMELB_STEERING_IMAGESENDCOMPONENT_H - -#include "constants.h" -#include "lb/SimulationState.h" -#include "lb/LbmParameters.h" -#include "steering/Network.h" -#include "steering/basic/SimulationParameters.h" -#include "vis/Control.h" - -namespace hemelb -{ - namespace steering - { - class ImageSendComponent - { - public: - ImageSendComponent(lb::SimulationState* iSimState, - vis::Control* iControl, - const lb::LbmParameters* iLbmParams, - Network* iNetwork, - unsigned inletCount); - ~ImageSendComponent(); - - void DoWork(const vis::PixelSet* pix); - void SetMaxFramerate(float maxFramerate){ - MaxFramerate=maxFramerate; - } - bool ShouldRenderNewNetworkImage(); - - bool isConnected; - int send_array_length; - - private: - Network* mNetwork; - lb::SimulationState* mSimState; - vis::Control* mVisControl; - const unsigned inletCount; - float MaxFramerate; - char* xdrSendBuffer; - double lastRender; - - // data per pixel is - // 1 * int (pixel index) - // 3 * int (pixel RGB) - static const int bytes_per_pixel_data = 4 * 4; - - // Sent data: - // 2 * int (pixelsX, pixelsY) - // 1 * int (bytes of pixel data) - // pixel data (variable, up to COLOURED_PIXELS_MAX * bytes_per_pixel_data) - // SimulationParameters::paramsSizeB (metadata - mouse pressure and stress etc) - static const unsigned int XdrIntLength = 4; - static const unsigned int maxSendSize = 2 * XdrIntLength + 1 * XdrIntLength - + vis::Screen::COLOURED_PIXELS_MAX * bytes_per_pixel_data + SimulationParameters::paramsSizeB; - }; - } -} - -#endif /* HEMELB_STEERING_IMAGESENDCOMPONENT_H */ diff --git a/Code/steering/Network.h b/Code/steering/Network.h deleted file mode 100644 index 2944e1a5e..000000000 --- a/Code/steering/Network.h +++ /dev/null @@ -1,54 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_NETWORK_H -#define HEMELB_STEERING_NETWORK_H - -#include - -#include "steering/ClientConnection.h" -#include "net/IteratedAction.h" - -namespace hemelb -{ - namespace steering - { - - class Network : public net::IteratedAction - { - public: - Network(int iSteeringSessionId, reporting::Timers & timings); - - // Receive a bytestream of known length from a socket into a buffer. - bool recv_all(char *buf, const int length); - - // Send all bytes from a buffer of known length over a socket. - bool send_all(const char *buf, const int length); - - /** - * Use the time between MPI send and MPI receives being completed to perform the - * Network sends. - */ - void PreReceive(); - - bool IsConnected(); - - private: - void Break(int socket); - - long sendInternal(const char* data, long length, int socket); - - ClientConnection clientConnection; - - // Buffers to keep the data from partial sends and receives. - std::string sendBuf; - std::string recvBuf; - }; - - } -} - -#endif // HEMELB_STEERING_NETWORK_H diff --git a/Code/steering/SteeringComponent.h b/Code/steering/SteeringComponent.h deleted file mode 100644 index 91a8d7917..000000000 --- a/Code/steering/SteeringComponent.h +++ /dev/null @@ -1,104 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_STEERINGCOMPONENT_H -#define HEMELB_STEERING_STEERINGCOMPONENT_H - -#include "net/PhasedBroadcastRegular.h" -#include "lb/SimulationState.h" -#include "configuration/SimConfig.h" -#include "steering/Network.h" -#include "vis/DomainStats.h" -#include "vis/Control.h" -#include "steering/ImageSendComponent.h" - -namespace hemelb -{ - namespace steering - { - enum parameter - { - SceneCentreX = 0, - SceneCentreY = 1, - SceneCentreZ = 2, - Longitude = 3, - Latitude = 4, - Zoom = 5, - Brightness = 6, - PhysicalVelocityThresholdMax = 7, - PhysicalStressThrehsholdMaximum = 8, - PhysicalPressureThresholdMinimum = 9, - PhysicalPressureThresholdMaximum = 10, - GlyphLength = 11, - PixelsX = 12, - PixelsY = 13, - NewMouseX = 14, - NewMouseY = 15, - SetIsTerminal = 16, - Mode = 17, - StreaklinePerSimulation = 18, - StreaklineLength = 19, - MaxFramerate=20, - SetDoRendering = 21 - }; - - /** - * SteeringComponent - class for passing steering data to all nodes. - * - * We pass this data at regular intervals. No initial action is required by all nodes, and - * we only need to pass from the top-most node (which handles network communication) downwards, - * on one iteration between each pair of consecutive depths. - */ - class SteeringComponent : public net::PhasedBroadcastRegular - { - public: - SteeringComponent(Network* iNetwork, - vis::Control* iVisControl, - steering::ImageSendComponent* imageSendComponent, - net::Net * iNet, - lb::SimulationState * iSimState, - configuration::SimConfig* iSimConfig, - const util::UnitConverter* iUnits); - - static bool RequiresSeparateSteeringCore(); - - /* - * This function initialises all of the steering parameters, on all nodes. - * Although this appears to be part of the removed post-unstable reset functionality, it is also used during initialisation - * so is retained #244 - */ - void ClearValues(); - - bool readyForNextImage; - bool updatedMouseCoords; - - protected: - void ProgressFromParent(unsigned long splayNumber); - void ProgressToChildren(unsigned long splayNumber); - - void TopNodeAction(); - void Effect(); - - private: - void AssignValues(); - - const static int STEERABLE_PARAMETERS = 21; - const static unsigned int SPREADFACTOR = 10; - - bool isConnected; - - Network* mNetwork; - lb::SimulationState* mSimState; - vis::Control* mVisControl; - steering::ImageSendComponent* imageSendComponent; - float privateSteeringParams[STEERABLE_PARAMETERS + 1]; - const util::UnitConverter* mUnits; - configuration::SimConfig* simConfig; - }; - } -} - -#endif /* HEMELB_STEERING_STEERINGCOMPONENT_H */ diff --git a/Code/steering/basic/ClientConnection.cc b/Code/steering/basic/ClientConnection.cc deleted file mode 100644 index d11415039..000000000 --- a/Code/steering/basic/ClientConnection.cc +++ /dev/null @@ -1,167 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#include -#include -#include -#include - -#include "debug/Debugger.h" -#include "log/Logger.h" -#include "steering/ClientConnection.h" -#include "steering/basic/HttpPost.h" - -namespace hemelb -{ - namespace steering - { - ClientConnection::ClientConnection(int iSteeringSessionId, reporting::Timers & timings) - :mIsBusy(), timers(timings) - { - // Write the name of this machine to a file. - - { - char thisMachineName[255]; - gethostname(thisMachineName, 255); - std::FILE *f = std::fopen("env_details.asc", "w"); - std::fprintf(f, "%s\n", thisMachineName); - std::fclose(f); - } - - mCurrentSocket = -1; - mIsBroken = false; - - // Create the socket. - mListeningSocket = socket(AF_INET, SOCK_STREAM, 0); - if (mListeningSocket == -1) - { - perror("socket"); - exit(1); - } - - // Make the socket reusable. - int yes = 1; - if (setsockopt(mListeningSocket, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(int)) == -1) - { - perror("setsockopt"); - exit(1); - } - - // Bind to the socket. - { - struct sockaddr_in my_address; - - my_address.sin_family = AF_INET; - my_address.sin_port = htons((in_port_t) MYPORT); - my_address.sin_addr.s_addr = INADDR_ANY; - memset(my_address.sin_zero, '\0', sizeof my_address.sin_zero); - - if (bind(mListeningSocket, (struct sockaddr *) &my_address, sizeof my_address) == -1) - { - perror("bind"); - exit(1); - } - } - - // Mark the socket as accepting incoming connections. - if (listen(mListeningSocket, CONNECTION_BACKLOG) == -1) - { - perror("listen"); - exit(1); - } - } - - ClientConnection::~ClientConnection() - { - close(mCurrentSocket); - close(mListeningSocket); - } - - int ClientConnection::GetWorkingSocket() - { - // Lock the mutex and release it when this goes out of scope - std::lock_guard lock(mIsBusy); - - // If we haven't yet had a socket, or the current one is broken, open a new one. - if (mCurrentSocket < 0 || mIsBroken) - { - // Accept an incoming connection from the client. - struct sockaddr_in clientAddress; - socklen_t socketSize = sizeof (clientAddress); - - int lOldSocket = mCurrentSocket; - - // Make the socket non-blocking, just while we try to accept on it, then set - // it back to what it was before. - { - int flags = fcntl(mListeningSocket, F_GETFL, 0); - if (flags == -1) - { - flags = 0; - } -#ifndef HEMELB_WAIT_ON_CONNECT - if (fcntl(mListeningSocket, F_SETFL, flags | O_NONBLOCK) < 0) - { - perror("flags"); - } -#else - log::Logger::Log("Waiting for steering client connection"); - timers[reporting::Timers::steeringWait].Start(); - -#endif - // Try to accept a socket (from the non-blocking socket) - mCurrentSocket - = accept(mListeningSocket, (struct sockaddr *) &clientAddress, &socketSize); -#ifdef HEMELB_WAIT_ON_CONNECT - timers[reporting::Timers::steeringWait].Stop(); - log::Logger::Log("Continuing after receiving steering connection."); -#endif - // We've got a socket - make that socket non-blocking too. - if (mCurrentSocket > 0) - { - log::Logger::Log("Steering client connected"); - flags = fcntl(mCurrentSocket, F_GETFL, 0); - if (flags == -1) - { - flags = 0; - } - if (fcntl(mCurrentSocket, F_SETFL, flags | O_NONBLOCK) < 0) - { - perror("flags"); - } - } - } - - // If we had a socket before, close it. - if (lOldSocket > 0) - { - close(lOldSocket); - } - - // We've only just created the socket so it shouldn't be broken. - mIsBroken = false; - } - - int lRet = mCurrentSocket; - - return lRet; - } - - void ClientConnection::ReportBroken(int iSocketNum) - { - // Lock the mutex and release it when this goes out of scope - std::lock_guard lock(mIsBusy); - if (mCurrentSocket == iSocketNum) - { - mIsBroken = true; - } - } - - } -} diff --git a/Code/steering/basic/HttpPost.cc b/Code/steering/basic/HttpPost.cc deleted file mode 100644 index 6c3145dec..000000000 --- a/Code/steering/basic/HttpPost.cc +++ /dev/null @@ -1,277 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "debug/Debugger.h" -#include "log/Logger.h" -#include "steering/basic/HttpPost.h" - -using namespace std; - -namespace hemelb -{ - namespace steering - { - - void HttpPost::get_host_details(char* rank_0_host_details) - { - // This really does modify the strings passed in. Without - // checking their lengths, of course. - char hostname[256]; - - // On specific machines, just get the host name and insert it into the rank_0_host_details parameter -#ifdef NGS2Leeds - gethostname(hostname, 256); - std::sprintf(rank_0_host_details, "%s.ngs.leeds.ac.uk:%i", hostname, MYPORT); -#elif NGS2Manchester - gethostname(hostname, 256); - std::sprintf(rank_0_host_details, "%s.vidar.ngs.manchester.ac.uk:%i", hostname, MYPORT); -#elif CCS - gethostname(hostname, 256); - std::sprintf(rank_0_host_details, "%s.chem.ucl.ac.uk:%i", hostname, MYPORT); -#elif LONI - gethostname(hostname, 256); - std::sprintf(rank_0_host_details, "%s:%i", hostname, MYPORT); -#elif NCSA - gethostname(hostname, 256); - std::sprintf(rank_0_host_details, "%s.ncsa.teragrid.org:%i", hostname, MYPORT); -#else - - // If not on a specific known machine, need to do something more clever. - struct utsname name; - - if (uname(&name) < 0) - { - std::fprintf(stderr, "STEER: Get_fully_qualified_hostname: uname failed\n"); - exit(0x0); - } - - // Get information about our host. - struct hostent *host = gethostbyname(name.nodename); - char ip_addr[16]; - - // Now go through every associated IP adress, and use the first valid, public one. - for (int address_id = 0; address_id < 4; address_id++) - { - // If no address, break. - if (host->h_addr_list[address_id] == NULL) - break; - - std::printf("checking Address ID %i...\n", address_id); - - // If IP4, print details. - if (host->h_length != 4) - { - std::printf("address is not IP4..\n"); - continue; - } - - std::sprintf(ip_addr, - "%d.%d.%d.%d", - (unsigned char) (host->h_addr_list[address_id][0]), - (unsigned char) (host->h_addr_list[address_id][1]), - (unsigned char) (host->h_addr_list[address_id][2]), - (unsigned char) (host->h_addr_list[address_id][3])); - - std::printf("NAME %s IP %s\n", host->h_name, ip_addr); - - // Private addresses (see RFC-1918). - if ((unsigned char) (host->h_addr_list[address_id][0]) == 10) - { - std::printf("IP %s is not public..\n", ip_addr); - continue; - } - - // Loopback addresses. - if ((unsigned char) (host->h_addr_list[address_id][0]) == 127) - { - std::printf("IP %s is not public..\n", ip_addr); - continue; - } - - // Private addresses. - if ((unsigned char) (host->h_addr_list[address_id][0]) == 172 - && (unsigned char) (host->h_addr_list[address_id][1]) >= 16 - && (unsigned char) (host->h_addr_list[address_id][1]) < 32) - { - std::printf("IP %s is not public..\n", ip_addr); - continue; - } - - // Private IP addresses. - if ((unsigned char) (host->h_addr_list[address_id][0]) == 192 - && (unsigned char) (host->h_addr_list[address_id][1]) == 168) - { - std::printf("IP %s is not public..\n", ip_addr); - continue; - } - } - - std::sprintf(rank_0_host_details, "%s:%i (IP %s)", host->h_name, MYPORT, ip_addr); -#endif - - log::Logger::Log("MPI public interface details - %s", rank_0_host_details); - } - - ssize_t HttpPost::Send_Request(int iSocket, const char *iMessage) - { - return send(iSocket, iMessage, strlen(iMessage), 0); - } - - int HttpPost::request(const char* hostname, const in_port_t port, const char* api, const char* resourceid) - { - // Get the host name to communicate with. - char host_name[1024]; - get_host_details(host_name); - - // Attempt to get a socket to use. - int sock = socket(AF_INET, SOCK_STREAM, 0); - if (sock == -1) - { - return -100; - } - - sockaddr_in sin; - sin.sin_family = AF_INET; - sin.sin_port = htons((in_port_t) port); - - // Get name for the other end of the connection. - struct hostent * host_addr = gethostbyname(hostname); - - if (host_addr == NULL) - { - return -103; - } - - sin.sin_addr.s_addr = * ((int*) *host_addr->h_addr_list); - - // Attempt to connect, but don't try for too long. - timeval tv; - memset(&tv, 0, sizeof (tv)); // so valgrind knows the whole struct is initialised. - tv.tv_sec = 2; - tv.tv_usec = 0; - - setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof (tv)); - setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof (tv)); - - if (connect(sock, (const struct sockaddr *) &sin, sizeof(sockaddr_in)) == -1) - { - return -101; - } - - // Now we perform the actual sending. - Send_Request(sock, "POST "); - Send_Request(sock, api); - Send_Request(sock, resourceid); - Send_Request(sock, " HTTP/1.0\r\n"); - Send_Request(sock, "Accept: */*\r\n"); - Send_Request(sock, "User-Agent: Mozilla/4.0\r\n"); - - char content_header[100]; - std::sprintf(content_header, "Content-Length: %d\r\n", int (std::strlen(host_name))); - - Send_Request(sock, content_header); - Send_Request(sock, "Accept-Language: en-us\r\n"); - Send_Request(sock, "Accept-Encoding: gzip, deflate\r\n"); - Send_Request(sock, "Host: "); - Send_Request(sock, "hostname"); - Send_Request(sock, "\r\n"); - Send_Request(sock, "Content-Type: text/plain\r\n"); - Send_Request(sock, "\r\n"); - Send_Request(sock, host_name); - Send_Request(sock, "\r\n"); - - /* If you need to send a basic authorization - * string Auth = "username:password"; - * Figureout a way to encode test into base64 ! - * string AuthInfo = base64_encode(reinterpret_cast(Auth.c_str()),Auth.length()); - * string sPassReq = "Authorization: Basic " + AuthInfo; - * Send_Request(sock, sPassReq.c_str()); - * */ - - // Receive 1 character at a time until a whole response is recovered. - int line_length = 0; - bool loop = true; - bool bHeader = false; - - string message; - - while (loop) - { - char c1[1]; - - // receive 1 char from the socket - ssize_t l = recv(sock, c1, 1, 0); - - // An error occurred. - if (l < 0) - loop = false; - - if (c1[0] == '\n') - { - // Received a newline. If the only character on this line, we're at the end of the - // response. - if (line_length == 0) - loop = false; - - line_length = 0; - - // if the message contains a success response code, we are in the header. - if (message.find("201 Created") != string::npos) - bHeader = true; - } - else if (c1[0] != '\r') - { - // Didn't get newline and didn't get carriage return. - line_length++; - } - - message += c1[0]; - } - - message = ""; - - // If we are indeed reading a header, read the rest of the message. - if (bHeader) - { - char p[1024]; - - ssize_t l; - - while ( (l = recv(sock, p, 1023, 0)) > 0) - { - p[l] = '\0'; - message += p; - } - } - else - { - return -102; - } - - return 0; - } - - } -} - diff --git a/Code/steering/basic/HttpPost.h b/Code/steering/basic/HttpPost.h deleted file mode 100644 index 7775ca193..000000000 --- a/Code/steering/basic/HttpPost.h +++ /dev/null @@ -1,56 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_BASIC_HTTPPOST_H -#define HEMELB_STEERING_BASIC_HTTPPOST_H - -namespace hemelb -{ - namespace steering - { - - class HttpPost - { - public: - /** - * Send an HTTP Request. - * - * @param hostname The name of the host. - * @param port The port to send to. - * @param api The API name to include in the request. - * @param resourceid An identifier for the desired resource. - * @return Error code. 0 on success. - */ - static int request(const char* hostname, - const in_port_t port, - const char* api, - const char* resourceid); - - private: - /** - *Get details and ip address for this host. - * - * @param rank_0_host_details - * @param ip_addr - */ - static void get_host_details(char* rank_0_host_details); - - /** - * Helper method to send a request. - * - * @param iSocket - * @param iMessage - * @return - */ - static ssize_t Send_Request(int iSocket, const char *iMessage); - - static const int MYPORT = 65250; - }; - - } -} - -#endif // HEMELB_STEERING_BASIC_HTTPPOST_H diff --git a/Code/steering/basic/ImageSendComponent.cc b/Code/steering/basic/ImageSendComponent.cc deleted file mode 100644 index ea6f61b68..000000000 --- a/Code/steering/basic/ImageSendComponent.cc +++ /dev/null @@ -1,118 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include - -#include "log/Logger.h" -#include "steering/ImageSendComponent.h" -#include "steering/Network.h" -#include "io/writers/xdr/XdrMemWriter.h" -#include "util/utilityFunctions.h" - -namespace hemelb -{ - namespace steering - { - // Use initialisation list to do the work. - ImageSendComponent::ImageSendComponent(lb::SimulationState* iSimState, - vis::Control* iControl, - const lb::LbmParameters* iLbmParams, - Network* iNetwork, - unsigned inletCountIn) : - mNetwork(iNetwork), mSimState(iSimState), mVisControl(iControl), inletCount(inletCountIn), MaxFramerate(25.0) - { - xdrSendBuffer = new char[maxSendSize]; - - // Suppress signals from a broken pipe. - signal(SIGPIPE, SIG_IGN); - - isConnected = false; - lastRender = 0.0; - } - - ImageSendComponent::~ImageSendComponent() - { - delete[] xdrSendBuffer; - } - - // This is original code with minimal tweaks to make it work with - // the new (Feb 2011) structure. - void ImageSendComponent::DoWork(const vis::PixelSet* pix) - { - isConnected = mNetwork->IsConnected(); - - if (!isConnected) - { - return; - } - - io::writers::xdr::XdrMemWriter imageWriter = io::writers::xdr::XdrMemWriter(xdrSendBuffer, maxSendSize); - - unsigned int initialPosition = imageWriter.getCurrentStreamPosition(); - - // Write the dimensions of the image, in terms of pixel count. - imageWriter << mVisControl->GetPixelsX() << mVisControl->GetPixelsY(); - - // Write the length of the pixel data - imageWriter << (int) (pix->GetPixelCount() * bytes_per_pixel_data); - - // Write the pixels themselves - mVisControl->WritePixels(&imageWriter, *pix, mVisControl->domainStats, mVisControl->visSettings); - - // Write the numerical data from the simulation, wanted by the client. - { - SimulationParameters sim; - - sim.timeStep = (int) mSimState->GetTimeStep(); - sim.time = mSimState->GetTime(); - sim.nInlets = inletCount; - - sim.mousePressure = mVisControl->visSettings.mouse_pressure; - sim.mouseStress = mVisControl->visSettings.mouse_stress; - - mVisControl->visSettings.mouse_pressure = -1.0; - mVisControl->visSettings.mouse_stress = -1.0; - - sim.pack(&imageWriter); - } - - // Send to the client. - log::Logger::Log("Sending network image at timestep %d",mSimState->GetTimeStep()); - mNetwork->send_all(xdrSendBuffer, imageWriter.getCurrentStreamPosition() - initialPosition); - } - - bool ImageSendComponent::ShouldRenderNewNetworkImage() - { - isConnected = mNetwork->IsConnected(); - - if (!isConnected) - { - return false; - } - - // If we're going to exceed 25Hz by rendering now, wait until next iteration. - { - double frameTimeStart = util::myClock(); - - double deltaTime = frameTimeStart - lastRender; - - if ( (1.0 / MaxFramerate) > deltaTime) - { - return false; - } - else - { - log::Logger::Log("Image-send component requesting new render, %f seconds since last one at step %d max rate is %f.", - deltaTime, mSimState->GetTimeStep(), MaxFramerate); - lastRender = frameTimeStart; - return true; - } - } - } - - } -} diff --git a/Code/steering/basic/Network.cc b/Code/steering/basic/Network.cc deleted file mode 100644 index 370cce47b..000000000 --- a/Code/steering/basic/Network.cc +++ /dev/null @@ -1,257 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "debug/Debugger.h" -#include "log/Logger.h" -#include "steering/Network.h" -#include "util/utilityFunctions.h" - -namespace hemelb -{ - namespace steering - { - Network::Network(int steeringSessionId, reporting::Timers & timings) : - clientConnection(steeringSessionId, timings) - { - - } - - /** - * Receive a bytestream of known length from a socket into a buffer. - * - * @param sockid - * @param buf - * @param length - * @return Returns true if we have successfully provided that much data. - */ - bool Network::recv_all(char *buf, const int length) - { - // Get a socket - int socketToClient = clientConnection.GetWorkingSocket(); - - if (socketToClient < 0) - { - return false; - } - - ssize_t bytesGot = 0; - // If we have some buffered data to receive, include that in our count. - if (recvBuf.length() > 0) - { - bytesGot += recvBuf.length(); - } - - log::Logger::Log("Steering component will try to receive %d bytes, has %d so far", - length, - bytesGot); - // While some data left to be received... - while (bytesGot < length) - { - // Receive some data (up to the remaining length) - ssize_t n = recv(socketToClient, buf + bytesGot, length - bytesGot, 0); - - // If there was an error, report it and return. - if (n <= 0) - { - // If there was no data and it wasn't simply that the socket would block, - // raise an error. - if (errno != EAGAIN) - { - log::Logger::Log("Steering component: broken network pipe... (%s)", - strerror(errno)); - Break(socketToClient); - } - else - { - // If we'd already received some data, store this in the buffer for the start of the - // next receive. - if (bytesGot > 0) - { - // The newly-received-byte count is the total received byte count minus the length - // of the buffer. - long int numNewBytes = bytesGot - recvBuf.length(); - recvBuf.append(buf + recvBuf.length(), numNewBytes); - log::Logger::Log("Steering component: blocked socket"); - } - } - log::Logger::Log("Steering component exiting after incomplete reception"); - // We didn't fully receive. - return false; - } - else - { - bytesGot += n; - log::Logger::Log("Steering component: received bytes... (New total %d)", bytesGot); - } - } - log::Logger::Log("Steering component is happy with what it has received"); - // Successfully received what we needed to. Now use the buffer to fill in the gaps, if - // we were using the buffer at the front of the received data. - if (recvBuf.length() > 0) - { - // The length of buffer to use is the minimum of the buffer size and the length of - // the string. - int copyLength = util::NumericalFunctions::min((int) recvBuf.length(), length); - - memcpy(buf, recvBuf.c_str(), copyLength); - - // Empty that much of the buffer. - recvBuf.erase(0, copyLength); - } - - return true; - } - - void Network::PreReceive() - { - // Calling send_all will attempt to flush the send buffer. - send_all(NULL, 0); - } - - bool Network::IsConnected() - { - int res = clientConnection.GetWorkingSocket(); - return res > 0; - } - - /** - * Send all bytes from a buffer of known length over a socket. - * - * @param sockid - * @param buf - * @param length - * @return Returns the number of bytes sent or -1 on failure. - */ - bool Network::send_all(const char *buf, const int length) - { - // Get a socket. - int socketToClient = clientConnection.GetWorkingSocket(); - - // If there's no such socket, we don't have a connection. Nothing to do. - if (socketToClient < 0) - { - return false; - } - log::Logger::Log("Steering component will try to send %d new bytes and a buffer of %d", - length, - sendBuf.length()); - // If we have buffered strings to be sent, send those first. - if (sendBuf.length() > 0) - { - long sent = sendInternal(sendBuf.c_str(), sendBuf.length(), socketToClient); - - // Broken socket? - if (sent < 0) - { - return false; - } - - // Did sending block? We'll try again next time. - if (sent < (long) sendBuf.length()) - { - sendBuf.erase(0, sent); - - // If the sending blocks, just ignore this most recent frame. This way, we stop the - // memory usage of the steering process spiralling, and keep current the images delivered - // to the client. - // What we *would* do is sendBuf.append(buf, length); - - log::Logger::Log("Steering component could not send all buffer, managed %d bytes", - sent); - return true; - } - // If not, we sent the whole buffer. - else - { - sendBuf.clear(); - } - } - log::Logger::Log("Steering component sent all the buffer, sending new data"); - // If we sent the whole buffer, try to send the new data. - long sent_bytes = sendInternal(buf, length, socketToClient); - - // Is the socket broken? - if (sent_bytes < 0) - { - log::Logger::Log("Steering component socket broke sending new bytes"); - return false; - } - // Did the socket block? Still return true, because we'll try again next time. - else if (sent_bytes < length) - { - log::Logger::Log("Steering component socket blocked after sending %d bytes, adding %d bytes to buffer", - sent_bytes, - length - sent_bytes); - sendBuf.append(buf + sent_bytes, length - sent_bytes); - } - - // Otherwise we sent the whole thing. - return true; - } - - void Network::Break(int socket) - { - clientConnection.ReportBroken(socket); - - recvBuf.clear(); - sendBuf.clear(); - } - - /** - * Returns -1 for a broken socket, otherwise the number of bytes we could send before blocking. - * - * @param data - * @param length - * @param socket - * @return - */ - long Network::sendInternal(const char* data, long length, int socket) - { - long sent_bytes = 0; - - while (sent_bytes < length) - { - long n = send(socket, data + sent_bytes, length - sent_bytes, 0); - - if (n <= 0) - { - // Distinguish between cases where the pipe fails because it'd block - // (No problem, we'll try again later) or because the pipe is broken. - if (errno == EWOULDBLOCK) - { - return sent_bytes; - } - else - { - log::Logger::Log("Network send had broken pipe... (%s)", strerror(errno)); - Break(socket); - - return -1; - } - } - else - { - sent_bytes += n; - } - } - - return sent_bytes; - } - - } -} diff --git a/Code/steering/basic/SimulationParameters.cc b/Code/steering/basic/SimulationParameters.cc deleted file mode 100644 index 6fd7d0392..000000000 --- a/Code/steering/basic/SimulationParameters.cc +++ /dev/null @@ -1,46 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "io/writers/xdr/XdrMemWriter.h" - -#include "vis/Control.h" -#include "steering/basic/SimulationParameters.h" - -namespace hemelb -{ - namespace steering - { - - SimulationParameters::SimulationParameters() - { - // C'tor initialises to the following defaults. - - timeStep = 0; - time = 0.0; - nInlets = 3; - mousePressure = -1.0; - mouseStress = -1.0; - } - - SimulationParameters::~SimulationParameters() - { - } - - void SimulationParameters::pack(io::writers::xdr::XdrWriter* writer) - { - writer->operator <<(timeStep); - - writer->operator <<(time); - - writer->operator <<(0); // Cycle is always zero, leave this in to stop steering clients breaking. - writer->operator <<(nInlets); - - writer->operator <<(mousePressure); - writer->operator <<(mouseStress); - } - - } -} diff --git a/Code/steering/basic/SimulationParameters.h b/Code/steering/basic/SimulationParameters.h deleted file mode 100644 index a74a882cc..000000000 --- a/Code/steering/basic/SimulationParameters.h +++ /dev/null @@ -1,42 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_BASIC_SIMULATIONPARAMETERS_H -#define HEMELB_STEERING_BASIC_SIMULATIONPARAMETERS_H - -#include "io/writers/xdr/XdrMemWriter.h" -#include "lb/SimulationState.h" - -namespace hemelb -{ - namespace steering - { - - class SimulationParameters - { - - public: - - int timeStep; - double time; - int nInlets; - double mousePressure; - double mouseStress; - - static const u_int paramsSizeB = 3 * sizeof(int) + 3 * sizeof(double); - - SimulationParameters(); - ~SimulationParameters(); - void pack(io::writers::xdr::XdrWriter* writer); - - private: - - }; - - } -} - -#endif // HEMELB_STEERING_BASIC_SIMULATIONPARAMETERS_H diff --git a/Code/steering/basic/SteeringComponentB.cc b/Code/steering/basic/SteeringComponentB.cc deleted file mode 100644 index 69b6e6765..000000000 --- a/Code/steering/basic/SteeringComponentB.cc +++ /dev/null @@ -1,109 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include - -#include "steering/SteeringComponent.h" -#include "steering/Network.h" -#include "io/writers/xdr/XdrMemReader.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace steering - { - SteeringComponent::SteeringComponent(Network* iNetwork, - vis::Control* iVisControl, - steering::ImageSendComponent* imageSendComponent, - net::Net * iNet, - lb::SimulationState * iSimState, - configuration::SimConfig* iSimConfig, - const util::UnitConverter* iUnits) : - net::PhasedBroadcastRegular(iNet, iSimState, SPREADFACTOR), - mNetwork(iNetwork), mSimState(iSimState), mVisControl(iVisControl), imageSendComponent(imageSendComponent), - mUnits(iUnits),simConfig(iSimConfig) - { - ClearValues(); - AssignValues(); - } - - void SteeringComponent::ProgressFromParent(unsigned long splayNumber) - { - ReceiveFromParent(privateSteeringParams, STEERABLE_PARAMETERS + 1); - } - - void SteeringComponent::ProgressToChildren(unsigned long splayNumber) - { - SendToChildren(privateSteeringParams, STEERABLE_PARAMETERS + 1); - } - - bool SteeringComponent::RequiresSeparateSteeringCore() - { - return true; - } - - void SteeringComponent::TopNodeAction() - { - /* - * The final steering parameter is DoRendering, which is true if we're connected and ready - * for the next frame. - */ - { - privateSteeringParams[STEERABLE_PARAMETERS] = (float) (isConnected && readyForNextImage); - } - - // Create a buffer for the data received. - const int num_chars = STEERABLE_PARAMETERS * sizeof(float) / sizeof(char); - const int bytes = sizeof(char) * num_chars; - - char steeringRecvBuffer[bytes]; - - // Get the open socket. - isConnected = mNetwork->IsConnected(); - - if (!isConnected) - { - return; - } - - bool newSteeringDataExists = false; - - // Keep looping through all data that we can read - if multiple steering commands have been - // received we only want to act on the last set. - while (true) - { - // Try to receive all the data. - if (mNetwork->recv_all(steeringRecvBuffer, num_chars)) - { - // We read data! - newSteeringDataExists = true; - } - // If it wasn't readable, we've exhausted all the received data at the socket, so - // break and actually use the last read data. - else - { - break; - } - } - - if (newSteeringDataExists) - { - // Initialise the stream reader. - io::writers::xdr::XdrMemReader steeringStream(steeringRecvBuffer, bytes); - - for (int i = 0; i < STEERABLE_PARAMETERS; i++) - steeringStream.readFloat(privateSteeringParams[i]); - } - } - - void SteeringComponent::Effect() - { - // TODO we need to make sure that doing this doesn't overwrite the values in the config.xml file. - // At the moment, it definitely does. - AssignValues(); - } - } -} diff --git a/Code/steering/common/Singleton.h b/Code/steering/common/Singleton.h deleted file mode 100644 index 675154883..000000000 --- a/Code/steering/common/Singleton.h +++ /dev/null @@ -1,47 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_COMMON_SINGLETON_H -#define HEMELB_STEERING_COMMON_SINGLETON_H - -namespace hemelb -{ - namespace steering - { - namespace _private - { - /** - * Base class for singleton behaviour. - * Uses CRTP to do static polymorphism. - */ - template - class Singleton - { - protected: - static Singleton* singleton; - Singleton() - { - } - - public: - static Derived* Init() - { - if (Singleton::singleton == NULL) - { - Singleton::singleton = new Derived(); - } - return static_cast(Singleton::singleton); - } - }; - - template - Singleton* Singleton::singleton = NULL; - - } - } -} - -#endif // HEMELB_STEERING_COMMON_SINGLETON_H diff --git a/Code/steering/common/Steerable.h b/Code/steering/common/Steerable.h deleted file mode 100644 index 49927669f..000000000 --- a/Code/steering/common/Steerable.h +++ /dev/null @@ -1,74 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_COMMON_STEERABLE_H -#define HEMELB_STEERING_COMMON_STEERABLE_H - -namespace hemelb -{ - namespace steering - { - namespace _private - { - // Base class for Steerable member vars - class SteerableBase - { - }; - } - - /** - * Template class for wrappers around member vars that allows them to - * be altered by the Steerer. Template parameter should be a suitable - * subclass of Tag<>; see example in Tags.cc. - */ - - template - class Steerable : public _private::SteerableBase - { - public: - typedef T TagClass; - typedef typename TagClass::WrappedType WrappedType; - - protected: - // The value we need to steer. - WrappedType mValue; - // Constant pointer to a mutable TagClass instance. - static TagClass* const tag; - - public: - Steerable() : - mValue(TagClass::InitialValue) - { - Steerable::tag->AddInstance(this); - } - - ~Steerable() - { - tag->RemoveInstance(this); - } - - WrappedType const& Get(void) - { - return mValue; - } - - protected: - void Set(WrappedType const& value) - { - mValue = value; - } - - // It's not the Tag that needs to be our friend, it's the Tag's - // special base class. Happily this is allowed by the standard but - // friend class TagClass is not. - friend class TagClass::Super; - }; - - template T* const Steerable::tag = T::Init(); - - } -} -#endif // HEMELB_STEERING_COMMON_STEERABLE_H diff --git a/Code/steering/common/Steerer.cc b/Code/steering/common/Steerer.cc deleted file mode 100644 index 5e49e2bdb..000000000 --- a/Code/steering/common/Steerer.cc +++ /dev/null @@ -1,24 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "steering/common/Steerer.h" - -// void Steerer::RegisterTag(TagBase* tag) -// { -// mTags.insert(std::make_pair(tag.TagName, tag)); -// } -namespace hemelb -{ - namespace steering - { - namespace _private - { - Steerer::Steerer() - { - } - } - } -} diff --git a/Code/steering/common/Steerer.h b/Code/steering/common/Steerer.h deleted file mode 100644 index 8224ee3ea..000000000 --- a/Code/steering/common/Steerer.h +++ /dev/null @@ -1,90 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_COMMON_STEERER_H -#define HEMELB_STEERING_COMMON_STEERER_H - -#include -#include -#include - -#include "steering/common/Singleton.h" -#include "steering/common/Tag.h" - -/* - * Global object to control steerable parameters. - * - * To work, this must have the following methods: - * - * SteeringType* Init(void); - * - * Returns the steerer object, creating it - * if need be. Currently do this by inheriting from the Singleton - * class template. - * - * template - * void SetTag(typename TagType::WrappedType value) - * - * Get the tag type from the list stored and sets its value to the one - * given. - * - * template - * void RegisterTag(TagType* tag) - * - * Called by the Tag<> instance to register itself. Must be accessible - * from there, i.e. make that a friend. However the standard doesn't - * allow partial specializations to be declared friend so we have to - * let any Tag<> be our friend, with: - * - * template friend class Tag; - * - */ -namespace hemelb -{ - namespace steering - { - namespace _private - { - class TagBase; - - template - class Tag; - } - class Steerer : public _private::Singleton - { - public: - // Singleton exposes a method static Steerer* Init(void); - - typedef std::map ContainerType; - typedef ContainerType::value_type PairType; - - template - void SetTag(typename TagType::WrappedType value) - { - TagType* tag = static_cast(mTags[TagType::TagString]); - tag->SetInstanceValues(value); - } - - protected: - template - void RegisterTag(TagType* tag) - { - mTags.insert(PairType(TagType::TagString, static_cast<_private::TagBase*>(tag))); - } - - ContainerType mTags; - Steerer(); - - // Singleton must be a friend. - friend class _private::Singleton; - // Standard apparently doesn't allow partial specializations to be - // declared friend so we have to let any Tag<> be our friend. - template friend class _private::Tag; - }; - - } -} -#endif // HEMELB_STEERING_COMMON_STEERER_H diff --git a/Code/steering/common/SteeringComponentC.cc b/Code/steering/common/SteeringComponentC.cc deleted file mode 100644 index d983a8db3..000000000 --- a/Code/steering/common/SteeringComponentC.cc +++ /dev/null @@ -1,153 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "steering/SteeringComponent.h" - -namespace hemelb -{ - namespace steering - { - void SteeringComponent::AssignValues() - { - mVisControl->visSettings.ctr_x += privateSteeringParams[SceneCentreX]; - mVisControl->visSettings.ctr_y += privateSteeringParams[SceneCentreY]; - mVisControl->visSettings.ctr_z += privateSteeringParams[SceneCentreZ]; - - float longitude = privateSteeringParams[Longitude]; - float latitude = privateSteeringParams[Latitude]; - - float zoom = privateSteeringParams[Zoom]; - - mVisControl->visSettings.brightness = privateSteeringParams[Brightness]; - - // The minimum value here is by default 0.0 all the time - mVisControl->domainStats.physical_velocity_threshold_max = privateSteeringParams[PhysicalVelocityThresholdMax]; - - // The minimum value here is by default 0.0 all the time - mVisControl->domainStats.physical_stress_threshold_max = privateSteeringParams[PhysicalStressThrehsholdMaximum]; - - mVisControl->domainStats.physical_pressure_threshold_min = - privateSteeringParams[PhysicalPressureThresholdMinimum]; - mVisControl->domainStats.physical_pressure_threshold_max = privateSteeringParams[10]; - - mVisControl->visSettings.glyphLength = privateSteeringParams[GlyphLength]; - - float pixels_x = privateSteeringParams[PixelsX]; - float pixels_y = privateSteeringParams[PixelsY]; - - int newMouseX = int(privateSteeringParams[NewMouseX]); - int newMouseY = int(privateSteeringParams[NewMouseY]); - - if (newMouseX != mVisControl->visSettings.mouse_x || newMouseY != mVisControl->visSettings.mouse_y) - { - updatedMouseCoords = true; - mVisControl->visSettings.mouse_x = newMouseX; - mVisControl->visSettings.mouse_y = newMouseY; - } - - mSimState->SetIsTerminating(1 == (int) privateSteeringParams[SetIsTerminal]); - - // To swap between glyphs and streak line rendering... - // 0 - Only display the isosurfaces (wall pressure and stress) - // 1 - Isosurface and glyphs - // 2 - Wall pattern streak lines - mVisControl->visSettings.mode = (vis::VisSettings::Mode) (privateSteeringParams[Mode]); - - mVisControl->visSettings.streaklines_per_simulation = privateSteeringParams[StreaklinePerSimulation]; - mVisControl->visSettings.streakline_length = privateSteeringParams[StreaklineLength]; - - mSimState->SetIsRendering(1 == (int) privateSteeringParams[SetDoRendering]); - if (mSimState->IsRendering()) - { - readyForNextImage = false; - } - - mVisControl->UpdateImageSize((int) pixels_x, (int) pixels_y); - - distribn_t lattice_density_min = - mUnits->ConvertPressureToLatticeUnits(mVisControl->domainStats.physical_pressure_threshold_min) / Cs2; - distribn_t lattice_density_max = - mUnits->ConvertPressureToLatticeUnits(mVisControl->domainStats.physical_pressure_threshold_max) / Cs2; - distribn_t lattice_velocity_max = - mUnits->ConvertVelocityToLatticeUnits(mVisControl->domainStats.physical_velocity_threshold_max); - distribn_t lattice_stress_max = - mUnits->ConvertStressToLatticeUnits(mVisControl->domainStats.physical_stress_threshold_max); - - mVisControl->SetProjection((int) pixels_x, - (int) pixels_y, - mVisControl->visSettings.ctr_x, - mVisControl->visSettings.ctr_y, - mVisControl->visSettings.ctr_z, - longitude, - latitude, - zoom); - if (imageSendComponent != NULL) - { - imageSendComponent->SetMaxFramerate(privateSteeringParams[MaxFramerate]); - } - mVisControl->domainStats.density_threshold_min = lattice_density_min; - mVisControl->domainStats.density_threshold_minmax_inv = 1.0F / (lattice_density_max - lattice_density_min); - mVisControl->domainStats.velocity_threshold_max_inv = 1.0F / lattice_velocity_max; - mVisControl->domainStats.stress_threshold_max_inv = 1.0F / lattice_stress_max; - } - - void SteeringComponent::ClearValues() - { - readyForNextImage = false; - isConnected = false; - updatedMouseCoords = false; - - // scene center (dx,dy,dz) - privateSteeringParams[SceneCentreX] = simConfig->GetVisualisationCentre().x; - privateSteeringParams[SceneCentreY] = simConfig->GetVisualisationCentre().y; - privateSteeringParams[SceneCentreZ] = simConfig->GetVisualisationCentre().z; - - // longitude and latitude - privateSteeringParams[Longitude] = simConfig->GetVisualisationLongitude(); - privateSteeringParams[Latitude] = simConfig->GetVisualisationLatitude(); - - // zoom and brightness - privateSteeringParams[Zoom] = simConfig->GetVisualisationZoom(); - privateSteeringParams[Brightness] = simConfig->GetVisualisationBrightness(); - - // velocity and stress ranges - privateSteeringParams[PhysicalVelocityThresholdMax] = simConfig->GetMaximumVelocity(); - privateSteeringParams[PhysicalStressThrehsholdMaximum] = simConfig->GetMaximumStress(); - - // Minimum pressure and maximum pressure for Colour mapping - privateSteeringParams[PhysicalPressureThresholdMinimum] = 80.0F; - privateSteeringParams[10] = 120.0F; - - // Glyph length - privateSteeringParams[GlyphLength] = 1.0F; - - // Rendered frame size, pixel x and pixel y - privateSteeringParams[PixelsX] = 512.0F; - privateSteeringParams[PixelsY] = 512.0F; - - // x-y position of the mouse of the client - privateSteeringParams[NewMouseX] = -1.0F; - privateSteeringParams[NewMouseY] = -1.0F; - - // signal useful to terminate the simulation - privateSteeringParams[SetIsTerminal] = 0.0F; - - // Vis_mode - privateSteeringParams[Mode] = 0.0F; - - // vis_streaklines_per_pulsatile_period - privateSteeringParams[StreaklinePerSimulation] = 5.0F; - - // vis_streakline_length - privateSteeringParams[StreaklineLength] = 100.0F; - - privateSteeringParams[MaxFramerate] = 25.0F; - - // Value of DoRendering - privateSteeringParams[SetDoRendering] = 0.0F; - } - } -} diff --git a/Code/steering/common/Tag.h b/Code/steering/common/Tag.h deleted file mode 100644 index 89490da78..000000000 --- a/Code/steering/common/Tag.h +++ /dev/null @@ -1,125 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_COMMON_TAG_H -#define HEMELB_STEERING_COMMON_TAG_H - -#include -#include - -#include "steering/common/Singleton.h" -#include "steering/common/TagBase.h" - -#include "steering/common/Steerer.h" -#include "steering/common/Steerable.h" - -//class Steerer; -namespace hemelb -{ - namespace steering - { - namespace _private - { - /** - * Family of Tags for a type of variable T. Doesn't specify a tag - * string; that's the job of the the subclass. Hence we're a "static - * abstract base class" and using the Curiously Recurring Template - * Pattern/static polymorphism. Note this is without the common - * specialization when we don't have a subclass. - */ - - template - class Tag : public TagBase, - public Singleton > - { - public: - // Typedefs to avoid lots of parameterized templates - typedef T WrappedType; - typedef Tag TagType; - typedef Steerable SteerableType; - typedef std::set ContainerType; - // String for the tag - static const std::string TagString; - static const WrappedType InitialValue; - - protected: - // Contains of pointers to instances of SteerableType - ContainerType Instances; - // Our global Steerer. - static SteererClass* steerer; - - static DerivedTag* Init() - { - // Wrap the singleton behaviour, registering the instance with the - // steerer. Note have to use static polymorphism pattern here. - DerivedTag* self = static_cast (Singleton >::Init()); - - if (steerer == NULL) - steerer = SteererClass::Init(); - - steerer->SteererClass::RegisterTag(self); - return self; - } - - // Called by constructor of SteerableType - void AddInstance(SteerableType* const inst) - { - Instances.insert(inst); - } - // Called by destructor of SteerableType - void RemoveInstance(SteerableType* const inst) - { - Instances.erase(inst); - } - - public: - // Called by the Steerer to update our instances with the new value - void SetInstanceValues(const T value) - { - for (typename ContainerType::iterator i = Instances.begin(); i - != Instances.end(); ++i) - { - (*i)->Set(value); - } - } - - friend class Singleton > ; - friend class Steerable ; - - // Standard doesn't allow friend class SteererClass. - }; - - // Template static initializers - template - SteererClass* Tag::steerer = - SteererClass::Init(); - - template - const std::string Tag::TagString = ""; - - template - const typename Tag::WrappedType Tag::InitialValue = 0; - - } - } -} - -#define HEMELB_STEERING_DECLARE_TAG(name, type) \ -class name : \ - public hemelb::steering::_private::Tag \ - { \ - public: \ - typedef hemelb::steering::_private::Tag Super; \ - } - -#define HEMELB_STEERING_INITIALISE_TAG(name, value) \ - template<> const std::string name::Super::TagString = #name; \ - template<> const name::Super::WrappedType name::Super::InitialValue = value - -#endif // HEMELB_STEERING_COMMON_TAG_H diff --git a/Code/steering/common/TagBase.h b/Code/steering/common/TagBase.h deleted file mode 100644 index 60e1d7675..000000000 --- a/Code/steering/common/TagBase.h +++ /dev/null @@ -1,25 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_COMMON_TAGBASE_H -#define HEMELB_STEERING_COMMON_TAGBASE_H - -#include -#include -namespace hemelb -{ - namespace steering - { - namespace _private - { - // Base class for Tags - class TagBase - { - }; - } - } -} -#endif // HEMELB_STEERING_COMMON_TAGBASE_H diff --git a/Code/steering/common/Tags.cc b/Code/steering/common/Tags.cc deleted file mode 100644 index 377d5a4c2..000000000 --- a/Code/steering/common/Tags.cc +++ /dev/null @@ -1,43 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include - -#include "steering/common/Tags.h" -#include "steering/common/Steerer.h" -namespace hemelb -{ - namespace steering - { - HEMELB_STEERING_INITIALISE_TAG(ctr_x, 0.); - HEMELB_STEERING_INITIALISE_TAG(ctr_y, 0.); - HEMELB_STEERING_INITIALISE_TAG(ctr_z, 0.); - - HEMELB_STEERING_INITIALISE_TAG(longitude, 45.); - HEMELB_STEERING_INITIALISE_TAG(latitude, 45.); - - HEMELB_STEERING_INITIALISE_TAG(zoom, 1.); - HEMELB_STEERING_INITIALISE_TAG(brightness, 0.03); - - HEMELB_STEERING_INITIALISE_TAG(velocity_max, 0.1); - HEMELB_STEERING_INITIALISE_TAG(stress_max, 0.1); - - HEMELB_STEERING_INITIALISE_TAG(pressure_min, 80.); - HEMELB_STEERING_INITIALISE_TAG(pressure_max, 120.); - - HEMELB_STEERING_INITIALISE_TAG(glyph_length, 1.); - - HEMELB_STEERING_INITIALISE_TAG(pixels_x, 512); - HEMELB_STEERING_INITIALISE_TAG(pixels_y, 512); - - HEMELB_STEERING_INITIALISE_TAG(terminate_signal, false); - HEMELB_STEERING_INITIALISE_TAG(vis_mode, 0); - HEMELB_STEERING_INITIALISE_TAG(streaklines_per_pulsatile_period, 5); - HEMELB_STEERING_INITIALISE_TAG(streakline_length, 100); - HEMELB_STEERING_INITIALISE_TAG(doRendering, false); - - } -} diff --git a/Code/steering/common/Tags.h b/Code/steering/common/Tags.h deleted file mode 100644 index dd9100113..000000000 --- a/Code/steering/common/Tags.h +++ /dev/null @@ -1,47 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_STEERING_COMMON_TAGS_H -#define HEMELB_STEERING_COMMON_TAGS_H - -#include "steering/common/Tag.h" -#include "steering/common/Steerer.h" - -namespace hemelb -{ - namespace steering - { - HEMELB_STEERING_DECLARE_TAG(ctr_x, float); - HEMELB_STEERING_DECLARE_TAG(ctr_y, float); - HEMELB_STEERING_DECLARE_TAG(ctr_z, float); - - HEMELB_STEERING_DECLARE_TAG(longitude, float); - HEMELB_STEERING_DECLARE_TAG(latitude, float); - - HEMELB_STEERING_DECLARE_TAG(zoom, float); - HEMELB_STEERING_DECLARE_TAG(brightness, float); - - HEMELB_STEERING_DECLARE_TAG(velocity_max, float); - HEMELB_STEERING_DECLARE_TAG(stress_max, float); - - HEMELB_STEERING_DECLARE_TAG(pressure_min, float); - HEMELB_STEERING_DECLARE_TAG(pressure_max, float); - - HEMELB_STEERING_DECLARE_TAG(glyph_length, float); - - HEMELB_STEERING_DECLARE_TAG(pixels_x, unsigned int); - HEMELB_STEERING_DECLARE_TAG(pixels_y, unsigned int); - - HEMELB_STEERING_DECLARE_TAG(terminate_signal, bool); - HEMELB_STEERING_DECLARE_TAG(vis_mode, int); - HEMELB_STEERING_DECLARE_TAG(streaklines_per_pulsatile_period, unsigned int); - HEMELB_STEERING_DECLARE_TAG(streakline_length, float); - HEMELB_STEERING_DECLARE_TAG(doRendering, bool); - - } -} - -#endif // HEMELB_STEERING_COMMON_TAGS_H diff --git a/Code/steering/none/ClientConnection.cc b/Code/steering/none/ClientConnection.cc deleted file mode 100644 index e186a1284..000000000 --- a/Code/steering/none/ClientConnection.cc +++ /dev/null @@ -1,47 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#include -#include -#include - -#include "steering/ClientConnection.h" - -namespace hemelb -{ - namespace steering - { - - /** - * In the 'no-steering' code, this should do nothing. - * - * @param iSteeringSessionId - * @return - */ - ClientConnection::ClientConnection(int iSteeringSessionId, reporting::Timers & timings): - mIsBusy(), timers(timings) - { - } - - ClientConnection::~ClientConnection() - { - } - - int ClientConnection::GetWorkingSocket() - { - return -1; - } - - void ClientConnection::ReportBroken(int iSocketNum) - { - } - - } -} - diff --git a/Code/steering/none/ImageSendComponent.cc b/Code/steering/none/ImageSendComponent.cc deleted file mode 100644 index 7baf8333b..000000000 --- a/Code/steering/none/ImageSendComponent.cc +++ /dev/null @@ -1,36 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "steering/ImageSendComponent.h" - -namespace hemelb -{ - namespace steering - { - ImageSendComponent::ImageSendComponent(lb::SimulationState* iSimState, - vis::Control* iControl, - const lb::LbmParameters* iLbmParams, - Network* iNetwork, - unsigned int inletCountIn): inletCount(inletCountIn), MaxFramerate(25.0) - { - - } - - ImageSendComponent::~ImageSendComponent() - { - } - - void ImageSendComponent::DoWork(const vis::PixelSet* pix) - { - - } - - bool ImageSendComponent::ShouldRenderNewNetworkImage() - { - return false; - } - } -} diff --git a/Code/steering/none/Network.cc b/Code/steering/none/Network.cc deleted file mode 100644 index f1ad3e166..000000000 --- a/Code/steering/none/Network.cc +++ /dev/null @@ -1,86 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "debug/Debugger.h" -#include "log/Logger.h" -#include "steering/Network.h" -#include "util/utilityFunctions.h" - -namespace hemelb -{ - namespace steering - { - Network::Network(int steeringSessionId, reporting::Timers & timings) : - clientConnection(steeringSessionId, timings) - { - - } - - /** - * Do nothing. - * - * @param sockid - * @param buf - * @param length - * @return Returns true if we have successfully provided that much data. - */ - bool Network::recv_all(char *buf, const int length) - { - return false; - } - - void Network::PreReceive() - { - } - - bool Network::IsConnected() - { - return false; - } - - /** - * Do nothing. - * - * @param sockid - * @param buf - * @param length - * @return Returns the number of bytes sent or -1 on failure. - */ - bool Network::send_all(const char *buf, const int length) - { - return false; - } - - void Network::Break(int socket) - { - } - - /** - * Do nothing. - * - * @param data - * @param length - * @param socket - * @return - */ - long Network::sendInternal(const char* data, long length, int socket) - { - return -1; - } - - } -} diff --git a/Code/steering/none/SteeringComponentN.cc b/Code/steering/none/SteeringComponentN.cc deleted file mode 100644 index beb3ff4b8..000000000 --- a/Code/steering/none/SteeringComponentN.cc +++ /dev/null @@ -1,62 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "steering/SteeringComponent.h" - -namespace hemelb -{ - namespace steering - { - - /** - * In the 'no steering' version of the steering controller, we don't - * need to override any of the base classes methods, because we don't - * need to do anything. - * - * @param iNet - * @param iNetTop - * @param iSimState - * @return - */ - SteeringComponent::SteeringComponent(Network* network, - vis::Control* iVisControl, - steering::ImageSendComponent* imageSendComponent, - net::Net * iNet, - lb::SimulationState * iSimState, - configuration::SimConfig* iSimConfig, - const util::UnitConverter* iUnits) : - net::PhasedBroadcastRegular(iNet, iSimState, SPREADFACTOR), - mSimState(iSimState), mVisControl(iVisControl), imageSendComponent(imageSendComponent), mUnits(iUnits), simConfig(iSimConfig) - { - ClearValues(); - AssignValues(); - } - - bool SteeringComponent::RequiresSeparateSteeringCore() - { - return false; - } - - void SteeringComponent::ProgressFromParent(unsigned long splayNumber) - { - - } - void SteeringComponent::ProgressToChildren(unsigned long splayNumber) - { - - } - - void SteeringComponent::TopNodeAction() - { - - } - - void SteeringComponent::Effect() - { - AssignValues(); - } - } -} diff --git a/Code/timestep/Actor.cc b/Code/timestep/Actor.cc new file mode 100644 index 000000000..d97d8c883 --- /dev/null +++ b/Code/timestep/Actor.cc @@ -0,0 +1,23 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#include "timestep/Actor.h" + +namespace hemelb +{ + namespace timestep + { + void Actor::BeginAll() {} + void Actor::Begin() {} + void Actor::Receive() {} + void Actor::PreSend() {} + void Actor::Send() {} + void Actor::PreWait() {} + void Actor::Wait() {} + void Actor::End() {} + void Actor::EndAll() {} + } +} + diff --git a/Code/timestep/Actor.h b/Code/timestep/Actor.h new file mode 100644 index 000000000..7e4108e20 --- /dev/null +++ b/Code/timestep/Actor.h @@ -0,0 +1,35 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_TIMESTEP_ACTOR_H +#define HEMELB_TIMESTEP_ACTOR_H + +namespace hemelb +{ + namespace timestep + { + + class Actor + { + protected: + friend class TimeStepManager; + + inline virtual ~Actor() { + } + virtual void BeginAll() = 0; + virtual void Begin() = 0; + virtual void Receive() = 0; + virtual void PreSend() = 0; + virtual void Send() = 0; + virtual void PreWait() = 0; + virtual void Wait() = 0; + virtual void End() = 0; + virtual void EndAll() = 0; + + }; + } +} + +#endif diff --git a/Code/timestep/CMakeLists.txt b/Code/timestep/CMakeLists.txt new file mode 100644 index 000000000..60f8686f5 --- /dev/null +++ b/Code/timestep/CMakeLists.txt @@ -0,0 +1,7 @@ +# This file is part of HemeLB and is Copyright (C) +# the HemeLB team and/or their institutions, as detailed in the +# file AUTHORS. This software is provided under the terms of the +# license in the file LICENSE. + +add_library(hemelb_timestep + TimeStepManager.cc Actor.cc CollectiveActor.cc) diff --git a/Code/timestep/CollectiveActor.cc b/Code/timestep/CollectiveActor.cc new file mode 100644 index 000000000..bd1b0e436 --- /dev/null +++ b/Code/timestep/CollectiveActor.cc @@ -0,0 +1,47 @@ +// +// Copyright (C) University College London, 2007-2012, all rights reserved. +// +// This file is part of HemeLB and is CONFIDENTIAL. You may not work +// with, install, use, duplicate, modify, redistribute or share this +// file, or any part thereof, other than as allowed by any agreement +// specifically made by you with University College London. +// + +#include "timestep/CollectiveActor.h" +#include "Exception.h" + +namespace hemelb +{ + namespace timestep + { + CollectiveActor::CollectiveActor(comm::Communicator::ConstPtr comm, reporting::Timer& wTimer) : + isCollectiveRunning(false), mustWait(false), collectiveComm(comm->Duplicate()), waitTimer(wTimer), collectiveReq() + { + } + + void CollectiveActor::Receive() + { + } + + /** + * Wait on the collectives to finish. + */ + void CollectiveActor::Wait(void) + { + if (!isCollectiveRunning) + throw Exception() << "Can only wait if the collective is running"; + + waitTimer.Start(); + if (mustWait) { + collectiveReq->Wait(); + mustWait = false; + isCollectiveRunning = false; + } else { + bool done = collectiveReq->Test(); + if (done) + isCollectiveRunning = false; + } + waitTimer.Stop(); + } + } +} diff --git a/Code/timestep/CollectiveActor.h b/Code/timestep/CollectiveActor.h new file mode 100644 index 000000000..59ed393dd --- /dev/null +++ b/Code/timestep/CollectiveActor.h @@ -0,0 +1,58 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_TIMESTEP_COLLECTIVEACTOR_H +#define HEMELB_TIMESTEP_COLLECTIVEACTOR_H + +#include "timestep/Actor.h" +#include "reporting/Timers.h" +#include "comm/Request.h" + +namespace hemelb +{ + namespace timestep + { + class CollectiveActor : public Actor + { + public: + inline void MustFinishThisTimeStep() + { + + mustWait = true; + } + + protected: + CollectiveActor(comm::Communicator::ConstPtr comm, reporting::Timer& waitTimer); + + // This should be a no-op for a collective + virtual void Receive(); + + // Subclass must supply the send method with the collective + // virtual void Send(); + + // Progress the communication + virtual void Wait(void) final; + + bool isCollectiveRunning; + bool mustWait; + + /** + * Private communicator for non-blocking collectives. + */ + comm::Communicator::Ptr collectiveComm; + /** + * Timings for the wait etc. + */ + reporting::Timer& waitTimer; + /** + * Request object for the collective + */ + comm::Request::Ptr collectiveReq; + }; + + } +} + +#endif /* HEMELB_NET_COLLECTIVEACTION_H */ diff --git a/Code/timestep/TimeStepManager.cc b/Code/timestep/TimeStepManager.cc new file mode 100644 index 000000000..b40a669d0 --- /dev/null +++ b/Code/timestep/TimeStepManager.cc @@ -0,0 +1,61 @@ +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#include "timestep/TimeStepManager.h" +#include "timestep/Actor.h" +#include "debug/Debugger.h" + +namespace hemelb +{ + namespace timestep + { + TimeStepManager::TimeStepManager(unsigned nPhases) : + mPhases(nPhases) + { + } + // Register the actor for the phase + void TimeStepManager::AddToPhase(unsigned phase, Actor* actor) { + mPhases[phase].actors.push_back(actor); + } + + void TimeStepManager::DoStep() + { + // BeginAll + for (auto& ph: mPhases) + for (auto ap: ph.actors) + ap->BeginAll(); + + for (auto& ph: mPhases) + { + for (auto ap: ph.actors) + ap->Begin(); + + for (auto ap: ph.actors) + ap->Receive(); + + for (auto ap: ph.actors) + ap->PreSend(); + + for (auto ap: ph.actors) + ap->Send(); + + for (auto ap: ph.actors) + ap->PreWait(); + + for (auto ap: ph.actors) + ap->Wait(); + + for (auto ap: ph.actors) + ap->End(); + } + + for (auto& ph: mPhases) + for (auto ap: ph.actors) + ap->EndAll(); + + } + + } +} diff --git a/Code/timestep/TimeStepManager.h b/Code/timestep/TimeStepManager.h new file mode 100644 index 000000000..b9e5c4a2f --- /dev/null +++ b/Code/timestep/TimeStepManager.h @@ -0,0 +1,75 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_TIMESTEP_TIMESTEPMANAGER_H +#define HEMELB_TIMESTEP_TIMESTEPMANAGER_H + +#include + +namespace hemelb +{ + namespace timestep + { + + // A TimeStep is made up of one or more Phases of computation and + // communication. + + // A Phase has multiple Actions which follow the same pattern: + // + // * Begin - do any necessary computation/memory allocation before + // the receives begin + // + // * Receive - start non-blocking receive operations + // + // * PreSend - do any necessary computation before data can be sent + // + // * Send - initiate non-blocking sends (or collectives) + // + // * PreWait - do any available computation that does not depend + // on data to be received + // + // * Wait - wait for communication to complete + // + // * End - do computation that depends on received data + + // In addition there are two special Actions, BeginAll and EndAll + // that will be executed before and after the main phases. + + // Thus for e.g. 2 phases, the ordering of Actions will be: + // + // BeginAll0, BeginAll1 + // Begin0, Recv0, PreSend0, Send0, PreWait0, Wait0, End0, + // Begin1, Recv1, PreSend1, Send1, PreWait1, Wait1, End1, + // EndAll0, EndAll1 + + // It is important to note that each Action can have multiple + // Actors performing their tasks + class Actor; + + // This class will arrange the calling of the different components + // of HemeLB through a single timestep. + class TimeStepManager + { + public: + TimeStepManager(unsigned nPhases); + // Register the actor for the phase + void AddToPhase(unsigned phase, Actor* actor); + // Perform a timestep + void DoStep(); + private: + //friend class Actor; + + struct Phase + { + std::vector actors; + }; + + std::vector mPhases; + }; + } +} + +#endif // HEMELB_TIMESTEP_TIMESTEPMANAGER_H diff --git a/Code/unittests/FourCubeLatticeData.h b/Code/unittests/FourCubeLatticeData.h index 3bf2353a3..854a09f0d 100644 --- a/Code/unittests/FourCubeLatticeData.h +++ b/Code/unittests/FourCubeLatticeData.h @@ -58,7 +58,7 @@ namespace hemelb * * @return */ - static FourCubeLatticeData* Create(const net::IOCommunicator& comm, site_t sitesPerBlockUnit = 6, proc_t rankCount = 1) + static FourCubeLatticeData* Create(comm::Communicator::ConstPtr comm, site_t sitesPerBlockUnit = 6, proc_t rankCount = 1) { hemelb::geometry::Geometry readResult(util::Vector3D::Ones(), sitesPerBlockUnit); @@ -228,7 +228,7 @@ namespace hemelb } protected: - FourCubeLatticeData(hemelb::geometry::Geometry& readResult, const net::IOCommunicator& comms) : + FourCubeLatticeData(hemelb::geometry::Geometry& readResult, comm::Communicator::ConstPtr comms) : hemelb::geometry::LatticeData(lb::lattices::D3Q15::GetLatticeInfo(), readResult, comms) { diff --git a/Code/unittests/SimulationMasterTests.h b/Code/unittests/SimulationMasterTests.h index 0003bfdcc..e9964877f 100644 --- a/Code/unittests/SimulationMasterTests.h +++ b/Code/unittests/SimulationMasterTests.h @@ -27,14 +27,10 @@ namespace hemelb public: void setUp() { - argc = 7; + argc = 3; argv[0] = "hemelb"; - argv[2] = "four_cube.xml"; argv[1] = "-in"; - argv[3] = "-i"; - argv[4] = "1"; - argv[5] = "-ss"; - argv[6] = "1111"; + argv[2] = "four_cube.xml"; FolderTestFixture::setUp(); CopyResourceToTempdir("four_cube.xml"); CopyResourceToTempdir("four_cube.gmy"); diff --git a/Code/unittests/configuration/CommandLineTests.h b/Code/unittests/configuration/CommandLineTests.h index cb17dbae2..671b4535c 100644 --- a/Code/unittests/configuration/CommandLineTests.h +++ b/Code/unittests/configuration/CommandLineTests.h @@ -31,14 +31,10 @@ namespace hemelb void setUp() { configFile = Resource("four_cube.xml").Path(); - argc = 7; + argc = 3; argv[0] = "hemelb"; - argv[2] = configFile.c_str(); argv[1] = "-in"; - argv[3] = "-i"; - argv[4] = "1"; - argv[5] = "-ss"; - argv[6] = "1111"; + argv[2] = configFile.c_str(); FolderTestFixture::setUp(); options = new hemelb::configuration::CommandLine(argc, argv); } diff --git a/Code/unittests/geometry/GeometryReaderTests.h b/Code/unittests/geometry/GeometryReaderTests.h index fbc64d901..8dba1e7c1 100644 --- a/Code/unittests/geometry/GeometryReaderTests.h +++ b/Code/unittests/geometry/GeometryReaderTests.h @@ -39,8 +39,7 @@ namespace hemelb { FolderTestFixture::setUp(); timings = new reporting::Timers(Comms()); - reader = new GeometryReader(false, - hemelb::lb::lattices::D3Q15::GetLatticeInfo(), + reader = new GeometryReader(hemelb::lb::lattices::D3Q15::GetLatticeInfo(), *timings, Comms()); lattice = NULL; fourCube = FourCubeLatticeData::Create(Comms()); diff --git a/Code/unittests/geometry/NeedsTests.h b/Code/unittests/geometry/NeedsTests.h index 1005aba0e..6d4214c7c 100644 --- a/Code/unittests/geometry/NeedsTests.h +++ b/Code/unittests/geometry/NeedsTests.h @@ -7,7 +7,7 @@ #ifndef HEMELB_UNITTESTS_GEOMETRY_NEEDSTESTS_H #define HEMELB_UNITTESTS_GEOMETRY_NEEDSTESTS_H #include "geometry/needs/Needs.h" -#include "unittests/helpers/MockNetHelper.h" +#include "unittests/helpers/MockCommsHelper.h" #include "unittests/helpers/CppUnitCompareVectors.h" #include @@ -19,7 +19,7 @@ namespace hemelb { using namespace hemelb::geometry; using namespace hemelb::unittests::helpers; - class NeedsTests : public CppUnit::TestFixture, MockNetHelper + class NeedsTests : public CppUnit::TestFixture, MockCommsHelper { CPPUNIT_TEST_SUITE (NeedsTests); CPPUNIT_TEST (TestReadingOne); @@ -38,81 +38,38 @@ namespace hemelb void tearDown() { delete mockedNeeds; - MockNetHelper::tearDown(); + MockCommsHelper::tearDown(); } void TestReadingOne() { SetupMocks(6, 2, 5, 0); - CPPUNIT_ASSERT_EQUAL(communicatorMock->Size(),5); + CPPUNIT_ASSERT_EQUAL(communicator->Size(), 5); // Start to record the expected communications calls. - // First will come, sending to the reading cores, each of the lengths. - // I would expect to send to the other reading core, my count of needed cores - int core_0_requires_from_0_count = 1; - int core_0_requires_from_1_count = 1; - - netMock->RequireSend(&core_0_requires_from_0_count, 1, 0, "Count"); - netMock->RequireSend(&core_0_requires_from_1_count, 1, 1, "Count"); - - // And I would expect the reading core to post a receive from each of the other cores, - // asking for its count of needed blocks from this reading core - int core_0_requires_count = 1; - int core_1_requires_count = 2; - int core_2_requires_count = 1; - int core_3_requires_count = 2; - int core_4_requires_count = 1; - - netMock->RequireReceive(&core_0_requires_count, 1, 0, "Count"); - netMock->RequireReceive(&core_1_requires_count, 1, 1, "Count"); - netMock->RequireReceive(&core_2_requires_count, 1, 2, "Count"); - netMock->RequireReceive(&core_3_requires_count, 1, 3, "Count"); - netMock->RequireReceive(&core_4_requires_count, 1, 4, "Count"); - - // Then, I would expect to send to myself, my needs - std::vector core_0_requires_from_0; - core_0_requires_from_0.push_back(0); - netMock->RequireSend(&core_0_requires_from_0[0], 1, 0, "Needs"); - - // Then, I would expect to send to the other reading core, my needs - std::vector core_0_requires_from_1; - core_0_requires_from_1.push_back(1); - netMock->RequireSend(&core_0_requires_from_1[0], 1, 1, "Needs"); - - // Then, I should receieve from myself, my own requirements - // Then, I would expect to receive the lists of needed blocks themselves. - // Core 0, I expect it to need blocks 0,1 - std::vector core_0_requires; - core_0_requires.push_back(0); - netMock->RequireReceive(&core_0_requires[0], 1, 0, "Needs"); - - // Then, I would expect to receive the lists of needed blocks themselves. - // From core 1, the other reading core, I expect it to need blocks 0,1,2 - std::vector core_1_requires; - core_1_requires.push_back(0); - - core_1_requires.push_back(2); - netMock->RequireReceive(&core_1_requires[0], 2, 1, "Needs"); - // From core 2, I expect it to need blocks 1,2,3 - std::vector core_2_requires; - - core_2_requires.push_back(2); - - netMock->RequireReceive(&core_2_requires[0], 1, 2, "Needs"); - // Then, I would expect to receive the lists of needed blocks themselves. - // From core 3, I expect it to need blocks 2,3,4 - std::vector core_3_requires; - core_3_requires.push_back(2); - - core_3_requires.push_back(4); - netMock->RequireReceive(&core_3_requires[0], 2, 3, "Needs"); - // Then, I would expect to receive the lists of needed blocks themselves. - // From core 4, I expect it to need blocks 3,4,5 - std::vector core_4_requires; - - core_4_requires.push_back(4); - - netMock->RequireReceive(&core_4_requires[0], 1, 4, "Needs"); + // First we gather the counts of blocks needed on each reading core + const std::vector block_size_reading_core_0 = {1, 2, 1, 2, 1}; + // We don't really care about what the other ranks need from RC1 + const std::vector block_size_reading_core_1 = {1, 0, 2, 0, 0}; + + MockComms()->RequireGather(block_size_reading_core_0, 0); + MockComms()->RequireGather(block_size_reading_core_1, 1); + + // Now we list the blocks needed from reading core 0 (us) + // Note counts match above + const std::vector blocks_needed_from_reading_core_0 = { + 0, + 0, 2, + 2, + 2, 4, + 4 + }; + MockComms()->RequireGatherV(blocks_needed_from_reading_core_0, block_size_reading_core_0, 0); + // Blocks needed from RC1 + const std::vector blocks_needed_from_reading_core_1 = {1, 0, 0}; + MockComms()->RequireGatherV(blocks_needed_from_reading_core_1, block_size_reading_core_1, 1); + ShareMockNeeds(); + // Finally, I would expect the resulting array of needs on core one to be as planned: std::vector needing_block_0; needing_block_0.push_back(0); @@ -136,34 +93,40 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(needing_block_3, mockedNeeds->ProcessorsNeedingBlock(3)); CPPUNIT_ASSERT_EQUAL(needing_block_4, mockedNeeds->ProcessorsNeedingBlock(4)); CPPUNIT_ASSERT_EQUAL(needing_block_5, mockedNeeds->ProcessorsNeedingBlock(5)); - - netMock->ExpectationsAllCompleted(); + // I guess this means that no comms went through the net } void TestNonReading() { + // We are core 2 - a no reading one SetupMocks(6, 2, 5, 2); - - // Start to record the expected communications calls. - // First will come, sending to the reading cores, each of the lengths. - // So I would expect the non-reading core to post a send to each of the reading cores, its count of needed blocks - int core_2_requires_from_0_count = 1; - int core_2_requires_from_1_count = 2; - - netMock->RequireSend(&core_2_requires_from_0_count, 1, 0); - netMock->RequireSend(&core_2_requires_from_1_count, 1, 1); - - // Then, I would expect to send my list of needed blocks - // From core 2, the other reading core, I expect it to need blocks 1,2,3 - std::vector core_2_requires_from_0; - std::vector core_2_requires_from_1; - - core_2_requires_from_1.push_back(1); - core_2_requires_from_0.push_back(2); - core_2_requires_from_1.push_back(3); - - netMock->RequireSend(&core_2_requires_from_0[0], 1, 0); - netMock->RequireSend(&core_2_requires_from_1[0], 2, 1); + + CPPUNIT_ASSERT_EQUAL(communicator->Size(),5); + + // Record the expected communications calls. + + // First we gather the counts of blocks needed on each reading core + // For RC0 we know this from above + const std::vector block_size_reading_core_0 = {1, 2, 1, 2, 1}; + // We don't really care about what the other ranks need from RC1 + const std::vector block_size_reading_core_1 = {0, 0, 2, 0, 0}; + + MockComms()->RequireGather(block_size_reading_core_0, 0); + MockComms()->RequireGather(block_size_reading_core_1, 1); + + // Now we list the blocks needed from RC0 + // Note counts match above + const std::vector blocks_needed_from_reading_core_0 = { + 0, + 0, 2, + 2, + 2, 4, + 4 + }; + MockComms()->RequireGatherV(blocks_needed_from_reading_core_0, block_size_reading_core_0, 0); + // Blocks needed from RC1 + const std::vector blocks_needed_from_reading_core_1 = {1 ,3}; + MockComms()->RequireGatherV(blocks_needed_from_reading_core_1, block_size_reading_core_1, 1); ShareMockNeeds(); // Finally, I would expect the resulting array of needs to be empty @@ -176,7 +139,6 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(empty_needs_array, mockedNeeds->ProcessorsNeedingBlock(4)); CPPUNIT_ASSERT_EQUAL(empty_needs_array, mockedNeeds->ProcessorsNeedingBlock(5)); - netMock->ExpectationsAllCompleted(); } void SetupMocks(const site_t block_count, @@ -188,7 +150,7 @@ namespace hemelb readingCores = reading_cores; rank = current_core; size = core_count; - MockNetHelper::setUp(core_count,current_core); + MockCommsHelper::setUp(core_count,current_core); inputNeededBlocks = std::vector(block_count); @@ -204,7 +166,7 @@ namespace hemelb mockedNeeds = new Needs(blockCount, inputNeededBlocks, readingCores, - *netMock, + communicator, false); } diff --git a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h index 27f7fe17b..66a45c96a 100644 --- a/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h +++ b/Code/unittests/geometry/neighbouring/NeighbouringDataManagerTests.h @@ -6,10 +6,11 @@ #ifndef HEMELB_UNITTESTS_GEOMETRY_NEIGHBOURING_NEIGHBOURINGDATAMANAGERTESTS_H #define HEMELB_UNITTESTS_GEOMETRY_NEIGHBOURING_NEIGHBOURINGDATAMANAGERTESTS_H -#include "net/phased/StepManager.h" -#include "net/phased/NetConcern.h" + #include "geometry/neighbouring/NeighbouringDataManager.h" -#include "unittests/helpers/MockNetHelper.h" +#include "unittests/helpers/MockCommsHelper.h" +#include "comm/MpiEnvironment.h" +#include "comm/AsyncConcern.h" namespace hemelb { @@ -21,7 +22,7 @@ namespace hemelb { using namespace hemelb::geometry::neighbouring; class NeighbouringDataManagerTests : public FourCubeBasedTestFixture, - public MockNetHelper + public MockCommsHelper { CPPUNIT_TEST_SUITE ( NeighbouringDataManagerTests); CPPUNIT_TEST ( TestConstruct); @@ -43,15 +44,16 @@ namespace hemelb { FourCubeBasedTestFixture::setUp(); data = &latDat->GetNeighbouringData(); - MockNetHelper::setUp(1, 0); - manager = new NeighbouringDataManager(*latDat, *data, *netMock); + MockCommsHelper::setUp(1, 0); + manager = new NeighbouringDataManager(*latDat, *data, commQ); } + void tearDown() { delete manager; FourCubeBasedTestFixture::tearDown(); - MockNetHelper::tearDown(); + MockCommsHelper::tearDown(); } void TestConstruct() @@ -90,27 +92,18 @@ namespace hemelb void TestShareNeedsOneProc() { - manager->RegisterNeededSite(43); - - // We should receive a signal that we need one from ourselves - std::vector countOfNeedsToZeroFromZero; - countOfNeedsToZeroFromZero.push_back(1); // expectation - std::vector countOfNeedsFromZeroToZero; - countOfNeedsFromZeroToZero.push_back(1); //fixture - netMock->RequireSend(&countOfNeedsToZeroFromZero.front(), 1, 0, "CountToSelf"); - netMock->RequireReceive(&countOfNeedsFromZeroToZero.front(), 1, 0, "CountFromSelf"); - - // Once we've received the signal that we need one from ourselves, we should receive that one. - std::vector needsShouldBeSentToSelf; - std::vector needsShouldBeReceivedFromSelf; - needsShouldBeSentToSelf.push_back(43); //expectation - needsShouldBeReceivedFromSelf.push_back(43); //fixture - netMock->RequireSend(&needsShouldBeSentToSelf.front(), 1, 0, "NeedToSelf"); - netMock->RequireReceive(&needsShouldBeSentToSelf.front(), 1, 0, "NeedFromSelf"); - + std::vector neededsites = {43}; + manager->RegisterNeededSite(neededsites[0]); + + // The MapA2A needs a barrier but as the only needs are + // local this is true immediately + MockComms()->RequireIbarrier([]() { return true; }); + + // We then send this need to ourself + MockComms()->RequireSend(neededsites, 0, 0); + MockComms()->RequireRecv(neededsites, 0, 0); manager->ShareNeeds(); - netMock->ExpectationsAllCompleted(); - + MockComms()->ExpectationsAllCompleted(); CPPUNIT_ASSERT_EQUAL(manager->GetNeedsForProc(0).size(), static_cast::size_type> (1)); CPPUNIT_ASSERT_EQUAL(manager->GetNeedsForProc(0).front(), static_cast (43)); @@ -118,24 +111,13 @@ namespace hemelb void TestShareConstantDataOneProc() { - // As for ShareNeeds test, set up the site as needed from itself. - std::vector countOfNeedsToZeroFromZero; - countOfNeedsToZeroFromZero.push_back(1); // expectation - std::vector countOfNeedsFromZeroToZero; - countOfNeedsFromZeroToZero.push_back(1); //fixture - netMock->RequireSend(&countOfNeedsToZeroFromZero.front(), 1, 0, "CountToSelf"); - netMock->RequireReceive(&countOfNeedsFromZeroToZero.front(), 1, 0, "CountFromSelf"); - - std::vector needsShouldBeSentToSelf; - std::vector needsShouldBeReceivedFromSelf; - needsShouldBeSentToSelf.push_back(43); //expectation - needsShouldBeReceivedFromSelf.push_back(43); //fixture - netMock->RequireSend(&needsShouldBeSentToSelf.front(), 1, 0, "NeedToSelf"); - netMock->RequireReceive(&needsShouldBeSentToSelf.front(), 1, 0, "NeedFromSelf"); - - manager->RegisterNeededSite(43); + std::vector neededsites = {43}; + manager->RegisterNeededSite(neededsites[0]); + + MockComms()->RequireIbarrier([]() { return true; }); + MockComms()->RequireSend(neededsites, 0, 0); + MockComms()->RequireRecv(neededsites, 0, 0); manager->ShareNeeds(); - netMock->ExpectationsAllCompleted(); // Now, transfer the data about that site. Site < LatticeData > exampleSite @@ -146,39 +128,37 @@ namespace hemelb SiteData expectedData = exampleSite.GetSiteData(); SiteData fixtureData = exampleSite.GetSiteData(); - netMock->RequireSend(&expectedData.GetWallIntersectionData(), - 1, - 0, - "WallIntersectionDataToSelf"); - netMock->RequireReceive(&fixtureData.GetWallIntersectionData(), - 1, - 0, - "WallIntersectionDataFromSelf"); - netMock->RequireSend(&expectedData.GetIoletIntersectionData(), - 1, - 0, - "IoletIntersectionDataToSelf"); - netMock->RequireReceive(&fixtureData.GetIoletIntersectionData(), - 1, - 0, - "IoletIntersectionDataFromSelf"); - netMock->RequireSend(&expectedData.GetIoletId(), 1, 0, "IoletIdToSelf"); - netMock->RequireReceive(&fixtureData.GetIoletId(), 1, 0, "IoletIdFromSelf"); - netMock->RequireSend(&expectedData.GetSiteType(), 1, 0, "SiteTypeToSelf"); - netMock->RequireReceive(&fixtureData.GetSiteType(), 1, 0, "SiteTypeFromSelf"); - - netMock->RequireSend(exampleSite.GetWallDistances(), + MockComms()->RequireSend(expectedData.GetWallIntersectionData(), + 0, 0); + MockComms()->RequireRecv(fixtureData.GetWallIntersectionData(), + 0, 0); + MockComms()->RequireSend(&expectedData.GetIoletIntersectionData(), + 1, + 0, + 0); + MockComms()->RequireRecv(&fixtureData.GetIoletIntersectionData(), + 1, + 0, + 0); + MockComms()->RequireSend(&expectedData.GetIoletId(), 1, 0, 0); + MockComms()->RequireRecv(&fixtureData.GetIoletId(), 1, 0, 0); + MockComms()->RequireSend(&expectedData.GetSiteType(), 1, 0, 0); + MockComms()->RequireRecv(&fixtureData.GetSiteType(), 1, 0, 0); + + MockComms()->RequireSend(exampleSite.GetWallDistances(), lb::lattices::D3Q15::NUMVECTORS - 1, 0, - "WallToSelf"); - netMock->RequireReceive(exampleSite.GetWallDistances(), + 0); + MockComms()->RequireRecv(exampleSite.GetWallDistances(), lb::lattices::D3Q15::NUMVECTORS - 1, 0, - "WallFromSelf"); - netMock->RequireSend(&exampleSite.GetWallNormal(), 1, 0, "NormalToSelf"); - netMock->RequireReceive(&exampleSite.GetWallNormal(), 1, 0, "NormalFromSelf"); + 0); + MockComms()->RequireSend(&exampleSite.GetWallNormal(), 1, 0, 0); + MockComms()->RequireRecv(&exampleSite.GetWallNormal(), 1, 0, 0); manager->TransferNonFieldDependentInformation(); - netMock->ExpectationsAllCompleted(); + + MockComms()->ExpectationsAllCompleted(); + NeighbouringSite transferredSite = data->GetSite(43); CPPUNIT_ASSERT_EQUAL(exampleSite.GetSiteData(), transferredSite.GetSiteData()); for (unsigned int direction = 0; direction < lb::lattices::D3Q15::NUMVECTORS - 1; direction++) @@ -191,50 +171,41 @@ namespace hemelb void TestShareFieldDataOneProc() { + + site_t targetGlobalOneDIdx = 43; LatticeVector targetGlobalThreeDIdx = latDat->GetSiteCoordsFromSiteId(targetGlobalOneDIdx); site_t targetLocalIdx = latDat->GetLocalContiguousIdFromGlobalNoncontiguousId(targetGlobalOneDIdx); for (unsigned int direction = 0; direction < 3; direction++) CPPUNIT_ASSERT_EQUAL(site_t(1), targetGlobalThreeDIdx[direction]); - - // begin by setting up mocks for the required site - std::vector countOfNeedsToZeroFromZero; - countOfNeedsToZeroFromZero.push_back(1); // expectation - std::vector countOfNeedsFromZeroToZero; - countOfNeedsFromZeroToZero.push_back(1); //fixture - netMock->RequireSend(&countOfNeedsToZeroFromZero.front(), 1, 0, "CountToSelf"); - netMock->RequireReceive(&countOfNeedsFromZeroToZero.front(), 1, 0, "CountFromSelf"); - - std::vector needsShouldBeSentToSelf; - std::vector needsShouldBeReceivedFromSelf; - needsShouldBeSentToSelf.push_back(targetGlobalOneDIdx); //expectation - needsShouldBeReceivedFromSelf.push_back(targetGlobalOneDIdx); //fixture - netMock->RequireSend(&needsShouldBeSentToSelf.front(), 1, 0, "NeedToSelf"); - netMock->RequireReceive(&needsShouldBeSentToSelf.front(), 1, 0, "NeedFromSelf"); + + MockComms()->RequireIbarrier([]() { return true; }); + std::vector neededsites = {targetGlobalOneDIdx}; + MockComms()->RequireSend(neededsites, 0, 0); + MockComms()->RequireRecv(neededsites, 0, 0); manager->RegisterNeededSite(targetGlobalOneDIdx); manager->ShareNeeds(); - netMock->ExpectationsAllCompleted(); // Now, transfer the data about that site. Site < LatticeData > exampleSite = latDat->GetSite(targetLocalIdx); // It should arrive in the NeighbouringDataManager, from the values sent from the localLatticeData - netMock->RequireSend(const_cast (exampleSite.GetFOld ()), - lb::lattices::D3Q15::NUMVECTORS, - 0, - "IntersectionDataToSelf"); + MockComms()->RequireSend(const_cast (exampleSite.GetFOld ()), + lb::lattices::D3Q15::NUMVECTORS, + 0, + 0); std::vector receivedFOld(lb::lattices::D3Q15::NUMVECTORS, 53.0); - netMock->RequireReceive(&(receivedFOld[0]), - lb::lattices::D3Q15::NUMVECTORS, - 0, - "IntersectionDataFromSelf"); + MockComms()->RequireRecv(&(receivedFOld[0]), + lb::lattices::D3Q15::NUMVECTORS, + 0, + 0); manager->TransferFieldDependentInformation(); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); NeighbouringSite transferredSite = data->GetSite(targetGlobalOneDIdx); for (unsigned int direction = 0; direction < lb::lattices::D3Q15::NUMVECTORS; direction++) @@ -248,46 +219,36 @@ namespace hemelb { site_t targetGlobalOneDIdx = 43; site_t targetLocalIdx = latDat->GetLocalContiguousIdFromGlobalNoncontiguousId(targetGlobalOneDIdx); - - // begin by setting up mocks for the required site - std::vector countOfNeedsToZeroFromZero; - countOfNeedsToZeroFromZero.push_back(1); // expectation - std::vector countOfNeedsFromZeroToZero; - countOfNeedsFromZeroToZero.push_back(1); //fixture - netMock->RequireSend(&countOfNeedsToZeroFromZero.front(), 1, 0, "CountToSelf"); - netMock->RequireReceive(&countOfNeedsFromZeroToZero.front(), 1, 0, "CountFromSelf"); - - std::vector needsShouldBeSentToSelf; - std::vector needsShouldBeReceivedFromSelf; - needsShouldBeSentToSelf.push_back(targetGlobalOneDIdx); //expectation - needsShouldBeReceivedFromSelf.push_back(targetGlobalOneDIdx); //fixture - netMock->RequireSend(&needsShouldBeSentToSelf.front(), 1, 0, "NeedToSelf"); - netMock->RequireReceive(&needsShouldBeSentToSelf.front(), 1, 0, "NeedFromSelf"); + + MockComms()->RequireIbarrier([]() { return true; }); + std::vector neededsites = {targetGlobalOneDIdx}; + MockComms()->RequireSend(neededsites, 0, 0); + MockComms()->RequireRecv(neededsites, 0, 0); manager->RegisterNeededSite(targetGlobalOneDIdx); manager->ShareNeeds(); - netMock->ExpectationsAllCompleted(); // Now, transfer the data about that site. Site < LatticeData > exampleSite = latDat->GetSite(targetLocalIdx); - // It should arrive in the NeighbouringDataManager, from the values sent from the localLatticeData + // It should arrive in the NeighbouringDataManager, from + // the values sent from the localLatticeData - netMock->RequireSend(const_cast (exampleSite.GetFOld ()), - lb::lattices::D3Q15::NUMVECTORS, - 0, - "IntersectionDataToSelf"); + MockComms()->RequireSend(const_cast (exampleSite.GetFOld ()), + lb::lattices::D3Q15::NUMVECTORS, + 0, + 0); std::vector receivedFOld(lb::lattices::D3Q15::NUMVECTORS, 53.0); - netMock->RequireReceive(&(receivedFOld[0]), - lb::lattices::D3Q15::NUMVECTORS, - 0, - "IntersectionDataFromSelf"); - - net::phased::StepManager stepManager; - stepManager.RegisterIteratedActorSteps(*manager, 0); - net::phased::NetConcern netConcern = net::phased::NetConcern(*netMock); - stepManager.RegisterCommsForAllPhases(netConcern); - stepManager.CallActions(); - netMock->ExpectationsAllCompleted(); + MockComms()->RequireRecv(&(receivedFOld[0]), + lb::lattices::D3Q15::NUMVECTORS, + 0, + 0); + + timestep::TimeStepManager stepManager(1); + stepManager.AddToPhase(0, manager); + comm::AsyncConcern netConcern = comm::AsyncConcern(commQ); + stepManager.AddToPhase(0, &netConcern); + stepManager.DoStep(); + MockComms()->ExpectationsAllCompleted(); NeighbouringSite transferredSite = data->GetSite(targetGlobalOneDIdx); for (unsigned int direction = 0; direction < lb::lattices::D3Q15::NUMVECTORS; direction++) diff --git a/Code/unittests/helpers/FourCubeBasedTestFixture.h b/Code/unittests/helpers/FourCubeBasedTestFixture.h index 992a110cf..08cbd5987 100644 --- a/Code/unittests/helpers/FourCubeBasedTestFixture.h +++ b/Code/unittests/helpers/FourCubeBasedTestFixture.h @@ -10,7 +10,6 @@ #include "configuration/SimConfig.h" #include "lb/collisions/Collisions.h" #include "lb/SimulationState.h" -#include "net/IOCommunicator.h" #include "unittests/FourCubeLatticeData.h" #include "unittests/OneInOneOutSimConfig.h" #include "unittests/helpers/FolderTestFixture.h" diff --git a/Code/unittests/helpers/HasCommsTestFixture.h b/Code/unittests/helpers/HasCommsTestFixture.h index 1e2b5f0f3..fb38ca024 100644 --- a/Code/unittests/helpers/HasCommsTestFixture.h +++ b/Code/unittests/helpers/HasCommsTestFixture.h @@ -7,7 +7,7 @@ #ifndef HEMELB_UNITTESTS_HELPERS_HASCOMMSTESTFIXTURE_H #define HEMELB_UNITTESTS_HELPERS_HASCOMMSTESTFIXTURE_H #include -#include "net/IOCommunicator.h" +#include "comm/Communicator.h" namespace hemelb { @@ -22,28 +22,35 @@ namespace hemelb void setUp() { //hemelbCommunicator = net::IOCommunicator::Instance(); + asyncCommQ = comm::Async::New(hemelbCommunicator); } void tearDown() { //hemelbCommunicator = NULL; + asyncCommQ = nullptr; } - static void Init(const net::IOCommunicator& inst) + static void Init(comm::Communicator::ConstPtr inst) { - hemelbCommunicator = &inst; + hemelbCommunicator = inst; } protected: - static const net::IOCommunicator& Comms() + static comm::Communicator::ConstPtr Comms() { - return *hemelbCommunicator; + return hemelbCommunicator; } + comm::Async::Ptr Async() + { + return asyncCommQ; + } private: - static const net::IOCommunicator* hemelbCommunicator; + static comm::Communicator::ConstPtr hemelbCommunicator; + comm::Async::Ptr asyncCommQ; }; - const net::IOCommunicator* HasCommsTestFixture::hemelbCommunicator = NULL; + comm::Communicator::ConstPtr HasCommsTestFixture::hemelbCommunicator = NULL; } } } diff --git a/Code/unittests/helpers/MockCommsHelper.h b/Code/unittests/helpers/MockCommsHelper.h new file mode 100644 index 000000000..9f2fb87e5 --- /dev/null +++ b/Code/unittests/helpers/MockCommsHelper.h @@ -0,0 +1,48 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_UNITTESTS_HELPERS_MOCKCOMMSHELPER_H +#define HEMELB_UNITTESTS_HELPERS_MOCKCOMMSHELPER_H + +#include "unittests/helpers/MockCommunicator.h" + +namespace hemelb +{ + namespace unittests + { + namespace helpers + { + class MockCommsHelper + { + protected: + MockCommsHelper() + { + } + void setUp(const proc_t core_count, const proc_t current_core) + { + communicator = std::make_shared(current_core, core_count); + commQ = comm::Async::New(communicator); + //communicator = communicatorMock; + } + void tearDown() + { + } + + std::shared_ptr MockComms() { + auto ans = std::dynamic_pointer_cast(communicator); + CPPUNIT_ASSERT_MESSAGE("Cannot cast Communicator to a MockCommunicator", ans); + return ans; + } + //std::shared_ptr communicatorMock; + comm::Communicator::Ptr communicator; + comm::Async::Ptr commQ; + }; + + } + } +} + +#endif // HEMELB_UNITTESTS_HELPERS_MOCKCOMMSHELPER_H diff --git a/Code/unittests/helpers/MockCommunicator.h b/Code/unittests/helpers/MockCommunicator.h new file mode 100644 index 000000000..dc36dc023 --- /dev/null +++ b/Code/unittests/helpers/MockCommunicator.h @@ -0,0 +1,466 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATOR_H +#define HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATOR_H + +#include +#include + +#include "comm/Communicator.h" +#include "comm/Request.h" + +namespace hemelb +{ + namespace unittests + { + namespace helpers + { + using namespace comm; + + class MockRequest : public comm::Request + { + public: + MockRequest() : comm::Request(), testCount(0) { + } + virtual void Wait() { + + } + virtual bool Test() { + if (testCount > 1) { + Wait(); + return true; + } else { + ++testCount; + return false; + } + } + + private: + int testCount; + }; + + class MockIbarrierRequest : public MockRequest + { + public: + typedef std::function NullaryPredicate; + MockIbarrierRequest(NullaryPredicate fn) : done(fn){ + } + virtual void Wait() { + CPPUNIT_ASSERT_MESSAGE("Predicate must be done before waiting!", done()); + } + virtual bool Test() { + return done(); + } + private: + NullaryPredicate done; + }; + + class MockRequestList : public comm::RequestList + { + std::vector reqs; + public: + virtual size_t size() const { + return reqs.size(); + } + virtual void resize(size_t i) { + reqs.resize(i); + } + virtual void push_back(Request::Ptr rp) { + reqs.push_back(rp); + } + + virtual void clear() { + reqs.clear(); + } + + virtual void set(size_t i, Request::Ptr rp) { + reqs[i] = rp; + } + + virtual void WaitAll() { + for (auto r: reqs) { + r->Wait(); + } + } + virtual bool TestAll() { + bool alldone = true; + for (auto r: reqs) { + if (!r->Test()) { + alldone = false; + break; + } + } + return alldone; + } + }; + + // class MockSendReq : public MockRequest + // { + // MockSendReq(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + // int dest, int tag) : mSendbuf(sendbuf), mSendcount(sendcount), mSendtype(sendtype), + // mDest(dest), mTag(tag) + // { + // } + + // private: + // const void *mSendbuf; + // int mSendcount; + // MPI_Datatype mSendtype; + // int mDest; + // int mTag; + // }; + + class MockCommunicator : public comm::Communicator + { + public: + /*** + * Constructor for a dummy communicator + * Can be useful for testing but can't actually be used + * @param rank + * @param size + */ + MockCommunicator(int rank_, int size_) : + rank(rank_), size(size_) + { + + } + + virtual inline int Rank() const + { + return rank; + } + virtual inline int Size() const + { + return size; + } + +#define NOTIMPLEMENTED CPPUNIT_FAIL("Communicator function not implemented") + + virtual void Abort(int errCode) const { NOTIMPLEMENTED; } + + virtual Ptr Duplicate() const { NOTIMPLEMENTED; return nullptr; } + + virtual std::shared_ptr GetGroup() const { NOTIMPLEMENTED; return nullptr; } + virtual Ptr Create(std::shared_ptr grp) const { NOTIMPLEMENTED; return nullptr; } + virtual std::shared_ptr OpenFile(const std::string& filename, int mode, + const MPI_Info info = MPI_INFO_NULL) const { NOTIMPLEMENTED; return nullptr; } + + virtual std::shared_ptr MakeRequestList() const { + return std::make_shared(); + } + virtual void Barrier() const { NOTIMPLEMENTED; } + + virtual std::shared_ptr Ibarrier() const { + CPPUNIT_ASSERT(ibarrier_conditions.size()); + auto& cond = ibarrier_conditions.front(); + auto ans = std::make_shared(cond); + ibarrier_conditions.pop_front(); + return ans; + } + + virtual bool Iprobe(int source, int tag, MPI_Status* stat=MPI_STATUS_IGNORE) const { + // Look through the pending recvs and point to the first one that matches + auto maybe_match = recv_data.begin(); + auto end = recv_data.end(); + for (; maybe_match != end; ++maybe_match) { + if (source == MPI_ANY_SOURCE || source == maybe_match->src) { + if (tag == MPI_ANY_TAG || tag == maybe_match->tag) { + break; + } + } + } + + // No match - done + if (maybe_match == end) { + return false; + } + + // Got a match, setup the status then move it to the front + // of the queue + stat->MPI_SOURCE = maybe_match->src; + stat->MPI_TAG = maybe_match->tag; + auto tmp = std::move(*maybe_match); + recv_data.erase(maybe_match); + recv_data.push_front(std::move(tmp)); + return true; + } + + void ExpectationsAllCompleted() { + CPPUNIT_ASSERT_EQUAL(size_t(0), gather_data.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), gatherv_data.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), send_data.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), ssend_data.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), recv_data.size()); + CPPUNIT_ASSERT_EQUAL(size_t(0), ibarrier_conditions.size()); + } + + template + void RequireGather(const std::vector& data, int root) + { + CPPUNIT_ASSERT_EQUAL_MESSAGE("Number of elements must be a multiple of comm size", + 0, int(data.size()) % Size()); + int sendcount = data.size() / Size(); + gather_data.push_back({data.data() + sendcount*Rank(), sendcount, comm::MpiDataType(), + data.data(), sendcount, comm::MpiDataType(), + root}); + } + + template + void RequireGatherV(const std::vector& data, const std::vector& recvcounts, int root) + { + CPPUNIT_ASSERT_EQUAL(Size(), int(recvcounts.size())); + std::vector displs(Size()); + int total = 0; + for(size_t i = 0; i < Size(); ++i) { + displs[i] = total; + total += recvcounts[i]; + } + auto my_send_data = data.data() + displs[Rank()]; + CPPUNIT_ASSERT_EQUAL_MESSAGE("Size of data does not match sum of recvcounts", + size_t(total), data.size()); + gatherv_data.push_back({my_send_data, recvcounts[Rank()], comm::MpiDataType(), + data.data(), recvcounts.data(), std::move(displs), comm::MpiDataType(), + root}); + } + + template + void RequireSend(const T* valPtr, int count, int dest, int tag) + { + static_assert(!std::is_pointer::value, + "You are trying to require the sending of a pointer type"); + static_assert(!is_std_vector::value, + "You are trying to require the sending of a std::vector type"); + + CPPUNIT_ASSERT(dest >= 0); + CPPUNIT_ASSERT(dest < Size()); + send_data.push_back({valPtr, count, comm::MpiDataType(), dest, tag}); + } + template + void RequireSend(const T& val, int dest, int tag) + { + RequireSend(&val, 1, dest, tag); + } + template + void RequireSend(const std::vector& vals, int dest, int tag) + { + RequireSend(vals.data(), vals.size(), dest, tag); + } + + template + void RequireRecv(const T* valPtr, int count, int src, int tag) + { + static_assert(!std::is_pointer::value, + "You are trying to require the receiving of a pointer type"); + static_assert(!is_std_vector::value, + "You are trying to require the receiving of a std::vector type"); + CPPUNIT_ASSERT(src >= 0); + CPPUNIT_ASSERT(src < Size()); + recv_data.push_back({valPtr, count, comm::MpiDataType(), src, tag}); + } + template + void RequireRecv(const T& val, int src, int tag) + { + RequireRecv(&val, 1, src, tag); + } + template + void RequireRecv(const std::vector& vals, int src, int tag) + { + RequireRecv(vals.data(), vals.size(), src, tag); + } + + void RequireIbarrier(MockIbarrierRequest::NullaryPredicate cond) { + ibarrier_conditions.push_back(cond); + } + private: + int rank, size; + + struct SendData { + const void* expected_buf; + int count; + MPI_Datatype type; + int dest; + int tag; + }; + + struct RecvData { + const void* ans_buf; + int count; + MPI_Datatype type; + int src; + int tag; + }; + + struct GatherData { + const void* send_buf; + int sendcount; + MPI_Datatype sendtype; + const void* ans_buf; + int recvcount; + MPI_Datatype recvtype; + int root; + }; + + struct GatherVData { + const void* send_buf; + int sendcount; + MPI_Datatype sendtype; + const void* ans_buf; + const int* recvcounts; + const std::vector displs; + MPI_Datatype recvtype; + + int root; + }; + + mutable std::deque send_data; + mutable std::deque ssend_data; + mutable std::deque recv_data; + mutable std::deque gather_data; + mutable std::deque gatherv_data; + mutable std::deque ibarrier_conditions; + + virtual void BcastImpl(void* buf, int count, MPI_Datatype dt, int root) const { NOTIMPLEMENTED; } + virtual std::shared_ptr IbcastImpl(void* buf, int count, MPI_Datatype dt, int root) const { NOTIMPLEMENTED; return nullptr; } + virtual void AllreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const { NOTIMPLEMENTED; } + virtual std::shared_ptr IallreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op) const { NOTIMPLEMENTED; return nullptr; } + virtual std::shared_ptr IreduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const { NOTIMPLEMENTED; return nullptr; } + virtual void ReduceImpl(const void* send, void* ans, int count, MPI_Datatype dt, MPI_Op op, int root) const { NOTIMPLEMENTED; } + + virtual void GatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype, + int root) const { + CPPUNIT_ASSERT(gather_data.size()); + + auto& gd = gather_data.front(); + CPPUNIT_ASSERT_EQUAL(gd.sendcount, sendcount); + CPPUNIT_ASSERT_EQUAL(gd.sendtype, sendtype); + CPPUNIT_ASSERT_EQUAL(gd.root, root); + int elSize; + MPI_Type_size(gd.sendtype, &elSize); + + { // assert that send[:] == relevant bit of stored data + int size_bytes = sendcount * elSize; + const char* expected = static_cast(gd.send_buf); + const char* actual = static_cast(send); + for (auto i = 0; i < size_bytes; ++i) + CPPUNIT_ASSERT_EQUAL(expected[i], actual[i]); + } + + if (Rank() == root) { + CPPUNIT_ASSERT_EQUAL(gd.recvcount, recvcount); + CPPUNIT_ASSERT_EQUAL(gd.recvtype, recvtype); + + std::memcpy(recv, gd.ans_buf, recvcount * elSize * Size()); + } + gather_data.pop_front(); + } + + virtual void GathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype, + int root) const { + CPPUNIT_ASSERT(gatherv_data.size()); + + auto& gd = gatherv_data.front(); + CPPUNIT_ASSERT_EQUAL(gd.sendcount, sendcount); + CPPUNIT_ASSERT_EQUAL(gd.sendtype, sendtype); + CPPUNIT_ASSERT_EQUAL(gd.root, root); + + int elSize; + MPI_Type_size(gd.sendtype, &elSize); + + { // assert that send[:] == relevant bit of stored data + int size_bytes = sendcount * elSize; + const char* expected = static_cast(gd.send_buf); + const char* actual = static_cast(sendbuf); + for (auto i = 0; i < size_bytes; ++i) + CPPUNIT_ASSERT_EQUAL(expected[i], actual[i]); + } + + if (Rank() == root) { + // The recv counts and displacements must match on root + for(auto i = 0; i < Size(); ++i) { + CPPUNIT_ASSERT_EQUAL(gd.recvcounts[i], recvcounts[i]); + CPPUNIT_ASSERT_EQUAL(gd.displs[i], displs[i]); + } + CPPUNIT_ASSERT_EQUAL(gd.recvtype, recvtype); + + int nElTotal = std::accumulate(recvcounts, recvcounts+Size(), 0); + std::memcpy(recvbuf, gd.ans_buf, nElTotal*elSize); + } + gatherv_data.pop_front(); + } + + virtual void AllgatherImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } + virtual void AllgathervImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + void *recvbuf, const int* recvcounts, const int* displs, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } + virtual void AlltoallImpl(const void* send, int sendcount, MPI_Datatype sendtype, + void* recv, int recvcount, MPI_Datatype recvtype) const { NOTIMPLEMENTED; } + virtual void SendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { + CPPUNIT_ASSERT(send_data.size()); + + auto& sd = send_data.front(); + CPPUNIT_ASSERT_EQUAL(sd.count, sendcount); + CPPUNIT_ASSERT_EQUAL(sd.type, sendtype); + CPPUNIT_ASSERT_EQUAL(sd.dest, dest); + CPPUNIT_ASSERT_EQUAL(sd.tag, tag); + send_data.pop_front(); + } + virtual void SsendImpl(const void *sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { + CPPUNIT_ASSERT(ssend_data.size()); + + auto& sd = ssend_data.front(); + CPPUNIT_ASSERT_EQUAL(sd.count, sendcount); + CPPUNIT_ASSERT_EQUAL(sd.type, sendtype); + CPPUNIT_ASSERT_EQUAL(sd.dest, dest); + CPPUNIT_ASSERT_EQUAL(sd.tag, tag); + ssend_data.pop_front(); + } + + virtual void RecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int src, int tag, MPI_Status* stat) const { + CPPUNIT_ASSERT(recv_data.size()); + + auto& rd = recv_data.front(); + CPPUNIT_ASSERT_EQUAL(rd.count, recvcount); + CPPUNIT_ASSERT_EQUAL(rd.type, recvtype); + CPPUNIT_ASSERT_EQUAL(rd.src, src); + CPPUNIT_ASSERT_EQUAL(rd.tag, tag); + + int elSize; + MPI_Type_size(rd.type, &elSize); + + std::memcpy(recvbuf, rd.ans_buf, recvcount * elSize); + recv_data.pop_front(); + + } + + virtual std::shared_ptr IsendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { + SendImpl(sendbuf, sendcount, sendtype, dest, tag); + return std::make_shared(); + } + + virtual std::shared_ptr IssendImpl(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + int dest, int tag) const { + SsendImpl(sendbuf, sendcount, sendtype, dest, tag); + return std::make_shared(); + } + virtual std::shared_ptr IrecvImpl(void* recvbuf, int recvcount, MPI_Datatype recvtype, + int source, int tag) const { + RecvImpl(recvbuf, recvcount, recvtype, source, tag, MPI_STATUS_IGNORE); + return std::make_shared(); + } + }; + + } + } +} + +#endif // HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATOR_H diff --git a/Code/unittests/helpers/MockCommunicatorTests.h b/Code/unittests/helpers/MockCommunicatorTests.h new file mode 100644 index 000000000..c7dd372d8 --- /dev/null +++ b/Code/unittests/helpers/MockCommunicatorTests.h @@ -0,0 +1,188 @@ + +// This file is part of HemeLB and is Copyright (C) +// the HemeLB team and/or their institutions, as detailed in the +// file AUTHORS. This software is provided under the terms of the +// license in the file LICENSE. + +#ifndef HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATORTESTS_H +#define HEMELB_UNITTESTS_HELPERS_MOCKCOMMUNICATORTESTS_H + +#include +#include "unittests/helpers/MockCommunicator.h" + +namespace hemelb +{ + namespace unittests + { + namespace helpers + { + class MockCommunicatorTests : public CppUnit::TestFixture + { + CPPUNIT_TEST_SUITE( MockCommunicatorTests); + CPPUNIT_TEST( TestSimple); + + CPPUNIT_TEST( TestSend); + CPPUNIT_TEST( TestIsend); + + CPPUNIT_TEST( TestRecv); + CPPUNIT_TEST( TestIrecv); + + CPPUNIT_TEST( TestGatherRoot); + CPPUNIT_TEST( TestGatherNonRoot); + + CPPUNIT_TEST( TestGatherV); + + CPPUNIT_TEST( TestIprobe); + CPPUNIT_TEST( TestIbarrier); + + CPPUNIT_TEST_SUITE_END(); + + public: + + void TestSimple() { + auto comm = std::make_shared(0, 1023); + CPPUNIT_ASSERT_EQUAL(0, comm->Rank()); + CPPUNIT_ASSERT_EQUAL(1023, comm->Size()); + } + + void TestSend() { + // Send a message 0 -> 1 + auto comm = std::make_shared(0, 2); + int number = 42; + comm->RequireSend(number, 1, 0); + comm->Send(42, 1); + comm->ExpectationsAllCompleted(); + } + + void TestIsend() { + // Send a message 0 -> 1 + auto comm = std::make_shared(0, 2); + int number = 42; + comm->RequireSend(number, 1, 0); + auto req = comm->Isend(42, 1); + req->Wait(); + comm->ExpectationsAllCompleted(); + } + + void TestRecv() { + // Send a message 0 -> 1 + auto comm = std::make_shared(1, 2); + int number = 42; + comm->RequireRecv(number, 1, 0); + int ans; + comm->Recv(ans, 1); + comm->ExpectationsAllCompleted(); + CPPUNIT_ASSERT_EQUAL(number, ans); + } + + void TestIrecv() { + // Send a message 0 -> 1 + auto comm = std::make_shared(1, 2); + int number = 42; + comm->RequireRecv(number, 1, 0); + int ans; + auto req = comm->Irecv(ans, 1); + req->Wait(); + comm->ExpectationsAllCompleted(); + CPPUNIT_ASSERT_EQUAL(number, ans); + } + + void TestGatherRoot() { + const int rank = 2; + const int size = 8; + auto comm = std::make_shared(rank, size); + + std::vector vals{9,8,7,6,5,4,3,2}; + comm->RequireGather(vals, 2); + + int myval = 7; + auto ans = comm->Gather(myval, 2); + + CPPUNIT_ASSERT_EQUAL(size, int(ans.size())); + comm->ExpectationsAllCompleted(); + } + void TestGatherNonRoot() { + const int rank = 0; + const int size = 8; + auto comm = std::make_shared(rank, size); + + std::vector vals{9,8,7,6,5,4,3,2}; + comm->RequireGather(vals, 2); + + int myval = 9; + auto ans = comm->Gather(myval, 2); + + CPPUNIT_ASSERT_EQUAL(0, int(ans.size())); + comm->ExpectationsAllCompleted(); + } + + void TestGatherV() { + const int rank = 0; + const int size = 4; + auto comm = std::make_shared(rank, size); + + std::vector counts{1,2,3,0}; + std::vector data{3, 4,5, 6,7,8}; + comm->RequireGatherV(data, counts, 0); + + std::vector mydata{3}; + auto ans = comm->GatherV(mydata, counts, 0); + CPPUNIT_ASSERT_EQUAL(data, ans); + comm->ExpectationsAllCompleted(); + } + + void TestIprobe() { + const int rank = 0; + const int size = 4; + auto comm = std::make_shared(rank, size); + + int val1 = 42; + int val2 = 24; + comm->RequireRecv(val1, 1, 0); + comm->RequireRecv(val2, 2, 0); + // message q has recvs ordered (1,2) + MPI_Status stat; + CPPUNIT_ASSERT(comm->Iprobe(2, MPI_ANY_TAG, &stat)); + // messages should now be ordered (2,1) + int ans; + comm->Recv(ans, 2); + CPPUNIT_ASSERT_EQUAL(val2, ans); + + comm->Recv(ans, 1); + CPPUNIT_ASSERT_EQUAL(val1, ans); + + CPPUNIT_ASSERT(!comm->Iprobe(MPI_ANY_SOURCE, MPI_ANY_TAG, &stat)); + comm->ExpectationsAllCompleted(); + } + + void TestIbarrier() { + const int rank = 0; + const int size = 4; + auto comm = std::make_shared(rank, size); + + // Try a dead simple one first + comm->RequireIbarrier([]() { + return true; + }); + auto req = comm->Ibarrier(); + CPPUNIT_ASSERT(req->Test()); + + // Now this guy will count the calls and only work on the 3rd go + int i = 0; + comm->RequireIbarrier([&i]() { + return ++i == 3; + }); + req = comm->Ibarrier(); + CPPUNIT_ASSERT(!req->Test()); + CPPUNIT_ASSERT(!req->Test()); + req->Wait(); + + comm->ExpectationsAllCompleted(); + } + }; + CPPUNIT_TEST_SUITE_REGISTRATION( MockCommunicatorTests); + } + } +} + +#endif diff --git a/Code/unittests/helpers/MockNetHelper.h b/Code/unittests/helpers/MockNetHelper.h deleted file mode 100644 index ccba5f125..000000000 --- a/Code/unittests/helpers/MockNetHelper.h +++ /dev/null @@ -1,70 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_HELPERS_MOCKNETHELPER_H -#define HEMELB_UNITTESTS_HELPERS_MOCKNETHELPER_H - -#include "unittests/net/NetMock.h" -namespace hemelb -{ - namespace unittests - { - namespace helpers - { - class MockMpiCommunicator : public net::MpiCommunicator - { - public: - /*** - * Constructor for a dummy communicator - * Can be useful for testing but can't actually be used - * @param rank - * @param size - */ - MockMpiCommunicator(int rank_, int size_) : - MpiCommunicator(), rank(rank_), size(size_) - { - - } - - virtual inline int Rank() const - { - return rank; - } - virtual inline int Size() const - { - return size; - } - private: - int rank, size; - }; - - class MockNetHelper - { - protected: - MockNetHelper() : - communicatorMock(NULL), netMock(NULL) - { - } - void setUp(const proc_t core_count, const proc_t current_core) - { - communicatorMock = new MockMpiCommunicator(current_core, core_count); - netMock = new net::NetMock(*communicatorMock); - } - void tearDown() - { - delete communicatorMock; - delete netMock; - } - - net::MpiCommunicator *communicatorMock; - net::NetMock *netMock; - }; - - } - } -} - -#endif // HEMELB_UNITTESTS_HELPERS_RANDOMSOURCE_H diff --git a/Code/unittests/helpers/helpers.h b/Code/unittests/helpers/helpers.h index 647f44859..aaa9708fc 100644 --- a/Code/unittests/helpers/helpers.h +++ b/Code/unittests/helpers/helpers.h @@ -8,5 +8,6 @@ #define HEMELB_UNITTESTS_HELPERS_HELPERS_H #include "unittests/helpers/RandomSourceTests.h" +#include "unittests/helpers/MockCommunicatorTests.h" #endif /* HEMELB_UNITTESTS_HELPERS_HELPERS_H */ diff --git a/Code/unittests/io/PathManagerTests.h b/Code/unittests/io/PathManagerTests.h index 3e073f58f..0c52a1580 100644 --- a/Code/unittests/io/PathManagerTests.h +++ b/Code/unittests/io/PathManagerTests.h @@ -29,15 +29,11 @@ namespace hemelb void setUp() { FolderTestFixture::setUp(); - argc = 7; + argc = 3; processorCount = 5; argv[0] = "hemelb"; - argv[2] = "config.xml"; argv[1] = "-in"; - argv[3] = "-i"; - argv[4] = "1"; - argv[5] = "-ss"; - argv[6] = "1111"; + argv[2] = "config.xml"; } void tearDown() @@ -50,7 +46,6 @@ namespace hemelb { ConstructManager(); AssertPresent("results"); - AssertPresent("results/Images"); } void TestNameInventionLocalConfig() @@ -64,7 +59,6 @@ namespace hemelb { ConstructPathConfigManager(); AssertPresent("results"); - AssertPresent("results/Images"); } void TestNameInventionPathConfig() diff --git a/Code/unittests/lbtests/BroadcastMocks.h b/Code/unittests/lbtests/BroadcastMocks.h deleted file mode 100644 index 3c73460f2..000000000 --- a/Code/unittests/lbtests/BroadcastMocks.h +++ /dev/null @@ -1,298 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_LBTESTS_BROADCASTMOCKS_H -#define HEMELB_UNITTESTS_LBTESTS_BROADCASTMOCKS_H - -#include "net/PhasedBroadcastRegular.h" -#include "lb/SimulationState.h" -#include "net/net.h" - -/** - * This file defines two PhasedBroadcastRegular mocks. One that behaves like the root - * node of the broadcast tree and another for a leaf node. - * - * @todo Split file into .h and .cc ? - */ - -namespace hemelb -{ - namespace net - { - /** - * In this mock, we pretend that the current process is the root node of a phased - * broadcast and that a pair of values is going up the tree. The mock simulates - * three cycles of communications (i.e. three messages travelling up the tree and - * reaching the root node): - * 1) All children report (14,15). - * 2) One child reports (1,100) and the rest (14,15). - * 3) All back to (14,15). - */ - class BroadcastMockRootNode : public net::PhasedBroadcastRegular<> - { - - public: - BroadcastMockRootNode(net::Net * net, const lb::SimulationState * simState, unsigned int spreadFactor); - - virtual ~BroadcastMockRootNode(); - - /* - * Overwritten IteraredAction methods that implement the mock. - */ - void RequestComms(); - void PostReceive(); - void EndIteration(); - - protected: - /** - * Receives data from each child. This is a set length per child, and each child's - * data is inserted contiguously into the provided array. - * - * @param dataStart Pointer to the start of the array. - * @param countPerChild Number of elements to receive per child. - */ - template - void ReceiveFromChildren(T* dataStart, int countPerChild); - - private: - /** Number of children nodes in the tree */ - unsigned spreadFactor; - - /** Number of times that the phased broadcast has been completed */ - unsigned callCounter; - }; - - BroadcastMockRootNode::BroadcastMockRootNode(net::Net * net, - const lb::SimulationState * simState, - unsigned int spreadFactor) : - net::PhasedBroadcastRegular<>(net, simState, spreadFactor), spreadFactor(spreadFactor), callCounter(0) - { - } - - BroadcastMockRootNode::~BroadcastMockRootNode() - { - } - - void BroadcastMockRootNode::RequestComms() - { - // Action taken when a node has to receive from its children in the tree. Calls ReceiveFromChildren mock below. - ProgressFromChildren(0); - } - - void BroadcastMockRootNode::PostReceive() - { - // Action taken after data has been received from its children - PostReceiveFromChildren(0); - - // Action taken after data has been sent to its parent - PostSendToParent(0); - } - - void BroadcastMockRootNode::EndIteration() - { - // Action taken when upwards-travelling data reaches the top node - TopNodeAction(); - - // Action taken by all nodes when downwards-travelling data has been sent to every node. - Effect(); - } - - template - void BroadcastMockRootNode::ReceiveFromChildren(T* dataStart, int countPerChild) - { - assert(countPerChild == 3); - - switch (callCounter++) - { - case 0: - for (unsigned childIndex = 0; childIndex < spreadFactor; childIndex++) - { - dataStart[childIndex * countPerChild] = 14.0; - dataStart[childIndex * countPerChild + 1] = 15.0; - dataStart[childIndex * countPerChild + 2] = 0.01; - } - break; - - case 1: - for (unsigned childIndex = 0; childIndex < spreadFactor - 1; childIndex++) - { - dataStart[childIndex * countPerChild] = 14.0; - dataStart[childIndex * countPerChild + 1] = 15.0; - dataStart[childIndex * countPerChild + 2] = 1.0; - } - dataStart[ (spreadFactor - 1) * countPerChild] = 1.0; - dataStart[ (spreadFactor - 1) * countPerChild + 1] = 100.0; - dataStart[ (spreadFactor - 1) * countPerChild + 2] = 10.0; - break; - - case 2: - for (unsigned childIndex = 0; childIndex < spreadFactor; childIndex++) - { - dataStart[childIndex * countPerChild] = 14.0; - dataStart[childIndex * countPerChild + 1] = 15.0; - dataStart[childIndex * countPerChild + 2] = 0.01; - } - break; - - default: - // Sanity check. Control should never reach this branch - assert(false); - } - } - - /** - * In this mock, we pretend that the current process is a leaf node of a phased - * broadcast and that a pair of values is going down and up the tree. The mock - * simulates three complete executions of the phased broadcast algorithm. For all - * three executions, the current leaf node reports (12, 21.45) to its parent (upwards - * tree pass). For the downwards pass, the current leaf receives: - * 1) unitialised data. - * 2) the min and max values reported by the current leaf node itself (12, 21.45). - * No other hypothetical leaf node has reported any smaller min or larger max densities. - * 3) (1,100). Another hypothetical leaf node reported these values during the previous - * upwards pass. - */ - class BroadcastMockLeafNode : public net::PhasedBroadcastRegular<> - { - - public: - BroadcastMockLeafNode(net::Net * net, const lb::SimulationState * simState, unsigned int spreadFactor); - - virtual ~BroadcastMockLeafNode(); - - /* - * Overwritten IteraredAction methods that implement the mock. - */ - void RequestComms(); - void PostReceive(); - void EndIteration(); - - protected: - /** - * Receives data from each child. This is a set length per child, and each child's - * data is inserted contiguously into the provided array. - * - * @param dataStart Pointer to the start of the array. - * @param countPerChild Number of elements to receive per child. - */ - template - void ReceiveFromParent(T* dataStart, int countPerChild); - - /** - * Helper function for sending data to parent nodes. - */ - template - void SendToParent(T* data, int count); - - private: - /** Number of children nodes in the tree */ - // unsigned spreadFactor; - - /** Number of iterations of the phased broadcast algorithm */ - unsigned iterationCounter; - - /** Minimum and maximum values sent up by the leaf node */ - distribn_t minSentUp, maxSentUp, maxVelSentUp; - - /** - * Helper method to find out whether we are in a downward pass - * - * @param iterationCounter iteration number in the phased broadcast algorithm - * @return whether we are in a downward pass - */ - bool DownwardPass(unsigned iterationCounter); - }; - - BroadcastMockLeafNode::BroadcastMockLeafNode(net::Net * net, - const lb::SimulationState * simState, - unsigned int spreadFactor) : - net::PhasedBroadcastRegular<>(net, simState, spreadFactor), /*spreadFactor(spreadFactor),*/ iterationCounter(0) - { - } - - BroadcastMockLeafNode::~BroadcastMockLeafNode() - { - } - - void BroadcastMockLeafNode::RequestComms() - { - if (DownwardPass(iterationCounter)) - { - // Action taken when a node has to receive from its parent in the tree. Calls ReceiveFromParent mock below. - ProgressFromParent(0); - } - else - { - // Action taken when a node has to send to its parent in the tree. Calls SendToParent mock below - ProgressToParent(0); - } - - iterationCounter++; - } - - void BroadcastMockLeafNode::PostReceive() - { - // Action taken after data has been sent to its parent - PostSendToParent(0); - } - - void BroadcastMockLeafNode::EndIteration() - { - // Action taken by all nodes when downwards-travelling data has been sent to every node. - Effect(); - } - - template - void BroadcastMockLeafNode::ReceiveFromParent(T* dataStart, int countPerChild) - { - assert(countPerChild == 3); - - switch (iterationCounter) - { - case 0: - // The first pass down contains rubbish - dataStart[0] = DBL_MAX; - dataStart[1] = -DBL_MAX; - dataStart[1] = 0; - break; - - case 2: - dataStart[0] = minSentUp; - dataStart[1] = maxSentUp; - dataStart[2] = maxVelSentUp; - break; - - case 4: - dataStart[0] = 1.0; - dataStart[1] = 100.0; - dataStart[2] = 10.0; - break; - - default: - // Sanity check. Control should never reach this branch - assert(false); - } - - } - - template - void BroadcastMockLeafNode::SendToParent(T* data, int count) - { - assert(count == 3); - minSentUp = data[0]; - maxSentUp = data[1]; - maxVelSentUp = data[2]; - } - - bool BroadcastMockLeafNode::DownwardPass(unsigned iterationCounter) - { - return (iterationCounter % 2 == 0); - } - - } -} - -#endif /* HEMELB_UNITTESTS_LBTESTS_BROADCASTMOCKS_H */ diff --git a/Code/unittests/lbtests/CollisionTests.h b/Code/unittests/lbtests/CollisionTests.h index b63548bc3..36b8e9776 100644 --- a/Code/unittests/lbtests/CollisionTests.h +++ b/Code/unittests/lbtests/CollisionTests.h @@ -30,199 +30,8 @@ namespace hemelb class CollisionTests : public helpers::FourCubeBasedTestFixture { CPPUNIT_TEST_SUITE( CollisionTests); - CPPUNIT_TEST( TestNonZeroVelocityEquilibriumFixedDensity); - CPPUNIT_TEST( TestZeroVelocityEquilibriumFixedDensity); - CPPUNIT_TEST( TestZeroVelocityEquilibrium); CPPUNIT_TEST( TestNormal);CPPUNIT_TEST_SUITE_END(); public: - void TestNonZeroVelocityEquilibriumFixedDensity() - { - lb::iolets::BoundaryValues inletBoundary(geometry::INLET_TYPE, - latDat, - simConfig->GetInlets(), - simState, - Comms(), - *unitConverter); - - initParams.boundaryObject = &inletBoundary; - - lb::collisions::NonZeroVelocityEquilibriumFixedDensity > - nonZeroVFixedDensityILet(initParams); - - distribn_t allowedError = 1e-10; - - // Initialise the fOld and the hydro vars. - distribn_t fOld[lb::lattices::D3Q15::NUMVECTORS]; - - LbTestsHelper::InitialiseAnisotropicTestData(0, fOld); - - lb::kernels::HydroVars > hydroVars(fOld); - - // Test the pre-collision step, which should calculate the correct - // post-collisional density, velocity and equilibrium distribution. - geometry::Site dummySite(0, *latDat); - latDat->SetIoletId(0, 0); - - nonZeroVFixedDensityILet.CalculatePreCollision(hydroVars, dummySite); - - // Calculate the expected density, velocity and f_eq. - distribn_t expectedRho = inletBoundary.GetBoundaryDensity(0); - distribn_t expectedMomentum[3]; - - distribn_t originalRho; - LbTestsHelper::CalculateRhoMomentum(fOld, originalRho, expectedMomentum); - - // Now need to scale the momentum, expectedV, to account for the difference between - // original and enforced densities. - for (unsigned axis = 0; axis < 3; ++axis) - expectedMomentum[axis] *= (expectedRho / originalRho); - - distribn_t expectedFeq[lb::lattices::D3Q15::NUMVECTORS]; - LbTestsHelper::CalculateLBGKEqmF(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq); - - // Compare. - LbTestsHelper::CompareHydros(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq, - "Non-0 velocity eqm fixed density, calculate pre collision", - hydroVars, - allowedError); - - // Next, compare the collision function itself. The result should be the equilibrium - // distribution. - nonZeroVFixedDensityILet.Collide(lbmParams, hydroVars); - - for (unsigned int ii = 0; ii < lb::lattices::D3Q15::NUMVECTORS; ++ii) - { - CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("Non-0 velocity eqm fixed density, collide", - expectedFeq[ii], - hydroVars.GetFPostCollision()[ii], - allowedError); - } - } - - void TestZeroVelocityEquilibriumFixedDensity() - { - lb::iolets::BoundaryValues outletBoundary(geometry::OUTLET_TYPE, - latDat, - simConfig->GetOutlets(), - simState, - Comms(), - *unitConverter); - initParams.boundaryObject = &outletBoundary; - - lb::collisions::ZeroVelocityEquilibriumFixedDensity > - zeroVFixedDensityOLet(initParams); - - distribn_t allowedError = 1e-10; - - // Initialise the fOld and the hydro vars. - distribn_t fOld[lb::lattices::D3Q15::NUMVECTORS]; - - LbTestsHelper::InitialiseAnisotropicTestData(0, fOld); - - lb::kernels::HydroVars > hydroVars(fOld); - - // Test the pre-collision step, which should calculate the correct - // post-collisional density, velocity and equilibrium distribution. - latDat->SetIoletId(0, 0); - zeroVFixedDensityOLet.CalculatePreCollision(hydroVars, latDat->GetSite(0)); - - // Calculate the expected density, velocity and f_eq. - distribn_t expectedRho = outletBoundary.GetBoundaryDensity(0); - distribn_t expectedMomentum[3] = { 0., 0., 0. }; - - distribn_t expectedFeq[lb::lattices::D3Q15::NUMVECTORS]; - LbTestsHelper::CalculateLBGKEqmF(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq); - - // Compare. - LbTestsHelper::CompareHydros(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq, - "0 velocity eqm fixed density, calculate pre collision", - hydroVars, - allowedError); - - // Next, compare the collision function itself. The result should be the equilibrium - // distribution. - zeroVFixedDensityOLet.Collide(lbmParams, hydroVars); - - for (unsigned int ii = 0; ii < lb::lattices::D3Q15::NUMVECTORS; ++ii) - { - CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("0 velocity eqm fixed density, collide", - expectedFeq[ii], - hydroVars.GetFPostCollision()[ii], - allowedError); - } - } - - void TestZeroVelocityEquilibrium() - { - lb::collisions::ZeroVelocityEquilibrium > zeroVEqm(initParams); - - distribn_t allowedError = 1e-10; - - // Initialise the fOld and the hydro vars. - distribn_t fOld[lb::lattices::D3Q15::NUMVECTORS]; - - LbTestsHelper::InitialiseAnisotropicTestData(0, fOld); - - lb::kernels::HydroVars > hydroVars(fOld); - - // Test the pre-collision step, which should calculate the correct - // post-collisional density, velocity and equilibrium distribution. - zeroVEqm.CalculatePreCollision(hydroVars, latDat->GetSite(0)); - - // Calculate the expected density, velocity and f_eq. - distribn_t expectedRho = 0.0; - distribn_t expectedMomentum[3] = { 0., 0., 0. }; - - for (unsigned int ii = 0; ii < lb::lattices::D3Q15::NUMVECTORS; ++ii) - { - expectedRho += fOld[ii]; - } - - distribn_t expectedFeq[lb::lattices::D3Q15::NUMVECTORS]; - LbTestsHelper::CalculateLBGKEqmF(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq); - - // Compare. - LbTestsHelper::CompareHydros(expectedRho, - expectedMomentum[0], - expectedMomentum[1], - expectedMomentum[2], - expectedFeq, - "0 velocity eqm, calculate pre collision", - hydroVars, - allowedError); - - // Next, compare the collision function itself. The result should be the equilibrium - // distribution. - zeroVEqm.Collide(lbmParams, hydroVars); - - for (unsigned int ii = 0; ii < lb::lattices::D3Q15::NUMVECTORS; ++ii) - { - CPPUNIT_ASSERT_DOUBLES_EQUAL_MESSAGE("0 velocity eqm, collide", - expectedFeq[ii], - hydroVars.GetFPostCollision()[ii], - allowedError); - } - } void TestNormal() { diff --git a/Code/unittests/lbtests/IncompressibilityCheckerTests.h b/Code/unittests/lbtests/IncompressibilityCheckerTests.h index 1aace7791..7f28158ab 100644 --- a/Code/unittests/lbtests/IncompressibilityCheckerTests.h +++ b/Code/unittests/lbtests/IncompressibilityCheckerTests.h @@ -9,12 +9,14 @@ #include -#include "lb/IncompressibilityChecker.hpp" +#include "lb/IncompressibilityChecker.h" +#include "timestep/TimeStepManager.h" + #include "unittests/FourCubeLatticeData.h" -#include "unittests/lbtests/BroadcastMocks.h" #include "unittests/reporting/Mocks.h" #include "unittests/helpers/FourCubeBasedTestFixture.h" +#include "debug/Debugger.h" namespace hemelb { @@ -26,8 +28,7 @@ namespace hemelb { CPPUNIT_TEST_SUITE (IncompressibilityCheckerTests); CPPUNIT_TEST (TestIncompressibilityCheckerRootNode); - CPPUNIT_TEST (TestIncompressibilityCheckerLeafNode); - CPPUNIT_TEST_SUITE_END(); + CPPUNIT_TEST (TestIncompressibilityCheckerLeafNode);CPPUNIT_TEST_SUITE_END(); public: void setUp() @@ -38,7 +39,9 @@ namespace hemelb cache->densityCache.SetRefreshFlag(); cache->velocityCache.SetRefreshFlag(); - lbtests::LbTestsHelper::UpdatePropertyCache(*latDat, *cache, *simState); + lbtests::LbTestsHelper::UpdatePropertyCache(*latDat, + *cache, + *simState); // These are the smallest and largest density values in FourCubeLatticeData by default //! @23 The lattice class below must be consistent with the one used in FourCubeLatticeData. Consider templating FourCubeLatticeData over lattice class, so both can be controlled from the test. @@ -52,7 +55,6 @@ namespace hemelb eps = 1e-9; timings = new hemelb::reporting::Timers(Comms()); - net = new net::Net(Comms()); } void tearDown() @@ -61,33 +63,31 @@ namespace hemelb FourCubeBasedTestFixture::tearDown(); } - void AdvanceActorOneTimeStep(net::IteratedAction& actor) + void AdvanceActorOneTimeStep(timestep::Actor& actor) { - cache->densityCache.SetRefreshFlag(); - LbTestsHelper::UpdatePropertyCache(*latDat, *cache, *simState); - - actor.RequestComms(); - actor.PreSend(); - actor.PreReceive(); - actor.PostReceive(); - actor.EndIteration(); - } + timestep::TimeStepManager tsm(1); + tsm.AddToPhase(0, &actor); + tsm.DoStep(); + } void TestIncompressibilityCheckerRootNode() { - hemelb::lb::IncompressibilityChecker incompChecker(latDat, - net, - simState, - *cache, - *timings, - 10.0); // Will accept a max/min of (21.45, 12) but not (100,1) + hemelb::lb::IncompressibilityChecker incompChecker(latDat, + Comms(), + simState, + *cache, + *timings, + 10.0); // Will accept a max/min of (21.45, 12) but not (100,1) // Not available until the first broadcast has finished - CPPUNIT_ASSERT(!incompChecker.AreDensitiesAvailable()); AdvanceActorOneTimeStep(incompChecker); - CPPUNIT_ASSERT_DOUBLES_EQUAL(smallestDefaultDensity, incompChecker.GetGlobalSmallestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(largestDefaultDensity, incompChecker.GetGlobalLargestDensity(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(smallestDefaultDensity, + incompChecker.GetGlobalSmallestDensity(), + eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(largestDefaultDensity, + incompChecker.GetGlobalLargestDensity(), + eps); CPPUNIT_ASSERT_DOUBLES_EQUAL( (largestDefaultDensity - smallestDefaultDensity) / hemelb::lb::REFERENCE_DENSITY, incompChecker.GetMaxRelativeDensityDifference(), @@ -97,48 +97,60 @@ namespace hemelb incompChecker.GetGlobalLargestVelocityMagnitude(), eps); - // The broadcast mock injects some smaller and larger densities (1,100) coming from one of the children - CPPUNIT_ASSERT(incompChecker.AreDensitiesAvailable()); + // Insert values at 1 & 100 + debug::Debugger::Get()->BreakHere(); + cache->densityCache.Put(0, 1.0); + cache->densityCache.Put(1, 100.0); + cache->velocityCache.Put(2, LatticeVelocity(10.0, 0., 0.0)); + AdvanceActorOneTimeStep(incompChecker); CPPUNIT_ASSERT_DOUBLES_EQUAL(1.0, incompChecker.GetGlobalSmallestDensity(), eps); CPPUNIT_ASSERT_DOUBLES_EQUAL(100.0, incompChecker.GetGlobalLargestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(99.0, incompChecker.GetMaxRelativeDensityDifference(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(99.0, + incompChecker.GetMaxRelativeDensityDifference(), + eps); CPPUNIT_ASSERT(!incompChecker.IsDensityDiffWithinRange()); - CPPUNIT_ASSERT_DOUBLES_EQUAL(10.0, incompChecker.GetGlobalLargestVelocityMagnitude(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(10.0, + incompChecker.GetGlobalLargestVelocityMagnitude(), + eps); - // The previous values are not reported by any children anymore. Testing that the checker remembers them - CPPUNIT_ASSERT(incompChecker.AreDensitiesAvailable()); + // Reset the previous values. Testing that the checker remembers the extrema + cache->densityCache.SetRefreshFlag(); + lbtests::LbTestsHelper::UpdatePropertyCache(*latDat, + *cache, + *simState); AdvanceActorOneTimeStep(incompChecker); CPPUNIT_ASSERT_DOUBLES_EQUAL(1.0, incompChecker.GetGlobalSmallestDensity(), eps); CPPUNIT_ASSERT_DOUBLES_EQUAL(100.0, incompChecker.GetGlobalLargestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(99.0, incompChecker.GetMaxRelativeDensityDifference(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(99.0, + incompChecker.GetMaxRelativeDensityDifference(), + eps); CPPUNIT_ASSERT(!incompChecker.IsDensityDiffWithinRange()); - CPPUNIT_ASSERT_DOUBLES_EQUAL(10.0, incompChecker.GetGlobalLargestVelocityMagnitude(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(10.0, + incompChecker.GetGlobalLargestVelocityMagnitude(), + eps); } void TestIncompressibilityCheckerLeafNode() { - hemelb::lb::IncompressibilityChecker incompChecker(latDat, - net, - simState, - *cache, - *timings, - 10.0); // Will accept a max/min of (21.45, 12) but not (100,1) - - // First pass down the tree (uninitialised values being broadcasted) - AdvanceActorOneTimeStep(incompChecker); - - // First pass up the tree (root node gets min/max values reported by the leaf node being simulated) - AdvanceActorOneTimeStep(incompChecker); + hemelb::lb::IncompressibilityChecker incompChecker(latDat, + Comms(), + simState, + *cache, + *timings, + 10.0); // Will accept a max/min of (21.45, 12) but not (100,1) - // Second pass down the tree (all nodes get the min/max values reported by the leaf node being simulated) AdvanceActorOneTimeStep(incompChecker); // This time the current leaf node reported global minimum and maximum values - CPPUNIT_ASSERT_DOUBLES_EQUAL(smallestDefaultDensity, incompChecker.GetGlobalSmallestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(largestDefaultDensity, incompChecker.GetGlobalLargestDensity(), eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(smallestDefaultDensity, + incompChecker.GetGlobalSmallestDensity(), + eps); + CPPUNIT_ASSERT_DOUBLES_EQUAL(largestDefaultDensity, + incompChecker.GetGlobalLargestDensity(), + eps); CPPUNIT_ASSERT_DOUBLES_EQUAL( (largestDefaultDensity - smallestDefaultDensity) / hemelb::lb::REFERENCE_DENSITY, incompChecker.GetMaxRelativeDensityDifference(), @@ -147,18 +159,6 @@ namespace hemelb CPPUNIT_ASSERT_DOUBLES_EQUAL(largestDefaultVelocityMagnitude, incompChecker.GetGlobalLargestVelocityMagnitude(), eps); - - // Second pass up. The broadcast mock injects some smaller and larger densities (1,100) coming from another hypothetical leaf node. - AdvanceActorOneTimeStep(incompChecker); - - // Third pass down. (1,100) arrives to all the leaf nodes. - AdvanceActorOneTimeStep(incompChecker); - - CPPUNIT_ASSERT_DOUBLES_EQUAL(1.0, incompChecker.GetGlobalSmallestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(100.0, incompChecker.GetGlobalLargestDensity(), eps); - CPPUNIT_ASSERT_DOUBLES_EQUAL(99.0, incompChecker.GetMaxRelativeDensityDifference(), eps); - CPPUNIT_ASSERT(!incompChecker.IsDensityDiffWithinRange()); - CPPUNIT_ASSERT_DOUBLES_EQUAL(10.0, incompChecker.GetGlobalLargestVelocityMagnitude(), eps); } private: @@ -167,7 +167,6 @@ namespace hemelb distribn_t largestDefaultDensity; distribn_t largestDefaultVelocityMagnitude; hemelb::reporting::Timers* timings; - net::Net* net; distribn_t eps; }; diff --git a/Code/unittests/lbtests/StreamerTests.h b/Code/unittests/lbtests/StreamerTests.h index d79a03bf5..52885010c 100644 --- a/Code/unittests/lbtests/StreamerTests.h +++ b/Code/unittests/lbtests/StreamerTests.h @@ -33,14 +33,14 @@ namespace hemelb */ class StreamerTests : public helpers::FourCubeBasedTestFixture { - CPPUNIT_TEST_SUITE ( StreamerTests); - CPPUNIT_TEST ( TestSimpleCollideAndStream); - CPPUNIT_TEST ( TestBouzidiFirdaousLallemand); - CPPUNIT_TEST ( TestSimpleBounceBack); - CPPUNIT_TEST ( TestGuoZhengShi); - CPPUNIT_TEST ( TestNashZerothOrderPressureIolet); - CPPUNIT_TEST ( TestNashZerothOrderPressureBB); - CPPUNIT_TEST ( TestJunkYangEquivalentToBounceBack);CPPUNIT_TEST_SUITE_END(); + CPPUNIT_TEST_SUITE (StreamerTests); + CPPUNIT_TEST (TestSimpleCollideAndStream); + CPPUNIT_TEST (TestBouzidiFirdaousLallemand); + CPPUNIT_TEST (TestSimpleBounceBack); + CPPUNIT_TEST (TestGuoZhengShi); + CPPUNIT_TEST (TestNashZerothOrderPressureIolet); + CPPUNIT_TEST (TestNashZerothOrderPressureBB); + CPPUNIT_TEST (TestJunkYangEquivalentToBounceBack);CPPUNIT_TEST_SUITE_END(); public: void setUp() @@ -48,8 +48,8 @@ namespace hemelb FourCubeBasedTestFixture::setUp(); propertyCache = new lb::MacroscopicPropertyCache(*simState, *latDat); - normalCollision - = new lb::collisions::Normal >(initParams); + normalCollision = + new lb::collisions::Normal >(initParams); } void tearDown() @@ -62,8 +62,8 @@ namespace hemelb void TestSimpleCollideAndStream() { - lb::streamers::SimpleCollideAndStream > > simpleCollideAndStream(initParams); + lb::streamers::SimpleCollideAndStream< + lb::collisions::Normal > > simpleCollideAndStream(initParams); // Initialise fOld in the lattice data. We choose values so that each site has // an anisotropic distribution function, and that each site's function is @@ -71,32 +71,32 @@ namespace hemelb LbTestsHelper::InitialiseAnisotropicTestData(latDat); // Use the streaming operator on the entire lattice. - simpleCollideAndStream.StreamAndCollide (0, - latDat->GetLocalFluidSiteCount(), - lbmParams, - latDat, - *propertyCache); + simpleCollideAndStream.StreamAndCollide(0, + latDat->GetLocalFluidSiteCount(), + lbmParams, + latDat, + *propertyCache); // Now, go over each lattice site and check each value in f_new is correct. - for (site_t streamedToSite = 0; streamedToSite < latDat->GetLocalFluidSiteCount(); ++streamedToSite) + for (site_t streamedToSite = 0; streamedToSite < latDat->GetLocalFluidSiteCount(); + ++streamedToSite) { - geometry::Site < geometry::LatticeData > streamedSite - = latDat->GetSite(streamedToSite); + geometry::Site streamedSite = latDat->GetSite(streamedToSite); distribn_t* streamedToFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * streamedToSite); - for (unsigned int streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (unsigned int streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { - site_t - streamerIndex = - streamedSite.GetStreamedIndex (lb::lattices::D3Q15::INVERSEDIRECTIONS[streamedDirection]); + site_t streamerIndex = + streamedSite.GetStreamedIndex(lb::lattices::D3Q15::INVERSEDIRECTIONS[streamedDirection]); // If this site streamed somewhere sensible, it must have been streamed to. - if (streamerIndex >= 0 && streamerIndex < (lb::lattices::D3Q15::NUMVECTORS - * latDat->GetLocalFluidSiteCount())) + if (streamerIndex >= 0 + && streamerIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { site_t streamerSiteId = streamerIndex / lb::lattices::D3Q15::NUMVECTORS; @@ -106,8 +106,7 @@ namespace hemelb streamerFOld); // Calculate what the value streamed to site streamedToSite should be. - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamedSite); normalCollision->Collide(lbmParams, streamerHydroVars); @@ -129,22 +128,19 @@ namespace hemelb // an anisotropic distribution function, and that each site's function is // distinguishable. LbTestsHelper::InitialiseAnisotropicTestData(latDat); - lb::streamers::BouzidiFirdaousLallemand > >::Type bfl(initParams); - - bfl.StreamAndCollide (0, - latDat->GetLocalFluidSiteCount(), - lbmParams, - latDat, - *propertyCache); - bfl.PostStep (0, + lb::streamers::BouzidiFirdaousLallemand< + lb::collisions::Normal > >::Type bfl(initParams); + + bfl.StreamAndCollide(0, latDat->GetLocalFluidSiteCount(), lbmParams, latDat, *propertyCache); + bfl.PostStep(0, latDat->GetLocalFluidSiteCount(), lbmParams, latDat, *propertyCache); // Now, go over each lattice site and check each value in f_new is correct. - for (site_t streamedToSite = 0; streamedToSite < latDat->GetLocalFluidSiteCount(); ++streamedToSite) + for (site_t streamedToSite = 0; streamedToSite < latDat->GetLocalFluidSiteCount(); + ++streamedToSite) { const geometry::Site streamedSite = latDat->GetSite(streamedToSite); @@ -152,21 +148,21 @@ namespace hemelb distribn_t* streamedToFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * streamedToSite); - for (unsigned int streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (unsigned int streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { unsigned int oppDirection = lb::lattices::D3Q15::INVERSEDIRECTIONS[streamedDirection]; site_t streamerIndex = - streamedSite.GetStreamedIndex (oppDirection); + streamedSite.GetStreamedIndex(oppDirection); - geometry::Site < geometry::LatticeData > streamerSite - = latDat->GetSite(streamerIndex); + geometry::Site streamerSite = latDat->GetSite(streamerIndex); // If this site streamed somewhere sensible, it must have been streamed to. - if (streamerIndex >= 0 && streamerIndex < (lb::lattices::D3Q15::NUMVECTORS - * latDat->GetLocalFluidSiteCount())) + if (streamerIndex >= 0 + && streamerIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { site_t streamerSiteId = streamerIndex / lb::lattices::D3Q15::NUMVECTORS; @@ -176,8 +172,7 @@ namespace hemelb streamerFOld); // Calculate what the value streamed to site streamedToSite should be. - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamerSite); normalCollision->Collide(lbmParams, streamerHydroVars); @@ -202,18 +197,18 @@ namespace hemelb << " Data: " << streamedSite.GetSiteData().GetWallIntersectionData() << std::flush; CPPUNIT_ASSERT_MESSAGE("Expected to find a boundary " - "opposite an unstreamed-to direction " + message.str(), + "opposite an unstreamed-to direction " + message.str(), streamedSite.HasWall(oppDirection)); // Test disabled due to RegressionTests issue, see discussion in #87 CPPUNIT_ASSERT_MESSAGE("Expect defined cut distance opposite an unstreamed-to direction " + message.str(), - streamedSite.GetWallDistance (oppDirection) + streamedSite.GetWallDistance(oppDirection) != -1.0); // To verify the operation of the BFL boundary condition, we'll need: // - the distance to the wall * 2 distribn_t twoQ = 2.0 - * streamedSite.GetWallDistance (oppDirection); + * streamedSite.GetWallDistance(oppDirection); // - the post-collision distribution at the current site. distribn_t streamedToSiteFOld[lb::lattices::D3Q15::NUMVECTORS]; @@ -222,8 +217,7 @@ namespace hemelb LbTestsHelper::InitialiseAnisotropicTestData(streamedToSite, streamedToSiteFOld); - lb::kernels::HydroVars > - hydroVars(streamedToSiteFOld); + lb::kernels::HydroVars > hydroVars(streamedToSiteFOld); normalCollision->CalculatePreCollision(hydroVars, streamedSite); @@ -235,12 +229,12 @@ namespace hemelb distribn_t awayFromWallFOld[lb::lattices::D3Q15::NUMVECTORS]; site_t awayFromWallIndex = - streamedSite.GetStreamedIndex (streamedDirection) + streamedSite.GetStreamedIndex(streamedDirection) / lb::lattices::D3Q15::NUMVECTORS; // If there's a valid index in that direction, use BFL - if (awayFromWallIndex >= 0 && awayFromWallIndex - < latDat->GetLocalFluidSiteCount()) + if (awayFromWallIndex >= 0 + && awayFromWallIndex < latDat->GetLocalFluidSiteCount()) { const geometry::Site awayFromWallSite = latDat->GetSite(awayFromWallIndex); @@ -249,10 +243,10 @@ namespace hemelb LbTestsHelper::InitialiseAnisotropicTestData(awayFromWallIndex, awayFromWallFOld); - lb::kernels::HydroVars > - awayFromWallsHydroVars(awayFromWallFOld); + lb::kernels::HydroVars > awayFromWallsHydroVars(awayFromWallFOld); - normalCollision->CalculatePreCollision(awayFromWallsHydroVars, awayFromWallSite); + normalCollision->CalculatePreCollision(awayFromWallsHydroVars, + awayFromWallSite); // (find post-collision values using the collision operator). normalCollision->Collide(lbmParams, awayFromWallsHydroVars); @@ -264,9 +258,9 @@ namespace hemelb distribn_t oppWallOld = hydroVars.GetFPostCollision()[streamedDirection]; // The streamed value should be as given below. - distribn_t streamed = (twoQ < 1.0) - ? (toWallNew + twoQ * (toWallOld - toWallNew)) - : (oppWallOld + (1. / twoQ) * (toWallOld - oppWallOld)); + distribn_t streamed = (twoQ < 1.0) ? + (toWallNew + twoQ * (toWallOld - toWallNew)) : + (oppWallOld + (1. / twoQ) * (toWallOld - oppWallOld)); std::stringstream msg(std::stringstream::in); msg << "BouzidiFirdaousLallemand, PostStep: site " << streamedToSite @@ -318,34 +312,33 @@ namespace hemelb site_t offset = 0; // Mid-Fluid sites use simple collide and stream - lb::streamers::SimpleCollideAndStream > > simpleCollideAndStream(initParams); - - simpleCollideAndStream.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(0), - lbmParams, - latDat, - *propertyCache); + lb::streamers::SimpleCollideAndStream< + lb::collisions::Normal > > simpleCollideAndStream(initParams); + + simpleCollideAndStream.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(0), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(0); // Wall sites use simple bounce back - lb::streamers::SimpleBounceBack > >::Type simpleBounceBack(initParams); - - simpleBounceBack.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(1), - lbmParams, - latDat, - *propertyCache); + lb::streamers::SimpleBounceBack< + lb::collisions::Normal > >::Type simpleBounceBack(initParams); + + simpleBounceBack.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(1), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(1); // Consider inlet/outlets and their walls as mid-fluid sites - simpleCollideAndStream.StreamAndCollide (offset, - latDat->GetLocalFluidSiteCount() - - offset, - lbmParams, - latDat, - *propertyCache); + simpleCollideAndStream.StreamAndCollide(offset, + latDat->GetLocalFluidSiteCount() - offset, + lbmParams, + latDat, + *propertyCache); offset += latDat->GetLocalFluidSiteCount() - offset; // Sanity check @@ -358,7 +351,8 @@ namespace hemelb * depending on where they sit relative to the wall. We ignore mid-Fluid sites since * StreamAndCollide was tested before. */ - for (site_t wallSiteLocalIndex = 0; wallSiteLocalIndex < wallSitesCount; wallSiteLocalIndex++) + for (site_t wallSiteLocalIndex = 0; wallSiteLocalIndex < wallSitesCount; + wallSiteLocalIndex++) { site_t streamedToSite = firstWallSite + wallSiteLocalIndex; const geometry::Site streamedSite = @@ -366,18 +360,19 @@ namespace hemelb distribn_t* streamedToFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * streamedToSite); - for (unsigned int streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (unsigned int streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { unsigned oppDirection = lb::lattices::D3Q15::INVERSEDIRECTIONS[streamedDirection]; // Index of the site streaming to streamedToSite via direction streamedDirection site_t streamerIndex = - streamedSite.GetStreamedIndex (oppDirection); + streamedSite.GetStreamedIndex(oppDirection); // Is streamerIndex a valid index? - if (streamerIndex >= 0 && streamerIndex < (lb::lattices::D3Q15::NUMVECTORS - * latDat->GetLocalFluidSiteCount())) + if (streamerIndex >= 0 + && streamerIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { // The streamer index is a valid index in the domain, therefore stream and collide has happened site_t streamerSiteId = streamerIndex / lb::lattices::D3Q15::NUMVECTORS; @@ -388,8 +383,7 @@ namespace hemelb streamerFOld); // Calculate what the value streamed to site streamedToSite should be. - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamedSite); normalCollision->Collide(lbmParams, streamerHydroVars); @@ -410,8 +404,7 @@ namespace hemelb distribn_t streamerToSiteFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(streamedToSite, streamerToSiteFOld); - lb::kernels::HydroVars > - hydroVars(streamerToSiteFOld); + lb::kernels::HydroVars > hydroVars(streamerToSiteFOld); normalCollision->CalculatePreCollision(hydroVars, streamedSite); // Simulate post-collision using the collision operator. @@ -433,11 +426,11 @@ namespace hemelb void TestGuoZhengShi() { - lb::streamers::GuoZhengShi > >::Type guoZhengShi(initParams); + lb::streamers::GuoZhengShi< + lb::collisions::Normal > >::Type guoZhengShi(initParams); - for (double assignedWallDistance = 0.4; assignedWallDistance < 1.0; assignedWallDistance - += 0.5) + for (double assignedWallDistance = 0.4; assignedWallDistance < 1.0; + assignedWallDistance += 0.5) { // Initialise fOld in the lattice data. We choose values so that each site has // an anisotropic distribution function, and that each site's function is @@ -467,20 +460,19 @@ namespace hemelb assignedWallDistance); // Perform the collision and streaming. - guoZhengShi.StreamAndCollide (chosenSite, 1, lbmParams, latDat, *propertyCache); + guoZhengShi.StreamAndCollide(chosenSite, 1, lbmParams, latDat, *propertyCache); // Calculate the distributions at the chosen site up to post-collision. distribn_t streamerFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(chosenSite, streamerFOld); - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamer); normalCollision->Collide(lbmParams, streamerHydroVars); // Check each streamed direction. - for (Direction streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (Direction streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { switch (streamedDirection) { @@ -494,8 +486,8 @@ namespace hemelb // This is the first means of estimating from the source paper: only // use the nearest fluid site. - LatticeVelocity velocityEstimate1 = streamerHydroVars.momentum * (1. - 1. - / assignedWallDistance) / streamerHydroVars.density; + LatticeVelocity velocityEstimate1 = streamerHydroVars.momentum + * (1. - 1. / assignedWallDistance) / streamerHydroVars.density; distribn_t fNeqEstimate1 = streamerHydroVars.GetFNeq()[streamedDirection]; @@ -504,30 +496,28 @@ namespace hemelb // This is the second method for estimating: using the next fluid site // away from the wall. const site_t nextSiteAwayFromWall = streamer.GetStreamedIndex< - lb::lattices::D3Q15> (streamedDirection) - / lb::lattices::D3Q15::NUMVECTORS; + lb::lattices::D3Q15>(streamedDirection) / lb::lattices::D3Q15::NUMVECTORS; const geometry::Site& nextSiteAway = latDat->GetSite(nextSiteAwayFromWall); distribn_t nextSiteOutFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(nextSiteAwayFromWall, nextSiteOutFOld); - lb::kernels::HydroVars > - nextSiteOutHydroVars(nextSiteOutFOld); + lb::kernels::HydroVars > nextSiteOutHydroVars(nextSiteOutFOld); normalCollision->CalculatePreCollision(nextSiteOutHydroVars, nextSiteAway); LatticeVelocity velocityEstimate2 = nextSiteOutHydroVars.momentum - * (assignedWallDistance - 1.) / ( (1. + assignedWallDistance) - * nextSiteOutHydroVars.density); + * (assignedWallDistance - 1.) + / ( (1. + assignedWallDistance) * nextSiteOutHydroVars.density); distribn_t fNeqEstimate2 = nextSiteOutHydroVars.GetFNeq()[streamedDirection]; // The actual value is taken to be an interpolation between the two // estimates. - velocityWall = velocityEstimate1 * assignedWallDistance + velocityEstimate2 - * (1. - assignedWallDistance); + velocityWall = velocityEstimate1 * assignedWallDistance + + velocityEstimate2 * (1. - assignedWallDistance); - fNeqWall = assignedWallDistance * fNeqEstimate1 + (1. - assignedWallDistance) - * fNeqEstimate2; + fNeqWall = assignedWallDistance * fNeqEstimate1 + + (1. - assignedWallDistance) * fNeqEstimate2; } else { @@ -547,8 +537,8 @@ namespace hemelb momentumWall.z, fEqm); // Perform collision on the wall f's - distribn_t prediction = fEqm[streamedDirection] + (1.0 + lbmParams->GetOmega()) - * fNeqWall; + distribn_t prediction = fEqm[streamedDirection] + + (1.0 + lbmParams->GetOmega()) * fNeqWall; // This is the answer from the code we're testing distribn_t streamedFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * chosenSite)[streamedDirection]; @@ -576,8 +566,8 @@ namespace hemelb // This is the first means of estimating from the source paper: only // use the nearest fluid site. - LatticeVelocity velocityWall = streamerHydroVars.momentum * (1. - 1. - / assignedWallDistance) / streamerHydroVars.density; + LatticeVelocity velocityWall = streamerHydroVars.momentum + * (1. - 1. / assignedWallDistance) / streamerHydroVars.density; distribn_t fNeqWall = streamerHydroVars.GetFNeq()[streamedDirection]; @@ -593,8 +583,8 @@ namespace hemelb fEqm); // Perform collision on the wall f's - distribn_t prediction = fEqm[streamedDirection] + (1.0 - + lbmParams->GetOmega()) * fNeqWall; + distribn_t prediction = fEqm[streamedDirection] + + (1.0 + lbmParams->GetOmega()) * fNeqWall; // This is the answer from the code we're testing distribn_t streamedFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * chosenSite)[streamedDirection]; @@ -610,7 +600,7 @@ namespace hemelb default: // We have nothing to do with a wall so simple streaming const site_t streamedIndex = - streamer.GetStreamedIndex (streamedDirection); + streamer.GetStreamedIndex(streamedDirection); distribn_t streamedToFNew = *latDat->GetFNew(streamedIndex); // F_new should be equal to the value that was streamed from this other site @@ -647,44 +637,42 @@ namespace hemelb site_t offset = 0; // Mid-Fluid sites use simple collide and stream - lb::streamers::SimpleCollideAndStream > > simpleCollideAndStream(initParams); - - simpleCollideAndStream.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(0), - lbmParams, - latDat, - *propertyCache); + lb::streamers::SimpleCollideAndStream< + lb::collisions::Normal > > simpleCollideAndStream(initParams); + + simpleCollideAndStream.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(0), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(0); // Wall sites use junk and yang initParams.siteRanges.push_back(std::pair(offset, offset + latDat->GetMidDomainCollisionCount(1))); - lb::streamers::JunkYang > >::Type - junkYang(initParams); + lb::streamers::JunkYang > >::Type junkYang(initParams); - junkYang.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(1), - lbmParams, - latDat, - *propertyCache); - - junkYang.PostStep (offset, + junkYang.StreamAndCollide(offset, latDat->GetMidDomainCollisionCount(1), lbmParams, latDat, *propertyCache); + junkYang.PostStep(offset, + latDat->GetMidDomainCollisionCount(1), + lbmParams, + latDat, + *propertyCache); + offset += latDat->GetMidDomainCollisionCount(1); // Consider inlet/outlets and their walls as mid-fluid sites - simpleCollideAndStream.StreamAndCollide (offset, - latDat->GetLocalFluidSiteCount() - - offset, - lbmParams, - latDat, - *propertyCache); + simpleCollideAndStream.StreamAndCollide(offset, + latDat->GetLocalFluidSiteCount() - offset, + lbmParams, + latDat, + *propertyCache); offset += latDat->GetLocalFluidSiteCount() - offset; // Sanity check @@ -697,7 +685,8 @@ namespace hemelb * depending on where they sit relative to the wall. We ignore mid-Fluid sites since * StreamAndCollide was tested before. */ - for (site_t wallSiteLocalIndex = 0; wallSiteLocalIndex < wallSitesCount; wallSiteLocalIndex++) + for (site_t wallSiteLocalIndex = 0; wallSiteLocalIndex < wallSitesCount; + wallSiteLocalIndex++) { site_t streamedToSite = firstWallSite + wallSiteLocalIndex; const geometry::Site streamedSite = @@ -705,18 +694,19 @@ namespace hemelb distribn_t* streamedToFNew = latDat->GetFNew(lb::lattices::D3Q15::NUMVECTORS * streamedToSite); - for (unsigned int streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (unsigned int streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { unsigned oppDirection = lb::lattices::D3Q15::INVERSEDIRECTIONS[streamedDirection]; // Index of the site streaming to streamedToSite via direction streamedDirection site_t streamerIndex = - streamedSite.GetStreamedIndex (oppDirection); + streamedSite.GetStreamedIndex(oppDirection); // Is streamerIndex a valid index? - if (streamerIndex >= 0 && streamerIndex < (lb::lattices::D3Q15::NUMVECTORS - * latDat->GetLocalFluidSiteCount())) + if (streamerIndex >= 0 + && streamerIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { // The streamer index is a valid index in the domain, therefore stream and collide has happened site_t streamerSiteId = streamerIndex / lb::lattices::D3Q15::NUMVECTORS; @@ -727,8 +717,7 @@ namespace hemelb streamerFOld); // Calculate what the value streamed to site streamedToSite should be. - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamedSite); normalCollision->Collide(lbmParams, streamerHydroVars); @@ -749,8 +738,7 @@ namespace hemelb distribn_t streamerToSiteFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(streamedToSite, streamerToSiteFOld); - lb::kernels::HydroVars > - hydroVars(streamerToSiteFOld); + lb::kernels::HydroVars > hydroVars(streamerToSiteFOld); normalCollision->CalculatePreCollision(hydroVars, streamedSite); // Simulate post-collision using the collision operator. @@ -772,20 +760,21 @@ namespace hemelb void TestNashZerothOrderPressureIolet() { + auto async = comm::Async::New(Comms()); lb::iolets::BoundaryValues inletBoundary(geometry::INLET_TYPE, latDat, simConfig->GetInlets(), simState, - Comms(), + async, *unitConverter); initParams.boundaryObject = &inletBoundary; - lb::streamers::NashZerothOrderPressureIolet > >::Type ioletCollider(initParams); + lb::streamers::NashZerothOrderPressureIolet< + lb::collisions::Normal > >::Type ioletCollider(initParams); - for (double assignedWallDistance = 0.4; assignedWallDistance < 1.0; assignedWallDistance - += 0.5) + for (double assignedWallDistance = 0.4; assignedWallDistance < 1.0; + assignedWallDistance += 0.5) { // Initialise fOld in the lattice data. We choose values so that each site has // an anisotropic distribution function, and that each site's function is @@ -810,35 +799,31 @@ namespace hemelb latDat->SetIoletId(chosenSite, chosenBoundaryId); // Perform the collision and streaming. - ioletCollider.StreamAndCollide (chosenSite, - 1, - lbmParams, - latDat, - *propertyCache); + ioletCollider.StreamAndCollide(chosenSite, 1, lbmParams, latDat, *propertyCache); // Check each streamed direction. - for (Direction streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (Direction streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { // Calculate the distributions at the chosen site up to post-collision. distribn_t streamerFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(chosenSite, streamerFOld); - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamer); normalCollision->Collide(lbmParams, streamerHydroVars); // Calculate the streamed-to index. const site_t streamedIndex = - streamer.GetStreamedIndex (streamedDirection); + streamer.GetStreamedIndex(streamedDirection); // Check that simple collide and stream has happened when appropriate. // Is streamerIndex a valid index? (And is it not in one of the directions // that has been meddled with for the test)? - if (!streamer.HasIolet(streamedDirection) && streamedIndex >= 0 && streamedIndex - < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) + if (!streamer.HasIolet(streamedDirection) && streamedIndex >= 0 + && streamedIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { distribn_t streamedToFNew = *latDat->GetFNew(streamedIndex); @@ -886,20 +871,21 @@ namespace hemelb void TestNashZerothOrderPressureBB() { + auto async = comm::Async::New(Comms()); lb::iolets::BoundaryValues inletBoundary(geometry::INLET_TYPE, latDat, simConfig->GetInlets(), simState, - Comms(), + async, *unitConverter); initParams.boundaryObject = &inletBoundary; - lb::streamers::NashZerothOrderPressureIoletSBB > >::Type ioletCollider(initParams); + lb::streamers::NashZerothOrderPressureIoletSBB< + lb::collisions::Normal > >::Type ioletCollider(initParams); - for (double assignedIoletDistance = 0.4; assignedIoletDistance < 1.0; assignedIoletDistance - += 0.5) + for (double assignedIoletDistance = 0.4; assignedIoletDistance < 1.0; + assignedIoletDistance += 0.5) { // Initialise fOld in the lattice data. We choose values so that each site has // an anisotropic distribution function, and that each site's function is @@ -926,36 +912,32 @@ namespace hemelb latDat->SetIoletId(chosenSite, chosenBoundaryId); // Perform the collision and streaming. - ioletCollider.StreamAndCollide (chosenSite, - 1, - lbmParams, - latDat, - *propertyCache); + ioletCollider.StreamAndCollide(chosenSite, 1, lbmParams, latDat, *propertyCache); // Check each streamed direction. - for (Direction streamedDirection = 0; streamedDirection - < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) + for (Direction streamedDirection = 0; + streamedDirection < lb::lattices::D3Q15::NUMVECTORS; ++streamedDirection) { // Calculate the distributions at the chosen site up to post-collision. distribn_t streamerFOld[lb::lattices::D3Q15::NUMVECTORS]; LbTestsHelper::InitialiseAnisotropicTestData(chosenSite, streamerFOld); - lb::kernels::HydroVars > - streamerHydroVars(streamerFOld); + lb::kernels::HydroVars > streamerHydroVars(streamerFOld); normalCollision->CalculatePreCollision(streamerHydroVars, streamer); normalCollision->Collide(lbmParams, streamerHydroVars); // Calculate the streamed-to index. const site_t streamedIndex = - streamer.GetStreamedIndex (streamedDirection); + streamer.GetStreamedIndex(streamedDirection); // Check that simple collide and stream has happened when appropriate. // Is streamerIndex a valid index? (And is it not in one of the directions // that has been meddled with for the test)? if (!streamer.HasIolet(streamedDirection) && !streamer.HasWall(streamedDirection) - && streamedIndex >= 0 && streamedIndex < (lb::lattices::D3Q15::NUMVECTORS - * latDat->GetLocalFluidSiteCount())) + && streamedIndex >= 0 + && streamedIndex + < (lb::lattices::D3Q15::NUMVECTORS * latDat->GetLocalFluidSiteCount())) { distribn_t streamedToFNew = *latDat->GetFNew(streamedIndex); @@ -1019,7 +1001,7 @@ namespace hemelb lb::MacroscopicPropertyCache* propertyCache; lb::collisions::Normal >* normalCollision; }; - CPPUNIT_TEST_SUITE_REGISTRATION ( StreamerTests); + CPPUNIT_TEST_SUITE_REGISTRATION (StreamerTests); } } } diff --git a/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h b/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h index c72e20b27..49d6b8646 100644 --- a/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h +++ b/Code/unittests/lbtests/VirtualSiteIoletStreamerTests.h @@ -32,7 +32,7 @@ namespace hemelb public: //typedef hemelb::lb::streamers::VSExtra::UnitVec UnitVec; LocalVSExtra(lb::iolets::InOutLet& iolet) : - VSExtra (iolet) + VSExtra(iolet) { } const UnitVec& GetE1() const @@ -58,11 +58,11 @@ namespace hemelb */ class VirtualSiteIoletStreamerTests : public helpers::FourCubeBasedTestFixture { - CPPUNIT_TEST_SUITE( VirtualSiteIoletStreamerTests); - CPPUNIT_TEST( TestVirtualSiteMatrixInverse); - CPPUNIT_TEST( TestVirtualSiteConstruction); - CPPUNIT_TEST( TestStreamerInitialisation); - CPPUNIT_TEST( TestStep);CPPUNIT_TEST_SUITE_END(); + CPPUNIT_TEST_SUITE (VirtualSiteIoletStreamerTests); + CPPUNIT_TEST (TestVirtualSiteMatrixInverse); + CPPUNIT_TEST (TestVirtualSiteConstruction); + CPPUNIT_TEST (TestStreamerInitialisation); + CPPUNIT_TEST (TestStep);CPPUNIT_TEST_SUITE_END(); public: typedef lb::lattices::D3Q15 Lattice; @@ -75,7 +75,7 @@ namespace hemelb InOutLetCosine* GetIolet(lb::iolets::BoundaryValues* iolets) { InOutLetCosine* ans = - dynamic_cast (iolets->GetLocalIolet(0)); + dynamic_cast(iolets->GetLocalIolet(0)); CPPUNIT_ASSERT(ans != NULL); return ans; } @@ -91,7 +91,7 @@ namespace hemelb latDat, simConfig->GetInlets(), simState, - Comms(), + Async(), *unitConverter); InOutLetCosine* inlet = GetIolet(inletBoundary); // We have to make the outlet sane and consistent with the geometry now. @@ -110,7 +110,7 @@ namespace hemelb for (unsigned j = 1; j <= 4; ++j) { site_t siteId = latDat->GetContiguousSiteId(LatticeVector(i, j, 1)); - geometry::Site < geometry::LatticeData > site = latDat->GetSite(siteId); + geometry::Site site = latDat->GetSite(siteId); for (Direction p = 0; p < lattice->GetNumVectors(); ++p) { if (lattice->GetVector(p).z < 0.) @@ -128,7 +128,7 @@ namespace hemelb latDat, simConfig->GetOutlets(), simState, - Comms(), + Async(), *unitConverter); InOutLetCosine* outlet = GetIolet(outletBoundary); @@ -146,7 +146,7 @@ namespace hemelb for (unsigned j = 1; j <= 4; ++j) { site_t siteId = latDat->GetContiguousSiteId(LatticeVector(i, j, 4)); - geometry::Site < geometry::LatticeData > site = latDat->GetSite(siteId); + geometry::Site site = latDat->GetSite(siteId); for (Direction p = 0; p < lattice->GetNumVectors(); ++p) { if (lattice->GetVector(p).z > 0.) @@ -243,8 +243,8 @@ namespace hemelb // { 2.5, -2.5, 5.0 } }; // It's inverse is distribn_t velMatInv[3][3] = { { 0.25, 0., -0.125 }, { 0., 0.25, 0.125 }, { -0.125, - 0.125, - 0.325 } }; + 0.125, + 0.325 } }; for (unsigned i = 0; i < 3; ++i) { for (unsigned j = 0; j < 3; ++j) @@ -280,7 +280,7 @@ namespace hemelb // All the sites at the outlet plane (x, y, 3) should be in the cache. InOutLetCosine& outlet = *GetIolet(outletBoundary); - VSExtra* extra = dynamic_cast*> (outlet.GetExtraData()); + VSExtra* extra = dynamic_cast*>(outlet.GetExtraData()); CPPUNIT_ASSERT(extra != NULL); for (unsigned i = 1; i <= 4; ++i) @@ -298,8 +298,8 @@ namespace hemelb } // And the reverse is true: every cache entry should be a site at the outlet plane - for (RSHV::Map::iterator hvPtr = extra->hydroVarsCache.begin(); hvPtr - != extra->hydroVarsCache.end(); ++hvPtr) + for (RSHV::Map::iterator hvPtr = extra->hydroVarsCache.begin(); + hvPtr != extra->hydroVarsCache.end(); ++hvPtr) { site_t globalIdx = hvPtr->first; LatticeVector pos; @@ -327,32 +327,32 @@ namespace hemelb site_t offset = 0; offset += latDat->GetMidDomainCollisionCount(0); offset += latDat->GetMidDomainCollisionCount(1); - inletStreamer.DoStreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(2), - lbmParams, - static_cast (latDat), - *propertyCache); + inletStreamer.DoStreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(2), + lbmParams, + static_cast(latDat), + *propertyCache); offset += latDat->GetMidDomainCollisionCount(2); - outletStreamer.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(3), - lbmParams, - latDat, - *propertyCache); + outletStreamer.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(3), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(3); - inletStreamer.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(4), - lbmParams, - latDat, - *propertyCache); + inletStreamer.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(4), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(4); - outletStreamer.StreamAndCollide (offset, - latDat->GetMidDomainCollisionCount(5), - lbmParams, - latDat, - *propertyCache); + outletStreamer.StreamAndCollide(offset, + latDat->GetMidDomainCollisionCount(5), + lbmParams, + latDat, + *propertyCache); // Now every entry in the RSHV cache should have been updated CheckAllHVUpdated(inletBoundary, 1); @@ -362,39 +362,39 @@ namespace hemelb offset = 0; offset += latDat->GetMidDomainCollisionCount(0); offset += latDat->GetMidDomainCollisionCount(1); - inletStreamer.DoPostStep (offset, - latDat->GetMidDomainCollisionCount(2), - lbmParams, - static_cast (latDat), - *propertyCache); + inletStreamer.DoPostStep(offset, + latDat->GetMidDomainCollisionCount(2), + lbmParams, + static_cast(latDat), + *propertyCache); offset += latDat->GetMidDomainCollisionCount(2); - outletStreamer.DoPostStep (offset, - latDat->GetMidDomainCollisionCount(3), - lbmParams, - latDat, - *propertyCache); + outletStreamer.DoPostStep(offset, + latDat->GetMidDomainCollisionCount(3), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(3); - inletStreamer.DoPostStep (offset, - latDat->GetMidDomainCollisionCount(4), - lbmParams, - latDat, - *propertyCache); + inletStreamer.DoPostStep(offset, + latDat->GetMidDomainCollisionCount(4), + lbmParams, + latDat, + *propertyCache); offset += latDat->GetMidDomainCollisionCount(4); - outletStreamer.DoPostStep (offset, - latDat->GetMidDomainCollisionCount(5), - lbmParams, - latDat, - *propertyCache); + outletStreamer.DoPostStep(offset, + latDat->GetMidDomainCollisionCount(5), + lbmParams, + latDat, + *propertyCache); // Check that all the vsites have sensible hydro values InOutLetCosine* inlet = GetIolet(inletBoundary); - VSExtra * inExtra = dynamic_cast*> (inlet->GetExtraData()); + VSExtra * inExtra = dynamic_cast*>(inlet->GetExtraData()); - for (VirtualSite::Map::iterator vsIt = inExtra->vSites.begin(); vsIt - != inExtra->vSites.end(); ++vsIt) + for (VirtualSite::Map::iterator vsIt = inExtra->vSites.begin(); + vsIt != inExtra->vSites.end(); ++vsIt) { site_t vSiteGlobalIdx = vsIt->first; VirtualSite& vSite = vsIt->second; @@ -411,10 +411,10 @@ namespace hemelb } InOutLetCosine* outlet = GetIolet(outletBoundary); - VSExtra * outExtra = dynamic_cast*> (outlet->GetExtraData()); + VSExtra * outExtra = dynamic_cast*>(outlet->GetExtraData()); - for (VirtualSite::Map::iterator vsIt = outExtra->vSites.begin(); vsIt - != outExtra->vSites.end(); ++vsIt) + for (VirtualSite::Map::iterator vsIt = outExtra->vSites.begin(); + vsIt != outExtra->vSites.end(); ++vsIt) { site_t vSiteGlobalIdx = vsIt->first; VirtualSite& vSite = vsIt->second; @@ -439,9 +439,9 @@ namespace hemelb void CheckAllHVUpdated(lb::iolets::BoundaryValues* iolets, LatticeTimeStep expectedT) { VSExtra * extra = - dynamic_cast*> (iolets->GetLocalIolet(0)->GetExtraData()); - for (RSHV::Map::iterator hvPtr = extra->hydroVarsCache.begin(); hvPtr - != extra->hydroVarsCache.end(); ++hvPtr) + dynamic_cast*>(iolets->GetLocalIolet(0)->GetExtraData()); + for (RSHV::Map::iterator hvPtr = extra->hydroVarsCache.begin(); + hvPtr != extra->hydroVarsCache.end(); ++hvPtr) { site_t siteGlobalIdx = hvPtr->first; LatticeVector sitePos; @@ -505,7 +505,7 @@ namespace hemelb latDat->SwapOldAndNew(); } }; - CPPUNIT_TEST_SUITE_REGISTRATION( VirtualSiteIoletStreamerTests); + CPPUNIT_TEST_SUITE_REGISTRATION (VirtualSiteIoletStreamerTests); } } } diff --git a/Code/unittests/lbtests/iolets/BoundaryTests.h b/Code/unittests/lbtests/iolets/BoundaryTests.h index 85a70684c..e96e59bfb 100644 --- a/Code/unittests/lbtests/iolets/BoundaryTests.h +++ b/Code/unittests/lbtests/iolets/BoundaryTests.h @@ -34,15 +34,15 @@ namespace hemelb CPPUNIT_TEST(TestUpdate); CPPUNIT_TEST(TestUpdateFile); CPPUNIT_TEST_SUITE_END(); - + public: void TestConstruct() { - double targetStartDensity = unitConverter->ConvertPressureToLatticeUnits(80.0 - 1.0) / Cs2; + double targetStartDensity = unitConverter->ConvertPressureToLatticeUnits(80.0 - 1.0) / Cs2; inlets = new BoundaryValues(hemelb::geometry::INLET_TYPE, latDat, simConfig->GetInlets(), simState, - Comms(), + Async(), *unitConverter); CPPUNIT_ASSERT_DOUBLES_EQUAL(targetStartDensity, inlets->GetBoundaryDensity(0), 1e-9); delete inlets; @@ -53,7 +53,7 @@ namespace hemelb latDat, simConfig->GetInlets(), simState, - Comms(), + Async(), *unitConverter); CPPUNIT_ASSERT_DOUBLES_EQUAL(pressureToDensity(80.0 - 1.0), inlets->GetBoundaryDensity(0), 1e-9); @@ -91,7 +91,7 @@ namespace hemelb latDat, fileInletConfig->GetInlets(), simState, - Comms(), + Async(), *unitConverter); CPPUNIT_ASSERT_DOUBLES_EQUAL(pressureToDensity(78.0), inlets->GetBoundaryDensity(0), 1e-6); @@ -120,8 +120,10 @@ namespace hemelb + (pressure - REFERENCE_PRESSURE_mmHg) * mmHg_TO_PASCAL * inverseVelocity * inverseVelocity / (Cs2 * BLOOD_DENSITY_Kg_per_m3); } + + private: BoundaryValues *inlets; - }; + }; //BoundaryTests CPPUNIT_TEST_SUITE_REGISTRATION(BoundaryTests); } // iolets diff --git a/Code/unittests/lbtests/iolets/InOutLetTests.h b/Code/unittests/lbtests/iolets/InOutLetTests.h index 0d4925fc2..0f8115e75 100644 --- a/Code/unittests/lbtests/iolets/InOutLetTests.h +++ b/Code/unittests/lbtests/iolets/InOutLetTests.h @@ -137,12 +137,12 @@ namespace hemelb // Ok, now we have an inlet, check the values are right. CPPUNIT_ASSERT_EQUAL(std::string("./iolet.txt"), file->GetFilePath()); - CPPUNIT_ASSERT_DOUBLES_EQUAL(78.0, - converter.ConvertPressureToPhysicalUnits(file->GetPressureMin()), - 1e-6); - CPPUNIT_ASSERT_DOUBLES_EQUAL(82.0, - converter.ConvertPressureToPhysicalUnits(file->GetPressureMax()), - 1e-6); +// CPPUNIT_ASSERT_DOUBLES_EQUAL(78.0, +// converter.ConvertPressureToPhysicalUnits(file->GetPressureMin()), +// 1e-6); +// CPPUNIT_ASSERT_DOUBLES_EQUAL(82.0, +// converter.ConvertPressureToPhysicalUnits(file->GetPressureMax()), +// 1e-6); PhysicalPosition expected(-1.66017717834e-05, -4.58437586355e-05, -0.05); PhysicalPosition actual = converter.ConvertPositionToPhysicalUnits(file->GetPosition()); @@ -158,7 +158,7 @@ namespace hemelb + (82.0 - REFERENCE_PRESSURE_mmHg) * mmHg_TO_PASCAL * temp * temp / (Cs2 * BLOOD_DENSITY_Kg_per_m3); - CPPUNIT_ASSERT_DOUBLES_EQUAL(targetStartDensity, file->GetDensityMin(), 1e-6); +// CPPUNIT_ASSERT_DOUBLES_EQUAL(targetStartDensity, file->GetDensityMin(), 1e-6); CPPUNIT_ASSERT_DOUBLES_EQUAL(targetStartDensity, file->GetDensity(0), 1e-6); CPPUNIT_ASSERT_DOUBLES_EQUAL(targetMidDensity, file->GetDensity(state.GetTotalTimeSteps() / 2), @@ -286,7 +286,7 @@ namespace hemelb UncheckedSimConfig config(Resource("config_file_velocity_inlet.xml").Path()); lb::SimulationState state = lb::SimulationState(config.GetTimeStepLength(), config.GetTotalTimeSteps()); - double voxelSize = config.GetVoxelSize(); + const util::UnitConverter& converter = config.GetUnitConverter(); fileVel = static_cast(config.GetInlets()[0]); // at this stage, Initialise() has not been called, so the unit converter will be invalid, so we will not be able to convert to physical units. @@ -341,14 +341,6 @@ namespace hemelb ConcreteIolet* copy = new ConcreteIolet(*this); return copy; } - virtual LatticeDensity GetDensityMin() const - { - return 1.0; - } - virtual LatticeDensity GetDensityMax() const - { - return 1.0; - } virtual LatticeDensity GetDensity(hemelb::LatticeTimeStep) const { return 1.0; diff --git a/Code/unittests/main.cc b/Code/unittests/main.cc index 9be5a8502..666bab43e 100644 --- a/Code/unittests/main.cc +++ b/Code/unittests/main.cc @@ -13,14 +13,13 @@ #include #include "unittests/helpers/helpers.h" #include "unittests/lbtests/lbtests.h" -#include "unittests/vistests/vistests.h" #include "unittests/io/io.h" #include "unittests/reporting/reporting.h" #include "unittests/configuration/configuration.h" #include "unittests/geometry/geometry.h" #include "unittests/SimulationMasterTests.h" #include "unittests/extraction/extraction.h" -#include "unittests/net/net.h" + #include "unittests/multiscale/multiscale.h" #ifdef HEMELB_BUILD_MULTISCALE #include "unittests/multiscale/mpwide/mpwide.h" @@ -33,10 +32,10 @@ int main(int argc, char **argv) { // Start MPI and the logger. - hemelb::net::MpiEnvironment mpi(argc, argv); + hemelb::comm::MpiEnvironment mpi(argc, argv); hemelb::log::Logger::Init(); - hemelb::net::MpiCommunicator commWorld = hemelb::net::MpiCommunicator::World(); + hemelb::comm::Communicator::Ptr commWorld = mpi.World(); // Read options std::ostream * reportto = &std::cerr; @@ -60,7 +59,7 @@ int main(int argc, char **argv) hemelb::debug::Debugger::Init(debug, argv[0], commWorld); // Initialise the global IOCommunicator. - hemelb::net::IOCommunicator testCommunicator(commWorld); + hemelb::comm::Communicator::Ptr testCommunicator = commWorld; hemelb::unittests::helpers::HasCommsTestFixture::Init(testCommunicator); std::string testPath = (optind < argc) diff --git a/Code/unittests/multiscale/MockIntercommunicator.h b/Code/unittests/multiscale/MockIntercommunicator.h index 9945e93c5..6f4b12411 100644 --- a/Code/unittests/multiscale/MockIntercommunicator.h +++ b/Code/unittests/multiscale/MockIntercommunicator.h @@ -13,7 +13,6 @@ #include #include "multiscale/Intercommunicator.h" -#include "net/mpi.h" namespace hemelb { @@ -30,7 +29,7 @@ namespace hemelb typedef MPI_Datatype RuntimeType; template static RuntimeType GetType() { - return net::MpiDataTypeTraits::GetMpiDataType(); + return comm::MpiDataTypeTraits::GetMpiDataType(); } }; diff --git a/Code/unittests/multiscale/MockIntercommunicatorTests.h b/Code/unittests/multiscale/MockIntercommunicatorTests.h index 5e53e8f34..7ecbf4a8e 100644 --- a/Code/unittests/multiscale/MockIntercommunicatorTests.h +++ b/Code/unittests/multiscale/MockIntercommunicatorTests.h @@ -207,15 +207,11 @@ namespace hemelb // TODO: This test is fatal if run with LADDIOLET. See ticket #605. LADD_FAIL(); int argc; - const char* argv[7]; - argc = 7; + const char* argv[5]; + argc = 3; argv[0] = "hemelb"; - argv[2] = "four_cube_multiscale.xml"; argv[1] = "-in"; - argv[3] = "-i"; - argv[4] = "1"; - argv[5] = "-ss"; - argv[6] = "1111"; + argv[2] = "four_cube_multiscale.xml"; CopyResourceToTempdir("four_cube_multiscale.xml"); CopyResourceToTempdir("four_cube.gmy"); hemelb::configuration::CommandLine options(argc, argv); diff --git a/Code/unittests/net/LabelledRequest.h b/Code/unittests/net/LabelledRequest.h deleted file mode 100644 index c33044092..000000000 --- a/Code/unittests/net/LabelledRequest.h +++ /dev/null @@ -1,115 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_NET_LABELLEDREQUEST_H -#define HEMELB_UNITTESTS_NET_LABELLEDREQUEST_H -#include "net/StoredRequest.h" -namespace hemelb -{ - namespace unittests - { - namespace net - { - class LabelledRequest : public hemelb::net::SimpleRequest - { - public: - const std::string Label; - LabelledRequest(void *pointer, int count, MPI_Datatype type, proc_t rank, const std::string &label) : - SimpleRequest(pointer, count, type, rank), Label(label) - { - } - - virtual bool EnvelopeIdentical(const SimpleRequest & other) - { - bool this_ok = ( (Count == other.Count) && (Rank == other.Rank) && (Type == other.Type)); - if (!this_ok) - { - std::cerr << "Envelope different: " << Label << " R: " << Rank << " C: " << Count << " T: " << " : " - << " R" << other.Rank << " C " << other.Count << " T " << std::endl; - // uncomment the below for type diagnostics if your compiler supports them - /*std::cerr << "Types are:" << static_cast(Type) << " : " << static_cast(other.Type) << std::endl; - std::cerr << "Type candidates include" << " int: " << static_cast(MpiDataType()) - << " unsigned int :" << static_cast(MpiDataType()) - << " site_t: " << static_cast(MpiDataType()) - << " proc_t: " << static_cast(MpiDataType()) - << std::endl;*/ - } - /*else{ - std::cerr << "Envelope same: " << Label << " R: " << Rank << " C: " << Count << " T: " << " : " - << " R" << other.Rank << " C " << other.Count << " T " << std::endl; - }*/ - return this_ok; - } - - virtual bool PayloadIdentical(const SimpleRequest & other) - { - // reduction - bool ok = true; - for (int element = 0; element < Count; element++) - { - int size; - - MPI_Type_size(other.Type, &size); - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - bool this_ok = 0 - == std::memcmp(static_cast(other.Pointer) + size * element, - static_cast(Pointer) + size * element, - size); - if (!this_ok) - { - - std::cerr << "Unexpected data in request: " << Label << " R " << Rank << " C " << Count << " : " - << std::endl; - for (int i = 0; i < size; i++) - { - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - std::cerr << i << " : " - << static_cast(* (static_cast(Pointer) + size * element + i)) << " " - << static_cast(* (static_cast(other.Pointer) + size * element + i)) << std::endl; - } - std::cerr << std::endl; - } - /*// --- use this block for debugging - else - { - std::cerr << "Expected data in request: " << Label << " R " << Rank << " C " << Count << " : " - << std::endl; - for (int i = 0; i < size; i++) - { - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - std::cerr << i << " : " - << static_cast(* (static_cast(Pointer) + size * element + i)) << " " - << static_cast(* (static_cast(other.Pointer) + size * element + i)) << std::endl; - } - }*/ - ok = ok && this_ok; - } - return ok; - } - - virtual void Unpack(SimpleRequest & other) - { - for (int element = 0; element < Count; element++) - { - int size; - MPI_Type_size(other.Type, &size); - - // The below use of unsigned char is not formally correct (due to the possibility of char not having alignment 1) - // But we cannot currently see a better solution to avoid compiler warnings from void* arithmetic. - std::memcpy(static_cast(other.Pointer) + size * element, - static_cast(Pointer) + size * element, - size); - } - } - }; - } - } -} - -#endif diff --git a/Code/unittests/net/MpiTests.h b/Code/unittests/net/MpiTests.h index e2dbc0321..49192ceac 100644 --- a/Code/unittests/net/MpiTests.h +++ b/Code/unittests/net/MpiTests.h @@ -8,15 +8,16 @@ #define HEMELB_UNITTESTS_NET_MPITESTS_H #include -#include "net/mpi.h" +#include "comm/MpiCommunicator.h" +#include "comm/MpiEnvironment.h" namespace hemelb { namespace unittests { - namespace net + namespace comm { - using namespace hemelb::net; + using namespace hemelb::comm; /** * Unittests of the MPI abstraction layer * @@ -32,31 +33,12 @@ namespace hemelb void TestMpiComm() { MpiCommunicator commNull; - CPPUNIT_ASSERT(!commNull); + //CPPUNIT_ASSERT(!commNull); - MpiCommunicator commWorld = MpiCommunicator::World(); + Communicator::Ptr commWorld = MpiEnvironment::World(); CPPUNIT_ASSERT(commWorld); - CPPUNIT_ASSERT(commNull== commNull); - CPPUNIT_ASSERT(commWorld == commWorld); - CPPUNIT_ASSERT(commWorld != commNull); - - { - MpiCommunicator commWorld2 = commWorld; - CPPUNIT_ASSERT(commWorld2 == commWorld); - } - - { - MpiCommunicator commWorld2 = MpiCommunicator::World(); - CPPUNIT_ASSERT(commWorld2 == commWorld); - } - { - MpiGroup groupWorld = commWorld.Group(); - MpiCommunicator commWorld2 = commWorld.Create(groupWorld); - // Same ranks, but different context. - CPPUNIT_ASSERT(commWorld2 != commWorld); - } - } + } }; CPPUNIT_TEST_SUITE_REGISTRATION (MpiTests); } diff --git a/Code/unittests/net/NetMock.h b/Code/unittests/net/NetMock.h deleted file mode 100644 index b9ab9e40f..000000000 --- a/Code/unittests/net/NetMock.h +++ /dev/null @@ -1,44 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_NET_NETMOCK_H -#define HEMELB_UNITTESTS_NET_NETMOCK_H -#include -#include -#include -#include - -#include "constants.h" -#include "net/mpi.h" -#include "net/net.h" -#include "unittests/net/RecordingNet.h" - -namespace hemelb -{ - namespace unittests - { - namespace net - { - using namespace hemelb::net; - - class NetMock : public InterfaceDelegationNet, - public RecordingNet, - public ViaPointPointGathers, - public ViaPointPointAllToAll - { - public: - NetMock(net::MpiCommunicator & communicator) : - BaseNet(communicator), StoringNet(communicator), InterfaceDelegationNet(communicator), - RecordingNet(communicator), ViaPointPointGathers(communicator), - ViaPointPointAllToAll(communicator) - { - } - - }; - } - } -} -#endif // ONCE diff --git a/Code/unittests/net/RecordingNet.h b/Code/unittests/net/RecordingNet.h deleted file mode 100644 index 1c24f2bac..000000000 --- a/Code/unittests/net/RecordingNet.h +++ /dev/null @@ -1,186 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_NET_RECORDINGNET_H -#define HEMELB_UNITTESTS_NET_RECORDINGNET_H -#include -#include -#include -#include - -#include "constants.h" -#include "net/mpi.h" -#include "net/net.h" -#include "unittests/net/LabelledRequest.h" - -namespace hemelb -{ - namespace unittests - { - namespace net - { - using namespace hemelb::net; - /** - * A mock for the net class for testing. - * You must first give a complete, per-rank-ordered list of the sends and - * another of the receives that it is to carry out. - * - * For sends, it checks that the sent data is identical to that specified/ - * - * For recvs, it delivers the data that you specified. - * - * For both, it checks metadata. - * - * Once you have completed a "round" of communications, call - * ExpectationsAllCompleted to check that there are none outstanding. - * - */ - class RecordingNet : public virtual StoringNet - { - public: - RecordingNet(const MpiCommunicator& comms) : - BaseNet(comms), StoringNet(comms), requiredReceipts(), requiredSends() - { - } - - /** - * Specify that this rank should receive a message - * @param pointer - to the message data that will later be received - * @param count - of number of elements in the message - * @param rank - of the source of the message - * @param label - used in error reporting (should be unique) - */ - template void RequireReceive(T* pointer, unsigned int count, proc_t rank, - const std::string &label = "") - { - requiredReceipts[rank].push_back(LabelledRequest(pointer, - count, - MpiDataType(), - rank, - label)); - } - /** - * Specify that this rank should send a message - * @param pointer - to the message data that should be sent - * @param count - of the number of elements in the message - * @param rank - of the destination of the message - * @param label - used in error reporting (should be unique) - */ - template void RequireSend(T* pointer, unsigned int count, proc_t rank, - const std::string &label = "") - { - requiredSends[rank].push_back(LabelledRequest(pointer, - count, - MpiDataType(), - rank, - label)); - } - - /** - * Mock-execute queued receives. - * - * Does no actual communication, but copies in the mock data - * supplied to RequireReceive. It also checks that the receives - * match the required ones. - */ - void ReceivePointToPoint() - { - for (std::map::iterator it = receiveProcessorComms.begin(); - it != receiveProcessorComms.end(); ++it) - { - for (ProcComms::iterator message = it->second.begin(); message != it->second.end(); - message++) - { - - if (requiredReceipts[message->Rank].size() == 0) - { - // It should not be the case, that a request is made, but there is no recorded assertion in the list of recorded assertions. - // i.e., we've popped off all the recorded requests, and there are none left to match the request we just received. - CPPUNIT_ASSERT(requiredReceipts[message->Rank].size() != 0); - return; - } - - CPPUNIT_ASSERT(requiredReceipts[message->Rank].front().EnvelopeIdentical(*message)); - requiredReceipts[message->Rank].front().Unpack(*message); - - // This assertion just checks that "Unpack" has done it's job. - // It shouldn't fail due to problems in tested code - // Would only fail due to memory screwups or bad code in this class's Unpack method. - CPPUNIT_ASSERT(requiredReceipts[message->Rank].front().PayloadIdentical(*message)); - - requiredReceipts[message->Rank].pop_front(); - } - - } - - } - /** - * Mock-execute queued sends. - * - * Does no actual communication, but checks that the sent data - * matches the mock data supplied to RequireSend. It also checks that - * the sends match the required ones. - */ - void SendPointToPoint() - { - - for (std::map::iterator it = sendProcessorComms.begin(); - it != sendProcessorComms.end(); ++it) - { - for (ProcComms::iterator message = it->second.begin(); message != it->second.end(); - message++) - { - - if (requiredSends[message->Rank].size() == 0) - { - CPPUNIT_ASSERT(requiredSends[message->Rank].size() != 0); - return; - } - CPPUNIT_ASSERT(requiredSends[message->Rank].front().EnvelopeIdentical(*message)); - CPPUNIT_ASSERT(requiredSends[message->Rank].front().PayloadIdentical(*message)); - requiredSends[message->Rank].pop_front(); - } - - } - } - - /** - * Mock-wait - clears the message queue - */ - void WaitPointToPoint() - { - receiveProcessorComms.clear(); - sendProcessorComms.clear(); - } - - /** - * Assert that all required sends and receives have occurred. - */ - void ExpectationsAllCompleted() - { - for (std::map >::iterator receipts_from_core = - requiredReceipts.begin(); receipts_from_core != requiredReceipts.end(); - receipts_from_core++) - { - CPPUNIT_ASSERT(0 == receipts_from_core->second.size()); - } - for (std::map >::iterator sends_to_core = - requiredSends.begin(); sends_to_core != requiredSends.end(); sends_to_core++) - { - CPPUNIT_ASSERT(0 == sends_to_core->second.size()); - } - } - private: - - std::map > requiredReceipts; - std::map > requiredSends; - } - ; - - } - } -} -#endif diff --git a/Code/unittests/net/phased/StepManagerTests.h b/Code/unittests/net/phased/StepManagerTests.h index 754852dbb..53823b249 100644 --- a/Code/unittests/net/phased/StepManagerTests.h +++ b/Code/unittests/net/phased/StepManagerTests.h @@ -7,10 +7,10 @@ #ifndef HEMELB_UNITTESTS_NET_PHASED_STEPMANAGERTESTS_H #define HEMELB_UNITTESTS_NET_PHASED_STEPMANAGERTESTS_H #include "net/phased/StepManager.h" -#include "unittests/helpers/MockNetHelper.h" +#include "unittests/helpers/MockCommsHelper.h" #include "unittests/net/phased/MockConcern.h" #include "unittests/net/phased/MockIteratedAction.h" -#include "net/phased/NetConcern.h" +#include "comm/AsyncConcern.h" #include namespace hemelb @@ -23,7 +23,7 @@ namespace hemelb { using namespace hemelb::net::phased; using namespace hemelb::unittests::helpers; - class StepManagerTests : public CppUnit::TestFixture, public MockNetHelper + class StepManagerTests : public CppUnit::TestFixture, public MockCommsHelper { CPPUNIT_TEST_SUITE (StepManagerTests); CPPUNIT_TEST (TestConstruct); @@ -50,7 +50,7 @@ namespace hemelb public: StepManagerTests() : - MockNetHelper(),stepManager(NULL), action(NULL), concern(NULL), netConcern(NULL), action2(NULL), concern2(NULL) + MockCommsHelper(),stepManager(NULL), action(NULL), concern(NULL), netConcern(NULL), action2(NULL), concern2(NULL) { } @@ -62,7 +62,7 @@ namespace hemelb void tearDown() { - MockNetHelper::tearDown(); + MockCommsHelper::tearDown(); delete stepManager; delete netConcern; delete action; @@ -139,17 +139,17 @@ namespace hemelb int payload0Expectation = 5; int payload1Expectation = 1; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); - + MockComms()->RequireSend(payload1Expectation, 1, 11); + MockComms()->RequireRecv(payload0Expectation, 1, 11); + stepManager->RegisterCommsSteps(*netConcern, 0); + commQ->Isend(payload1, 1, 11); + commQ->Irecv(payload0, 1, 11); stepManager->CallActionsForPhase(0); CPPUNIT_ASSERT_EQUAL(payload0, 5); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); } void TestCallNonCommsActions() @@ -203,11 +203,12 @@ namespace hemelb int payload0Expectation = 5; int payload1Expectation = 1; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); - + MockComms()->RequireSend(payload1Expectation, 1, 1); + MockComms()->RequireRecv(payload0Expectation, 1, 1); + + commQ->Isend(payload1, 1, 1); + commQ->Irecv(payload0, 1, 1); + action = new MockIteratedAction("mockOne"); concern = new MockConcern("mockTwo"); @@ -228,7 +229,7 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, "), action->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 5); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); } void TestCallAllActionsOnePhase() @@ -240,11 +241,12 @@ namespace hemelb int payload0Expectation = 5; int payload1Expectation = 1; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); + MockComms()->RequireSend(payload1Expectation, 1, 0); + MockComms()->RequireRecv(payload0Expectation, 1, 0); + commQ->Isend(payload1, 1); + commQ->Irecv(payload0, 1); + action = new MockIteratedAction("mockOne"); concern = new MockConcern("mockTwo"); @@ -267,7 +269,7 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, EndIteration, "), action->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 5); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); } void TestCallAllActionsManyPhases() @@ -279,10 +281,11 @@ namespace hemelb int payload0Expectation = 5; int payload1Expectation = 1; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); + MockComms()->RequireSend(payload1Expectation, 1, 0); + MockComms()->RequireRecv(payload0Expectation, 1, 0); + + commQ->Isend(payload1, 1); + commQ->Irecv(payload0, 1); action = new MockIteratedAction("mockOne"); concern = new MockConcern("mockTwo"); @@ -325,7 +328,7 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, EndIteration, "), action2->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 5); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); } void TestCallAllActionsPhaseByPhase() @@ -368,16 +371,17 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string(""), action->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(std::string(""), action2->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 0); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); //------------------Phase 0 --------------------------------------------- int payload0Expectation = 5; int payload1Expectation = 1; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); + MockComms()->RequireSend(payload1Expectation, 1, 0); + MockComms()->RequireRecv(payload0Expectation, 1, 0); + + commQ->Isend(payload1, 1); + commQ->Irecv(payload0, 1); stepManager->CallActionsForPhase(0); @@ -392,17 +396,17 @@ namespace hemelb action->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(std::string(""), action2->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 5); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); // ------------------- Phase 1 ------------------------------------------ payload0Expectation = 13; payload1Expectation = 4; payload1 = 4; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); + MockComms()->RequireSend(payload1Expectation, 1, 0); + MockComms()->RequireRecv(payload0Expectation, 1, 0); + commQ->Isend(payload1, 1); + commQ->Irecv(payload0, 1); stepManager->CallActionsForPhase(1); @@ -413,17 +417,17 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, "), action2->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 13); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); // ------------------------- Phase 2 ----------------------------------------- payload0Expectation = 77; payload1Expectation = 16; payload1 = 16; - netMock->RequestSendR(payload1, 1); - netMock->RequestReceiveR(payload0, 1); - netMock->RequireSend(&payload1Expectation, 1, 1); - netMock->RequireReceive(&payload0Expectation, 1, 1); + MockComms()->RequireSend(payload1Expectation, 1, 0); + MockComms()->RequireRecv(payload0Expectation, 1, 0); + commQ->Isend(payload1, 1); + commQ->Irecv(payload0, 1); stepManager->CallActionsForPhase(2); @@ -436,7 +440,7 @@ namespace hemelb CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, "), action2->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(payload0, 77); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); // -------------------------- EndAll ----------------------------------------------- @@ -451,13 +455,13 @@ namespace hemelb action->CallsSoFar()); CPPUNIT_ASSERT_EQUAL(std::string("RequestComms, PreSend, PreReceive, PostReceive, EndIteration, "), action2->CallsSoFar()); - netMock->ExpectationsAllCompleted(); + MockComms()->ExpectationsAllCompleted(); } void SetupMocks(const proc_t core_count, const proc_t current_core) { - MockNetHelper::setUp(core_count,current_core); - netConcern = new NetConcern(*netMock); + MockCommsHelper::setUp(core_count,current_core); + netConcern = new AsyncConcern(commQ); } private: @@ -465,7 +469,7 @@ namespace hemelb StepManager *stepManager; MockIteratedAction *action; MockConcern *concern; - NetConcern *netConcern; + comm::AsyncConcern *netConcern; MockIteratedAction *action2; MockConcern *concern2; }; diff --git a/Code/unittests/reporting/Mocks.h b/Code/unittests/reporting/Mocks.h index 73cfd89e3..cf13365b0 100644 --- a/Code/unittests/reporting/Mocks.h +++ b/Code/unittests/reporting/Mocks.h @@ -30,7 +30,7 @@ namespace hemelb double fakeTime; }; - class MPICommsMock + /* class MPICommsMock { public: MPICommsMock(const net::IOCommunicator& ignored) : @@ -61,7 +61,7 @@ namespace hemelb } private: unsigned int calls; - }; + };*/ } } } diff --git a/Code/unittests/reporting/ReporterTests.h b/Code/unittests/reporting/ReporterTests.h index 6aad8c050..ef9392462 100644 --- a/Code/unittests/reporting/ReporterTests.h +++ b/Code/unittests/reporting/ReporterTests.h @@ -24,8 +24,8 @@ namespace hemelb { using namespace hemelb::reporting; - typedef TimersBase TimersMock; - typedef lb::IncompressibilityChecker IncompressibilityCheckerMock; + typedef TimersBase TimersMock; + typedef lb::IncompressibilityChecker IncompressibilityCheckerMock; class ReporterTests : public helpers::HasCommsTestFixture { @@ -40,14 +40,13 @@ namespace hemelb realTimers = new reporting::Timers(Comms()); buildInfo = new reporting::BuildInfo(); state = new hemelb::lb::SimulationState(0.0001, 1000); - net = new net::Net(Comms()); latticeData = FourCubeLatticeData::Create(Comms(), 6, 5); // The 5 here is to match the topology size in the MPICommsMock lbtests::LbTestsHelper::InitialiseAnisotropicTestData(latticeData); latticeData->SwapOldAndNew(); //Needed since InitialiseAnisotropicTestData only initialises FOld cache = new lb::MacroscopicPropertyCache(*state, *latticeData); cache->densityCache.SetRefreshFlag(); lbtests::LbTestsHelper::UpdatePropertyCache(*latticeData, *cache, *state); - incompChecker = new IncompressibilityCheckerMock(latticeData, net, state, *cache, *realTimers, 10.0); + incompChecker = new IncompressibilityCheckerMock(latticeData, Comms(), state, *cache, *realTimers, 10.0); reporter = new Reporter("mock_path", "exampleinputfile"); reporter->AddReportable(incompChecker); reporter->AddReportable(mockTimers); @@ -63,7 +62,6 @@ namespace hemelb delete realTimers; delete cache; delete incompChecker; - delete net; delete buildInfo; helpers::HasCommsTestFixture::tearDown(); } @@ -85,9 +83,6 @@ namespace hemelb } } mockTimers->Reduce(); // invoke the Timers MPI mock - reporter->Image(); - reporter->Image(); - reporter->Image(); for (unsigned int step = 0; step < 1000; step++) { state->Increment(); @@ -101,7 +96,6 @@ namespace hemelb "{{#PROCESSOR}}R{{RANK}}S{{SITES}} {{/PROCESSOR}}"); AssertTemplate(hemelb::reporting::mercurial_revision_number, "{{#BUILD}}{{REVISION}}{{/BUILD}}"); AssertTemplate(hemelb::reporting::build_time, "{{#BUILD}}{{TIME}}{{/BUILD}}"); - AssertValue("3", "IMAGES"); AssertValue("0.000100", "TIME_STEP_LENGTH"); AssertValue("1000", "TOTAL_TIME_STEPS"); AssertValue("1000", "STEPS"); @@ -135,8 +129,8 @@ namespace hemelb expectation << std::setprecision(3); for (unsigned int row = 0; row < Timers::numberOfTimers; row++) { - expectation << "N" << TimersMock::timerNames[row] << "L" << row * 10.0 << "MI" << row * 15.0 << "ME" - << row * 2.0 << "MA" << row * 5.0 << " " << std::flush; + expectation << "N" << TimersMock::timerNames[row] << "L" << row * 10.0 << "MI" << row * 10.0 << "ME" + << row * 10.0 << "MA" << row * 10.0 << " " << std::flush; } AssertTemplate(expectation.str(), "{{#TIMER}}N{{NAME}}L{{LOCAL}}MI{{MIN}}ME{{MEAN}}MA{{MAX}} {{/TIMER}}"); } @@ -152,7 +146,6 @@ namespace hemelb lb::MacroscopicPropertyCache* cache; IncompressibilityCheckerMock *incompChecker; - net::Net *net; hemelb::unittests::FourCubeLatticeData* latticeData; reporting::BuildInfo *buildInfo; }; diff --git a/Code/unittests/reporting/TimerTests.h b/Code/unittests/reporting/TimerTests.h index 82142f807..8356a4a7d 100644 --- a/Code/unittests/reporting/TimerTests.h +++ b/Code/unittests/reporting/TimerTests.h @@ -83,7 +83,7 @@ namespace hemelb public: void setUp() { - timers = new TimersBase(Comms()); + timers = new TimersBase(Comms()); } void tearDown() @@ -130,14 +130,14 @@ namespace hemelb timers->Reduce(); // trigger the mock for (unsigned int i = 0; i < Timers::numberOfTimers; i++) { - CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 5.0, timers->Maxes()[i], 1e-6); - CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 2.0, timers->Means()[i], 1e-6); - CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 15.0, timers->Mins()[i], 1e-6); + CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 10.0, timers->Maxes()[i], 1e-6); + CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 10.0, timers->Means()[i], 1e-6); + CPPUNIT_ASSERT_DOUBLES_EQUAL(i * 10.0, timers->Mins()[i], 1e-6); } } private: - TimersBase *timers; + TimersBase *timers; }; CPPUNIT_TEST_SUITE_REGISTRATION(TimersTests); diff --git a/Code/unittests/resources/config-velocity-iolet.xml b/Code/unittests/resources/config-velocity-iolet.xml index d47d880e7..bddcd22dd 100644 --- a/Code/unittests/resources/config-velocity-iolet.xml +++ b/Code/unittests/resources/config-velocity-iolet.xml @@ -35,16 +35,4 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/config.xml b/Code/unittests/resources/config.xml index 8b03a2cbc..6aeade27a 100644 --- a/Code/unittests/resources/config.xml +++ b/Code/unittests/resources/config.xml @@ -39,18 +39,6 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/config0_2_0.xml b/Code/unittests/resources/config0_2_0.xml index c89a0f1d3..40424f9d8 100644 --- a/Code/unittests/resources/config0_2_0.xml +++ b/Code/unittests/resources/config0_2_0.xml @@ -39,16 +39,4 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/config_file_inlet.xml b/Code/unittests/resources/config_file_inlet.xml index 99c220c94..3158b1e35 100644 --- a/Code/unittests/resources/config_file_inlet.xml +++ b/Code/unittests/resources/config_file_inlet.xml @@ -36,16 +36,4 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/config_file_velocity_inlet.xml b/Code/unittests/resources/config_file_velocity_inlet.xml index 6434ecfa6..2215ed0cc 100644 --- a/Code/unittests/resources/config_file_velocity_inlet.xml +++ b/Code/unittests/resources/config_file_velocity_inlet.xml @@ -37,16 +37,4 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/config_new_velocity_inlets.xml b/Code/unittests/resources/config_new_velocity_inlets.xml index ea8833e9a..8c0f3efbc 100644 --- a/Code/unittests/resources/config_new_velocity_inlets.xml +++ b/Code/unittests/resources/config_new_velocity_inlets.xml @@ -39,16 +39,4 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/four_cube.xml b/Code/unittests/resources/four_cube.xml index 0b7980253..4c9f7352d 100644 --- a/Code/unittests/resources/four_cube.xml +++ b/Code/unittests/resources/four_cube.xml @@ -39,18 +39,6 @@ - - - - - - - - - - - - diff --git a/Code/unittests/resources/four_cube_multiscale.xml b/Code/unittests/resources/four_cube_multiscale.xml index ace4c1c28..25536518e 100644 --- a/Code/unittests/resources/four_cube_multiscale.xml +++ b/Code/unittests/resources/four_cube_multiscale.xml @@ -37,18 +37,6 @@ - - - - - - - - - - - - diff --git a/Code/unittests/vistests/HslToRgbConvertorTests.h b/Code/unittests/vistests/HslToRgbConvertorTests.h deleted file mode 100644 index 4016ab492..000000000 --- a/Code/unittests/vistests/HslToRgbConvertorTests.h +++ /dev/null @@ -1,60 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_VISTESTS_HSLTORGBCONVERTORTESTS_H -#define HEMELB_UNITTESTS_VISTESTS_HSLTORGBCONVERTORTESTS_H - -#include - -namespace hemelb -{ - namespace unittests - { - namespace vistests - { - class HslToRgbConvertorTests : public CppUnit::TestFixture - { - CPPUNIT_TEST_SUITE( HslToRgbConvertorTests ); - CPPUNIT_TEST( TestColours ); - CPPUNIT_TEST_SUITE_END(); - public: - void TestColours() - { - DoColourTest(0, 0, 0, 0, 0, 0); - DoColourTest(1, 1, 1, 255, 255, 255); - DoColourTest(250, 0.5, 0.5, 84, 63, 191); - } - - private: - void DoColourTest(float iHue, - float iSaturation, - float iLightness, - unsigned char iExpectedRed, - unsigned char iExpectedGreen, - unsigned char iExpectedBlue) - { - unsigned char lRGB[3]; - - hemelb::vis::raytracer::HSLToRGBConverter::Convert(iHue, iSaturation, iLightness, lRGB); - - CPPUNIT_ASSERT_EQUAL_MESSAGE("HslRgbConvertor test differed on red channel", - iExpectedRed, - lRGB[0]); - CPPUNIT_ASSERT_EQUAL_MESSAGE("HslRgbConvertor test differed on green channel", - iExpectedGreen, - lRGB[1]); - CPPUNIT_ASSERT_EQUAL_MESSAGE("HslRgbConvertor test differed on blue channel", - iExpectedBlue, - lRGB[2]); - } - - }; - CPPUNIT_TEST_SUITE_REGISTRATION( HslToRgbConvertorTests ); - } - } -} - -#endif diff --git a/Code/unittests/vistests/vistests.h b/Code/unittests/vistests/vistests.h deleted file mode 100644 index 110ec63c4..000000000 --- a/Code/unittests/vistests/vistests.h +++ /dev/null @@ -1,12 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UNITTESTS_VISTESTS_VISTESTS_H -#define HEMELB_UNITTESTS_VISTESTS_VISTESTS_H - -#include "unittests/vistests/HslToRgbConvertorTests.h" - -#endif /* HEMELB_UNITTESTS_VISTESTS_VISTESTS_H */ diff --git a/Code/util/Vector3D.h b/Code/util/Vector3D.h index 93f455d30..ed2cbcbdb 100644 --- a/Code/util/Vector3D.h +++ b/Code/util/Vector3D.h @@ -12,7 +12,6 @@ #include #include #include -#include "util/static_assert.h" #include "util/utilityFunctions.h" #include "util/Vector3DArithmeticTraits.h" @@ -343,8 +342,10 @@ namespace hemelb */ T GetMagnitude() const { - HEMELB_STATIC_ASSERT(std::numeric_limits::is_specialized); - HEMELB_STATIC_ASSERT(!std::numeric_limits::is_integer); + static_assert(std::numeric_limits::is_specialized, + "type must have std::numeric_limits specialized"); + static_assert(!std::numeric_limits::is_integer, + "type must not be an integer"); return std::sqrt(GetMagnitudeSquared()); } @@ -720,11 +721,11 @@ namespace hemelb } #ifdef HEMELB_CODE -#include "net/MpiDataType.h" +#include "comm/MpiDataType.h" namespace hemelb { - namespace net + namespace comm { template<> MPI_Datatype MpiDataTypeTraits >::RegisterMpiDataType(); diff --git a/Code/util/Vector3DArithmeticTraits.h b/Code/util/Vector3DArithmeticTraits.h index 8e3aa497b..a0443b065 100644 --- a/Code/util/Vector3DArithmeticTraits.h +++ b/Code/util/Vector3DArithmeticTraits.h @@ -19,25 +19,21 @@ namespace hemelb template struct Vector3DArithmeticTraits { - /* - * If this assertion trips, it means that Vector3D::operator* or Vector3D::operator/ - * have not been tested with this combination of types (OperatorArgument1Type, OperatorArgument2Type) - * - * One would like to write HEMELB_STATIC_ASSERT(false), however if the - * static assertion is not dependent upon one or more template parameters, then - * the compiler is permitted to evaluate the static assertion at the point it is - * first seen, irrespective of whether the template is ever instantiated. - * - * See http://www.boost.org/doc/libs/1_49_0/doc/html/boost_staticassert.html - */ - HEMELB_STATIC_ASSERT(sizeof(OperatorArgument1Type) == 0); - - /* - * Boost alternative including an error message. If the C++0x static_assert feature is - * not available, BOOST_STATIC_ASSERT_MSG(x, msg) will be treated as BOOST_STATIC_ASSERT(x) - * and unfortunately you won't see the message. - */ - //BOOST_STATIC_ASSERT_MSG(sizeof(T1) == 0, "Vector3D has not been tested with this combination of types"); + /* + * If this assertion trips, it means that Vector3D::operator* or + * Vector3D::operator/ have not been tested with this + * combination of types (OperatorArgument1Type, + * OperatorArgument2Type) + * + * One would like to write static_assert(false), however if the + * static assertion is not dependent upon one or more template + * parameters, then the compiler is permitted to evaluate the + * static assertion at the point it is first seen, irrespective + * of whether the template is ever instantiated. + * + */ + static_assert(sizeof(OperatorArgument1Type) == 0, + "Vector3D has not been tested with this combination of types"); }; // Trivial case: both arguments share type. diff --git a/Code/util/Vector3DHemeLb.cc b/Code/util/Vector3DHemeLb.cc index 9f2e36df4..eb2a3ce70 100644 --- a/Code/util/Vector3DHemeLb.cc +++ b/Code/util/Vector3DHemeLb.cc @@ -4,13 +4,15 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. +#include "units.h" #include "util/Vector3D.h" -#include "net/mpi.h" +#include "comm/MpiError.h" +#include "comm/MpiDataType.h" #include "Exception.h" namespace hemelb { - namespace net + namespace comm { template MPI_Datatype GenerateTypeForVector() @@ -18,7 +20,7 @@ namespace hemelb const int typeCount = 1; int blocklengths[typeCount] = { 3 }; - MPI_Datatype types[typeCount] = { net::MpiDataType() }; + MPI_Datatype types[typeCount] = { comm::MpiDataType() }; MPI_Aint displacements[typeCount] = { 0 }; diff --git a/Code/util/static_assert.h b/Code/util/static_assert.h deleted file mode 100644 index 395b8ebec..000000000 --- a/Code/util/static_assert.h +++ /dev/null @@ -1,44 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_UTIL_STATIC_ASSERT_H -#define HEMELB_UTIL_STATIC_ASSERT_H - -#ifdef HEMELB_NO_STATIC_ASSERT - -#define HEMELB_STATIC_ASSERT(B) - -#else // HEMELB_NO_STATIC_ASSERT - -#define HEMELB_UTIL_STATIC_ASSERT_BOOL_CAST( x ) ((x) == 0 ? false : true) -#define HEMELB_JOIN(X,Y) HEMELB_DO_JOIN(X,Y) -#define HEMELB_DO_JOIN(X,Y) X##Y -namespace hemelb -{ - namespace util - { - - template struct STATIC_ASSERTION_FAILURE; - - template<> struct STATIC_ASSERTION_FAILURE - { - enum - { - value = 1 - }; - }; - - template struct static_assert_test - { - }; - -#define HEMELB_STATIC_ASSERT( B ) typedef ::hemelb::util::static_assert_test< sizeof(::hemelb::util::STATIC_ASSERTION_FAILURE< HEMELB_UTIL_STATIC_ASSERT_BOOL_CAST( B ) >) > HEMELB_JOIN(static_assert_typedef,__LINE__) - } -} - -#endif // HEMELB_NO_STATIC_ASSERT - -#endif // HEMELB_UTIL_STATIC_ASSERT_H diff --git a/Code/util/utilityFunctions.cc b/Code/util/utilityFunctions.cc index 16ecb8b76..3b5dfb6b6 100644 --- a/Code/util/utilityFunctions.cc +++ b/Code/util/utilityFunctions.cc @@ -4,7 +4,7 @@ // file AUTHORS. This software is provided under the terms of the // license in the file LICENSE. -#include "net/mpi.h" +#include namespace hemelb { namespace util diff --git a/Code/vis/BasicPixel.cc b/Code/vis/BasicPixel.cc deleted file mode 100644 index 89fedc596..000000000 --- a/Code/vis/BasicPixel.cc +++ /dev/null @@ -1,81 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "vis/BasicPixel.h" - -namespace hemelb -{ - namespace vis - { - BasicPixel::BasicPixel() - { - - } - BasicPixel::BasicPixel(int iIn, int jIn) : - i(iIn), j(jIn) - { - - } - - int BasicPixel::GetI() const - { - return i; - } - - int BasicPixel::GetJ() const - { - return j; - } - - bool BasicPixel::operator <(const BasicPixel& right) const - { - return (i < right.i || (i == right.i && j < right.j)); - } - - bool BasicPixel::operator ==(const BasicPixel& right) const - { - return (i == right.i && j == right.j); - } - - MPI_Datatype BasicPixel::GetMPIType() - { - const int typeCount = 2; - int blocklengths[typeCount] = { 1, 1 }; - - MPI_Datatype types[typeCount] = { MPI_INT, MPI_INT }; - - BasicPixel example; - - MPI_Aint displacements[typeCount]; - - MPI_Get_address(&example.i, &displacements[0]); - MPI_Get_address(&example.j, &displacements[1]); - - displacements[1] -= displacements[0]; - displacements[0] = 0; - - MPI_Datatype ret; - - HEMELB_MPI_CALL( - MPI_Type_create_struct, - (typeCount, blocklengths, displacements, types, &ret) - ); - - return ret; - } - } - - namespace net - { - template<> - MPI_Datatype net::MpiDataTypeTraits::RegisterMpiDataType() - { - MPI_Datatype ret = vis::BasicPixel::GetMPIType(); - HEMELB_MPI_CALL(MPI_Type_commit, (&ret)); - return ret; - } - } -} diff --git a/Code/vis/BasicPixel.h b/Code/vis/BasicPixel.h deleted file mode 100644 index 067c73420..000000000 --- a/Code/vis/BasicPixel.h +++ /dev/null @@ -1,56 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_BASICPIXEL_H -#define HEMELB_VIS_BASICPIXEL_H - -#include "net/mpi.h" - -namespace hemelb -{ - namespace vis - { - class BasicPixel - { - public: - BasicPixel(); - BasicPixel(int iIn, int jIn); - - int GetI() const; - int GetJ() const; - - bool operator <(const BasicPixel& right) const; - bool operator ==(const BasicPixel& right) const; - - /** - * NB This is not virtual, but it will be shadowed by derived classes. - * In the base case, we don't combine the pixels as they aren't associated - * with any data. - */ - void Combine(const BasicPixel& other) const - { - - } - - /** - * Produces an MPI Datatype object but doesn't commit it or manage its memory. - * @return - */ - static MPI_Datatype GetMPIType(); - - protected: - int i, j; - }; - } - - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); - } -} - -#endif /* HEMELB_VIS_BASICPIXEL_H */ diff --git a/Code/vis/CMakeLists.txt b/Code/vis/CMakeLists.txt deleted file mode 100644 index 08829223c..000000000 --- a/Code/vis/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ - -# This file is part of HemeLB and is Copyright (C) -# the HemeLB team and/or their institutions, as detailed in the -# file AUTHORS. This software is provided under the terms of the -# license in the file LICENSE. -add_library(hemelb_vis - GlyphDrawer.cc Control.cc Screen.cc Viewpoint.cc BasicPixel.cc Rendering.cc ResultPixel.cc - rayTracer/ClusterNormal.cc rayTracer/ClusterWithWallNormals.cc rayTracer/HSLToRGBConverter.cc - rayTracer/RayDataEnhanced.cc rayTracer/RayDataNormal.cc - streaklineDrawer/NeighbouringProcessor.cc streaklineDrawer/Particle.cc streaklineDrawer/ParticleManager.cc - streaklineDrawer/StreaklineDrawer.cc streaklineDrawer/VelocityField.cc streaklineDrawer/StreakPixel.cc - ) diff --git a/Code/vis/Control.cc b/Code/vis/Control.cc deleted file mode 100644 index 44457c8ce..000000000 --- a/Code/vis/Control.cc +++ /dev/null @@ -1,634 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include - -#include "log/Logger.h" -#include "util/utilityFunctions.h" -#include "vis/Control.h" -#include "vis/rayTracer/RayTracer.h" -#include "vis/GlyphDrawer.h" - -#include "io/writers/xdr/XdrFileWriter.h" - -namespace hemelb - -{ - namespace vis - { - Control::Control(lb::StressTypes iStressType, - net::Net* netIn, - lb::SimulationState* simState, - const lb::MacroscopicPropertyCache& propertyCache, - geometry::LatticeData* iLatDat, - reporting::Timer &atimer) : - net::PhasedBroadcastIrregular(netIn, simState, SPREADFACTOR), - propertyCache(propertyCache), latticeData(iLatDat), timer(atimer) - { - - visSettings.mStressType = iStressType; - - this->vis = new Vis; - - //sites_x etc are globals declared in net.h - vis->half_dim = util::Vector3D(iLatDat->GetSiteDimensions()) * 0.5F; - - vis->system_size = 2.F * fmaxf(vis->half_dim[0], fmaxf(vis->half_dim[1], vis->half_dim[2])); - - visSettings.mouse_x = -1; - visSettings.mouse_y = -1; - - initLayers(); - } - - void Control::initLayers() - { - // We don't have all the minima / maxima on one core, so we have to gather them. - // NOTE this only happens once, during initialisation, otherwise it would be - // totally unforgivable. - site_t block_min_x = std::numeric_limits < site_t > ::max(); - site_t block_min_y = std::numeric_limits < site_t > ::max(); - site_t block_min_z = std::numeric_limits < site_t > ::max(); - site_t block_max_x = std::numeric_limits < site_t > ::min(); - site_t block_max_y = std::numeric_limits < site_t > ::min(); - site_t block_max_z = std::numeric_limits < site_t > ::min(); - - for (geometry::BlockTraverser blockIt(*latticeData); blockIt.CurrentLocationValid(); blockIt.TraverseOne()) - { - if (blockIt.GetCurrentBlockData().IsEmpty()) - { - continue; - } - - block_min_x = util::NumericalFunctions::min(block_min_x, blockIt.GetCurrentLocation().x); - block_min_y = util::NumericalFunctions::min(block_min_y, blockIt.GetCurrentLocation().y); - block_min_z = util::NumericalFunctions::min(block_min_z, blockIt.GetCurrentLocation().z); - block_max_x = util::NumericalFunctions::max(block_max_x, blockIt.GetCurrentLocation().x); - block_max_y = util::NumericalFunctions::max(block_max_y, blockIt.GetCurrentLocation().y); - block_max_z = util::NumericalFunctions::max(block_max_z, blockIt.GetCurrentLocation().z); - } - - std::vector localMins(3), localMaxes(3); - - localMins[0] = block_min_x; - localMins[1] = block_min_y; - localMins[2] = block_min_z; - - localMaxes[0] = block_max_x; - localMaxes[1] = block_max_y; - localMaxes[2] = block_max_z; - - const net::MpiCommunicator& comms = this->mNet->GetCommunicator(); - std::vector mins = comms.AllReduce(localMins, MPI_MIN); - std::vector maxes = comms.AllReduce(localMaxes, MPI_MAX); - - visSettings.ctr_x = 0.5F * (float) (latticeData->GetBlockSize() * (mins[0] + maxes[0])); - visSettings.ctr_y = 0.5F * (float) (latticeData->GetBlockSize() * (mins[1] + maxes[1])); - visSettings.ctr_z = 0.5F * (float) (latticeData->GetBlockSize() * (mins[2] + maxes[2])); - - normalRayTracer = - new raytracer::RayTracer(latticeData, - &domainStats, - &screen, - &viewpoint, - &visSettings); - - myGlypher = new GlyphDrawer(latticeData, &screen, &domainStats, &viewpoint, &visSettings); - -#ifndef NO_STREAKLINES - myStreaker = new streaklinedrawer::StreaklineDrawer(*latticeData, screen, viewpoint, visSettings, propertyCache, mNet->GetCommunicator()); -#else - myStreaker = NULL; -#endif - // Note that rtInit does stuff to this->ctr_x (because this has - // to be global) - visSettings.ctr_x -= vis->half_dim[0]; - visSettings.ctr_y -= vis->half_dim[1]; - visSettings.ctr_z -= vis->half_dim[2]; - } - - void Control::SetProjection(const int &iPixels_x, - const int &iPixels_y, - const float &iLocal_ctr_x, - const float &iLocal_ctr_y, - const float &iLocal_ctr_z, - const float &iLongitude, - const float &iLatitude, - const float &iZoom) - { - float rad = 5.F * vis->system_size; - float dist = 0.5F * rad; - - //For now set the maximum draw distance to twice the radius; - visSettings.maximumDrawDistance = 2.0F * rad; - - util::Vector3D centre = util::Vector3D(iLocal_ctr_x, iLocal_ctr_y, iLocal_ctr_z); - - viewpoint.SetViewpointPosition(iLongitude * (float) DEG_TO_RAD, - iLatitude * (float) DEG_TO_RAD, - centre, - rad, - dist); - - screen.Set( (0.5F * vis->system_size) / iZoom, - (0.5F * vis->system_size) / iZoom, - iPixels_x, - iPixels_y, - rad, - &viewpoint); - } - - void Control::SetSomeParams(const float iBrightness, - const distribn_t iDensityThresholdMin, - const distribn_t iDensityThresholdMinMaxInv, - const distribn_t iVelocityThresholdMaxInv, - const distribn_t iStressThresholdMaxInv) - { - visSettings.brightness = iBrightness; - domainStats.density_threshold_min = iDensityThresholdMin; - - domainStats.density_threshold_minmax_inv = iDensityThresholdMinMaxInv; - domainStats.velocity_threshold_max_inv = iVelocityThresholdMaxInv; - domainStats.stress_threshold_max_inv = iStressThresholdMaxInv; - } - - void Control::UpdateImageSize(int pixels_x, int pixels_y) - { - screen.Resize(pixels_x, pixels_y); - } - - void Control::Render(unsigned long startIteration) - { - log::Logger::Log("Rendering."); - - PixelSet* ray = normalRayTracer->Render(propertyCache); - - PixelSet *glyph = NULL; - - if (visSettings.mode == VisSettings::ISOSURFACESANDGLYPHS) - { - glyph = myGlypher->Render(propertyCache); - } - else - { - glyph = myGlypher->GetUnusedPixelSet(); - glyph->Clear(); - } - - PixelSet *streak = NULL; - - if (myStreaker != NULL - && (visSettings.mStressType == lb::ShearStress || visSettings.mode == VisSettings::WALLANDSTREAKLINES)) - { - streak = myStreaker->Render(); - } - - localResultsByStartIt.insert(std::pair(startIteration, Rendering(glyph, ray, streak))); - } - - void Control::InitialAction(unsigned long startIteration) - { - timer.Start(); - - Render(startIteration); - - log::Logger::Log("Render stored for phased imaging."); - - timer.Stop(); - } - - void Control::WriteImage(io::writers::Writer* writer, - const PixelSet& imagePixels, - const DomainStats& domainStats, - const VisSettings& visSettings) const - { - *writer << (int) visSettings.mode; - - *writer << domainStats.physical_pressure_threshold_min << domainStats.physical_pressure_threshold_max - << domainStats.physical_velocity_threshold_max << domainStats.physical_stress_threshold_max; - - *writer << screen.GetPixelsX(); - *writer << screen.GetPixelsY(); - *writer << (int) imagePixels.GetPixelCount(); - - WritePixels(writer, imagePixels, domainStats, visSettings); - } - - int Control::GetPixelsX() const - { - return screen.GetPixelsX(); - } - - int Control::GetPixelsY() const - { - return screen.GetPixelsY(); - } - - void Control::WritePixels(io::writers::Writer* writer, - const PixelSet& imagePixels, - const DomainStats& domainStats, - const VisSettings& visSettings) const - { - const int bits_per_char = sizeof(char) * 8; - - for (unsigned int i = 0; i < imagePixels.GetPixelCount(); i++) - { - const ResultPixel& pixel = imagePixels.GetPixels()[i]; - - // Use a ray-tracer function to get the necessary pixel data. - unsigned index; - unsigned char rgb_data[12]; - - pixel.WritePixel(&index, rgb_data, domainStats, visSettings); - - *writer << (uint32_t) index; - - unsigned pix_data[3]; - pix_data[0] = (rgb_data[0] << (3 * bits_per_char)) + (rgb_data[1] << (2 * bits_per_char)) - + (rgb_data[2] << bits_per_char) + rgb_data[3]; - - pix_data[1] = (rgb_data[4] << (3 * bits_per_char)) + (rgb_data[5] << (2 * bits_per_char)) - + (rgb_data[6] << bits_per_char) + rgb_data[7]; - - pix_data[2] = (rgb_data[8] << (3 * bits_per_char)) + (rgb_data[9] << (2 * bits_per_char)) - + (rgb_data[10] << bits_per_char) + rgb_data[11]; - - for (int i = 0; i < 3; i++) - { - *writer << (uint32_t) pix_data[i]; - } - *writer << io::writers::Writer::eol; - } - } - - void Control::ProgressFromChildren(unsigned long startIteration, unsigned long splayNumber) - { - timer.Start(); - - if (splayNumber == 0) - { - for (unsigned int ii = 0; ii < GetChildren().size(); ++ii) - { - Rendering lRendering(myGlypher->GetUnusedPixelSet(), - normalRayTracer->GetUnusedPixelSet(), - myStreaker != NULL ? - myStreaker->GetUnusedPixelSet() : - NULL); - - lRendering.ReceivePixelCounts(mNet, GetChildren()[ii]); - - childrenResultsByStartIt.insert(std::pair(startIteration, lRendering)); - } - - log::Logger::Log("Receiving child image pixel count."); - } - else if (splayNumber == 1) - { - std::multimap::iterator renderings = - childrenResultsByStartIt.equal_range(startIteration).first; - - for (unsigned int ii = 0; ii < GetChildren().size(); ++ii) - { - Rendering& received = (*renderings).second; - - log::Logger::Log("Receiving child image pixel data (from it %li).", - startIteration); - - received.ReceivePixelData(mNet, GetChildren()[ii]); - - renderings++; - } - } - - timer.Stop(); - } - - void Control::ProgressToParent(unsigned long startIteration, unsigned long splayNumber) - { - timer.Start(); - - Rendering& rendering = (*localResultsByStartIt.find(startIteration)).second; - if (splayNumber == 0) - { - log::Logger::Log("Sending pixel count (from it %li).", startIteration); - - rendering.SendPixelCounts(mNet, GetParent()); - } - else if (splayNumber == 1) - { - log::Logger::Log("Sending pixel data (from it %li).", startIteration); - - rendering.SendPixelData(mNet, GetParent()); - } - - timer.Stop(); - } - - void Control::PostReceiveFromChildren(unsigned long startIteration, unsigned long splayNumber) - { - timer.Start(); - - // The first time round, ensure that we have enough memory to receive the image data that - // will come next time. - if (splayNumber == 0) - { - - } - if (splayNumber == 1) - { - std::pair < std::multimap::iterator , std::multimap::iterator - > its = childrenResultsByStartIt.equal_range(startIteration); - - Rendering local = (*localResultsByStartIt.find(startIteration)).second; - - std::multimap::iterator rendering = its.first; - while (rendering != its.second) - { - local.Combine( (*rendering).second); - - (*rendering).second.ReleaseAll(); - - rendering++; - } - - if (its.first != its.second) - { - its.first++; - childrenResultsByStartIt.erase(its.first, its.second); - } - - log::Logger::Log("Combining in child pixel data."); - } - - timer.Stop(); - } - - void Control::PostSendToParent(unsigned long startIteration, unsigned long splayNumber) - { - if (splayNumber == 1) - { - Rendering& rendering = (*localResultsByStartIt.find(startIteration)).second; - rendering.ReleaseAll(); - - localResultsByStartIt.erase(startIteration); - } - } - - bool Control::IsRendering() const - { - return IsInitialAction() || IsInstantBroadcast(); - } - - void Control::ClearOut(unsigned long startIt) - { - timer.Start(); - - bool found; - - do - { - found = false; - - if (localResultsByStartIt.size() > 0) - { - mapType::iterator it = localResultsByStartIt.begin(); - if (it->first <= startIt) - { - log::Logger::Log("Clearing out image cache from it %lu", it->first); - - (*it).second.ReleaseAll(); - - localResultsByStartIt.erase(it); - found = true; - } - } - } - while (found); - - do - { - found = false; - - if (childrenResultsByStartIt.size() > 0) - { - multimapType::iterator it = childrenResultsByStartIt.begin(); - if ( (*it).first <= startIt) - { - log::Logger::Log("Clearing out image cache from it %lu", (*it).first); - - (*it).second.ReleaseAll(); - - childrenResultsByStartIt.erase(it); - found = true; - } - } - } - while (found); - - do - { - found = false; - - if (renderingsByStartIt.size() > 0) - { - std::multimap*>::iterator it = renderingsByStartIt.begin(); - if ( (*it).first <= startIt) - { - log::Logger::Log("Clearing out image cache from it %lu", (*it).first); - - (*it).second->Release(); - renderingsByStartIt.erase(it); - found = true; - } - } - } - while (found); - - timer.Stop(); - } - - const PixelSet* Control::GetResult(unsigned long startIt) - { - log::Logger::Log("Getting image results from it %lu", startIt); - - if (renderingsByStartIt.count(startIt) != 0) - { - return (*renderingsByStartIt.find(startIt)).second; - } - - if (localResultsByStartIt.count(startIt) != 0) - { - Rendering finalRender = (*localResultsByStartIt.find(startIt)).second; - PixelSet *result = GetUnusedPixelSet(); - - finalRender.PopulateResultSet(result); - - renderingsByStartIt.insert(std::pair*>(startIt, result)); - return result; - } - else - { - return NULL; - } - } - - void Control::InstantBroadcast(unsigned long startIteration) - { - timer.Start(); - - log::Logger::Log("Performing instant imaging."); - - Render(startIteration); - - /* - * We do several iterations. - * - * On the first, every even proc passes data to the odd proc below, where it is merged. - * On the second, the difference is two, so proc 3 passes to 1, 7 to 5, 11 to 9 etc. - * On the third the differenec is four, so proc 5 passes to 1, 13 to 9 etc. - * . - * . - * . - * - * This continues until all data is passed back to processor one, which passes it to proc 0. - */ - - const net::MpiCommunicator& netComm = this->mNet->GetCommunicator(); - net::Net tempNet(netComm); - - Rendering* localBuffer = localResultsByStartIt.count(startIteration) > 0 ? - & (*localResultsByStartIt.find(startIteration)).second : - NULL; - Rendering receiveBuffer(myGlypher->GetUnusedPixelSet(), normalRayTracer->GetUnusedPixelSet(), myStreaker == NULL ? - NULL : - myStreaker->GetUnusedPixelSet()); - - // Start with a difference in rank of 1, doubling every time. - for (proc_t deltaRank = 1; deltaRank < netComm.Size(); deltaRank <<= 1) - { - // The receiving proc is all the ranks that are 1 modulo (deltaRank * 2) - for (proc_t receivingProc = 1; receivingProc < (netComm.Size() - deltaRank); - receivingProc += deltaRank << 1) - { - proc_t sendingProc = receivingProc + deltaRank; - - // If we're the sending proc, do the send. - if (netComm.Rank() == sendingProc) - { - localBuffer->SendPixelCounts(&tempNet, receivingProc); - - tempNet.Dispatch(); - - localBuffer->SendPixelData(&tempNet, receivingProc); - - tempNet.Dispatch(); - } - - // If we're the receiving proc, receive. - - else if (netComm.Rank() == receivingProc) - { - receiveBuffer.ReceivePixelCounts(&tempNet, sendingProc); - - tempNet.Dispatch(); - - receiveBuffer.ReceivePixelData(&tempNet, sendingProc); - - tempNet.Dispatch(); - - localBuffer->Combine(receiveBuffer); - } - } - } - - // Send the final image from proc 1 to 0. - if (netComm.Rank() == 1) - { - localBuffer->SendPixelCounts(&tempNet, 0); - - tempNet.Dispatch(); - - localBuffer->SendPixelData(&tempNet, 0); - - tempNet.Dispatch(); - } - // Receive the final image on proc 0. - else if (netComm.Rank() == 0 && netComm.Size() > 1) - { - receiveBuffer.ReceivePixelCounts(&tempNet, 1); - - tempNet.Dispatch(); - - receiveBuffer.ReceivePixelData(&tempNet, 1); - - tempNet.Dispatch(); - - localResultsByStartIt.erase(startIteration); - localResultsByStartIt.insert(std::pair(startIteration, Rendering(receiveBuffer))); - - log::Logger::Log("Inserting image at it %lu.", startIteration); - } - - if (netComm.Rank() != 0) - { - receiveBuffer.ReleaseAll(); - } - - timer.Stop(); - } - - void Control::SetMouseParams(double iPhysicalPressure, double iPhysicalStress) - { - visSettings.mouse_pressure = iPhysicalPressure; - visSettings.mouse_stress = iPhysicalStress; - } - - bool Control::MouseIsOverPixel(const PixelSet* result, float* density, float* stress) - { - if (visSettings.mouse_x < 0 || visSettings.mouse_y < 0) - { - return false; - } - - const std::vector& screenPix = result->GetPixels(); - - for (std::vector::const_iterator it = screenPix.begin(); it != screenPix.end(); ++it) - { - if ( (*it).GetRayPixel() != NULL && (*it).GetI() == visSettings.mouse_x && (*it).GetJ() == visSettings.mouse_y) - { - *density = (*it).GetRayPixel()->GetNearestDensity(); - *stress = (*it).GetRayPixel()->GetNearestStress(); - - return true; - } - } - - return false; - } - - void Control::ProgressStreaklines(unsigned long time_step, unsigned long period) - { - if (myStreaker != NULL) - { - timer.Start(); - myStreaker->ProgressStreaklines(time_step, period); - timer.Stop(); - } - } - - Control::~Control() - { - delete myStreaker; - delete vis; - delete myGlypher; - delete normalRayTracer; - } - - } // namespace vis -} -// namespace hemelb diff --git a/Code/vis/Control.h b/Code/vis/Control.h deleted file mode 100644 index 3d87a9c98..000000000 --- a/Code/vis/Control.h +++ /dev/null @@ -1,153 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_CONTROL_H -#define HEMELB_VIS_CONTROL_H - -#include - -#include "geometry/LatticeData.h" - -#include "lb/LbmParameters.h" -#include "lb/SimulationState.h" - -#include "net/net.h" -#include "net/PhasedBroadcastIrregular.h" - -#include "vis/DomainStats.h" -#include "vis/GlyphDrawer.h" -#include "vis/rayTracer/ClusterWithWallNormals.h" -#include "vis/rayTracer/RayDataNormal.h" -#include "vis/rayTracer/RayDataEnhanced.h" -#include "vis/rayTracer/RayTracer.h" -#include "vis/Rendering.h" -#include "vis/ResultPixel.h" -#include "vis/Screen.h" -#include "vis/streaklineDrawer/StreaklineDrawer.h" -#include "vis/Viewpoint.h" -#include "vis/VisSettings.h" -#include "reporting/Timers.h" - -namespace hemelb -{ - namespace vis - { - /** - * Class to control and use the effects of different visualisation methods. - * - * We use irregular phased broadcasting because we don't know in advance which iterations we'll - * need to generate images on. - * - * The initial action is used to render the image on each core. 2 communications are required - * between each pair of nodes so that the number of pixels can be communicated before the pixels - * themselves. No overlap is possible between communications at different depths as the pixels - * must be merged before they can be passed on. We don't need to pass info top-down, we only - * pass image components upwards towards the top node. - */ - class Control : public net::PhasedBroadcastIrregular, - private PixelSetStore > - { - public: - Control(lb::StressTypes iStressType, - net::Net* net, - lb::SimulationState* simState, - const lb::MacroscopicPropertyCache& propertyCache, - geometry::LatticeData* iLatDat, - reporting::Timer &atimer); - ~Control(); - - void SetSomeParams(const float iBrightness, - const distribn_t iDensityThresholdMin, - const distribn_t iDensityThresholdMinMaxInv, - const distribn_t iVelocityThresholdMaxInv, - const distribn_t iStressThresholdMaxInv); - - void SetProjection(const int &pixels_x, - const int &pixels_y, - const float &ctr_x, - const float &ctr_y, - const float &ctr_z, - const float &longitude, - const float &latitude, - const float &zoom); - - bool MouseIsOverPixel(const PixelSet* result, float* density, float* stress); - - void ProgressStreaklines(unsigned long time_step, unsigned long period); - - void UpdateImageSize(int pixels_x, int pixels_y); - void SetMouseParams(double iPhysicalPressure, double iPhysicalStress); - - const PixelSet* GetResult(unsigned long startIteration); - - void WritePixels(io::writers::Writer* writer, - const PixelSet& imagePixels, - const DomainStats& domainStats, - const VisSettings& visSettings) const; - void WriteImage(io::writers::Writer* writer, - const PixelSet& imagePixels, - const DomainStats& domainStats, - const VisSettings& visSettings) const; - - bool IsRendering() const; - - int GetPixelsX() const; - int GetPixelsY() const; - - Viewpoint viewpoint; - DomainStats domainStats; - VisSettings visSettings; - - protected: - void InitialAction(unsigned long startIteration); - void ProgressFromChildren(unsigned long startIteration, unsigned long splayNumber); - void ProgressToParent(unsigned long startIteration, unsigned long splayNumber); - void PostReceiveFromChildren(unsigned long startIteration, unsigned long splayNumber); - void PostSendToParent(unsigned long startIteration, unsigned long splayNumber); - void ClearOut(unsigned long startIteration); - void InstantBroadcast(unsigned long startIteration); - - private: - typedef net::PhasedBroadcastIrregular base; - typedef net::PhasedBroadcast deepbase; - typedef std::map mapType; - typedef std::multimap multimapType; - - // This is mainly constrained by the memory available per core. - static const unsigned int SPREADFACTOR = 2; - - struct Vis - { - util::Vector3D half_dim; - float system_size; - }; - - void initLayers(); - void Render(unsigned long startIteration); - - mapType localResultsByStartIt; - multimapType childrenResultsByStartIt; - std::multimap*> renderingsByStartIt; - - /** - * Cache of all the macroscopic fluid properties of each lattice site on this core. - */ - const lb::MacroscopicPropertyCache& propertyCache; - - geometry::LatticeData* latticeData; - Screen screen; - Vis* vis; - raytracer::RayTracer - *normalRayTracer; - GlyphDrawer *myGlypher; - streaklinedrawer::StreaklineDrawer *myStreaker; - - reporting::Timer &timer; - }; - } -} - -#endif // HEMELB_VIS_CONTROL_H diff --git a/Code/vis/DomainStats.h b/Code/vis/DomainStats.h deleted file mode 100644 index cc6af2927..000000000 --- a/Code/vis/DomainStats.h +++ /dev/null @@ -1,32 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_DOMAINSTATS_H -#define HEMELB_VIS_DOMAINSTATS_H - -#include "constants.h" - -namespace hemelb -{ - namespace vis - { - struct DomainStats - { - public: - distribn_t velocity_threshold_max_inv; - distribn_t stress_threshold_max_inv; - distribn_t density_threshold_min; - distribn_t density_threshold_minmax_inv; - float physical_pressure_threshold_min; - float physical_pressure_threshold_max; - float physical_velocity_threshold_max; - float physical_stress_threshold_max; - }; - } -} - -#endif /* HEMELB_VIS_DOMAINSTATS_H */ - diff --git a/Code/vis/GlyphDrawer.cc b/Code/vis/GlyphDrawer.cc deleted file mode 100644 index ba7f9ca95..000000000 --- a/Code/vis/GlyphDrawer.cc +++ /dev/null @@ -1,214 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "vis/GlyphDrawer.h" -#include "vis/Control.h" - -namespace hemelb -{ - namespace vis - { - // Constructor - GlyphDrawer::GlyphDrawer(geometry::LatticeData* iLatDat, - Screen* iScreen, - DomainStats* iDomainStats, - Viewpoint* iViewpoint, - VisSettings* iVisSettings) : - mLatDat(iLatDat), mScreen(iScreen), mDomainStats(iDomainStats), mViewpoint(iViewpoint), mVisSettings(iVisSettings) - { - for (geometry::BlockTraverser blockTrav(*mLatDat); blockTrav.CurrentLocationValid(); blockTrav.TraverseOne()) - { - // Get the block data for this block - if it has no site data, move on. - const geometry::Block& block = blockTrav.GetCurrentBlockData(); - - if (block.IsEmpty()) - { - continue; - } - - // We put the glyph at the site at the centre of the block... - const util::Vector3D midBlockSite = util::Vector3D(mLatDat->GetBlockSize()) / (site_t) 2; - - const site_t siteIdOnBlock = mLatDat->GetLocalSiteIdFromLocalSiteCoords(midBlockSite); - - // ... (only if there's fluid there). - if (block.SiteIsSolid(siteIdOnBlock)) - { - continue; - } - - // Create a glyph at the desired location - Glyph lGlyph; - - util::Vector3D globalSiteCoords = mLatDat->GetGlobalCoords(blockTrav.GetCurrentLocation(), - midBlockSite); - - lGlyph.x = float(globalSiteCoords.x) - 0.5F * float(mLatDat->GetSiteDimensions().x); - lGlyph.y = float(globalSiteCoords.y) - 0.5F * float(mLatDat->GetSiteDimensions().y); - lGlyph.z = float(globalSiteCoords.z) - 0.5F * float(mLatDat->GetSiteDimensions().z); - - lGlyph.siteId = block.GetLocalContiguousIndexForSite(siteIdOnBlock); - - mGlyphs.push_back(lGlyph); - } - } - - /** - * Destructor - */ - GlyphDrawer::~GlyphDrawer() - { - } - - /** - * Render a line between two points on the screen. - * - * @param endPoint1 - * @param endPoint2 - * @param iStressType - * @param mode - */ - void GlyphDrawer::RenderLine(const XYCoordinates& endPoint1, - const XYCoordinates& endPoint2, - const VisSettings* visSettings, - PixelSet* pixelSet) - { - // Store end points of the line and 'current' point (x and y). - int x = int(endPoint1.x); - int y = int(endPoint1.y); - - // Ensure increasing x from point 1 to point 2. - int x1, y1, x2, y2; - if (endPoint2.x < endPoint1.x) - { - x1 = int(endPoint2.x); - y1 = int(endPoint2.y); - x2 = x; - y2 = y; - } - else - { - x1 = x; - y1 = y; - x2 = int(endPoint2.x); - y2 = int(endPoint2.y); - } - - // Ensure increasing y. - if (y2 <= y1) - { - int temp = y2; - y2 = y; - y = temp; - } - - // Set dx with the difference between x-values at the endpoints. - int dx = x2 - x1; - int dy = y2 - y; - - if (dx > dy) - { - RenderLineHelper(x, y, dy, dy - dx, x2, visSettings, pixelSet); - } - else - { - RenderLineHelper(x, y, dx, dx - dy, y2, visSettings, pixelSet); - } - } - - /** - * Helper function for rendering a line. The template parameter indicates whether the end of - * the line will be limited by the x or y dimension. - * - * @tparam xLimited True if the line is limited by an x value, false if limited by a y value. - * @param x Starting, minimal value of x. - * @param y Starting, minimal value of y. - * @param smallerDelta is the smaller of the two deltas between initial- and end- x and y. - * @param deltaDifference is (smaller delta) - (larger delta) - * @param limit Maximal value in the limiting direction - */ - template - void GlyphDrawer::RenderLineHelper(int x, - int y, - const int smallerDelta, - const int deltaDifference, - const int limit, - const VisSettings* visSettings, - PixelSet* pixelSet) - { - // This variable tracks whether we are above or below the line we draw. - int overOrUnderMeasure = deltaDifference; - - while ( (xLimited && x <= limit) || (!xLimited && y <= limit)) - { - // If on screen, add a pixel. - if (x >= 0 && x < (int) mScreen->GetPixelsX() && y >= 0 && y < (int) mScreen->GetPixelsY()) - { - BasicPixel pixel(x, y); - pixelSet->AddPixel(pixel); - } - - // We are effectively adding the smallerDelta every time (and increment in the other direction), - // and we periodically subtract a largerDelta (and increment in the limited direction). - if (overOrUnderMeasure < 0) - { - overOrUnderMeasure += smallerDelta; - if (xLimited) - { - ++x; - } - else - { - ++y; - } - } - else - { - overOrUnderMeasure += deltaDifference; - ++y; - ++x; - } - } // end while - } - - /** - * Perform the rendering for each glyph. - */ - PixelSet* GlyphDrawer::Render(const lb::MacroscopicPropertyCache& propertyCache) - { - PixelSet* pixelSet = GetUnusedPixelSet(); - - pixelSet->Clear(); - - // For each glyph... - for (site_t n = 0; n < (site_t) mGlyphs.size(); n++) - { - // ... get the velocity at that point... - const util::Vector3D& velocity = propertyCache.velocityCache.Get(mGlyphs[n].siteId); - - // ... calculate the velocity vector multiplier... - const double temp = mVisSettings->glyphLength * ((distribn_t) mLatDat->GetBlockSize()) - * mDomainStats->velocity_threshold_max_inv; - - // ... calculate the two ends of the line we're going to draw... - util::Vector3D p1 = util::Vector3D(mGlyphs[n].x, mGlyphs[n].y, mGlyphs[n].z); - util::Vector3D p2 = p1 - + util::Vector3D(float(velocity.x * temp), float(velocity.y * temp), float(velocity.z * temp)); - - // ... transform to the location on the screen, and render. - XYCoordinates p3 = mViewpoint->FlatProject(p1); - XYCoordinates p4 = mViewpoint->FlatProject(p2); - - p3 = mScreen->TransformScreenToPixelCoordinates(p3); - p4 = mScreen->TransformScreenToPixelCoordinates(p4); - - RenderLine(p3, p4, mVisSettings, pixelSet); - } - - return pixelSet; - } - } -} diff --git a/Code/vis/GlyphDrawer.h b/Code/vis/GlyphDrawer.h deleted file mode 100644 index 0f9191b84..000000000 --- a/Code/vis/GlyphDrawer.h +++ /dev/null @@ -1,85 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_GLYPHDRAWER_H -#define HEMELB_VIS_GLYPHDRAWER_H - -#include "geometry/LatticeData.h" -#include "lb/MacroscopicPropertyCache.h" - -#include "vis/BasicPixel.h" -#include "vis/PixelSet.h" -#include "vis/PixelSetStore.h" -#include "vis/Screen.h" -#include "vis/DomainStats.h" -#include "vis/Viewpoint.h" -#include "vis/VisSettings.h" - -#include -#include - -namespace hemelb -{ - namespace vis - { - // Class for drawing glyphs. - class GlyphDrawer : public PixelSetStore > - { - public: - // Constructor and destructor - GlyphDrawer(geometry::LatticeData* iLatDat, - Screen* iScreen, - DomainStats* iDomainStats, - Viewpoint* iViewpoint, - VisSettings* iVisSettings); - ~GlyphDrawer(); - - // Function to perform the rendering. - PixelSet* Render(const lb::MacroscopicPropertyCache& propertyCache); - - private: - // A struct to represent a single glyph. - struct Glyph - { - /** - * The 3D coordinates of the glyph. - */ - float x, y, z; - /** - * The local contiguous site id near there. - */ - site_t siteId; - }; - - void RenderLine(const XYCoordinates& endPoint1, - const XYCoordinates& endPoint2, - const VisSettings* visSettings, - PixelSet*); - - template - void RenderLineHelper(int x, - int y, - int incE, - int incNE, - int limit, - const VisSettings* visSettings, - PixelSet*); - - geometry::LatticeData* mLatDat; - - Screen* mScreen; - DomainStats* mDomainStats; - Viewpoint* mViewpoint; - VisSettings* mVisSettings; - - std::vector mGlyphs; - - }; - - } -} - -#endif //HEMELB_VIS_GLYPHDRAWER_H diff --git a/Code/vis/PixelSet.h b/Code/vis/PixelSet.h deleted file mode 100644 index 778787908..000000000 --- a/Code/vis/PixelSet.h +++ /dev/null @@ -1,147 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_PIXELSET_H -#define HEMELB_VIS_PIXELSET_H - -#include -#include - -#include "log/Logger.h" -#include "net/mpi.h" -#include "net/net.h" -#include "vis/BasicPixel.h" -#include "vis/VisSettings.h" - -namespace hemelb -{ - namespace vis - { - /** - * Base pixel set implementation, including the functionality that allows storing of pixels - * in a vector for speedy MPI usage, but also storing a map of pixels -> Pixel* for efficient - * lookup in O(N) time. - */ - template - class PixelSet - { - public: - - PixelSet() - { - inUse = false; - count = 0; - } - - ~PixelSet() - { - } - - void Combine(const PixelSet &other) - { - typename std::vector::const_iterator otherPixels = other.pixels.begin(); - - while (otherPixels != other.pixels.end()) - { - AddPixel(*otherPixels); - otherPixels++; - } - } - - void AddPixel(const PixelType& newPixel) - { - BasicPixel location = BasicPixel(newPixel.GetI(), newPixel.GetJ()); - - if (pixelLookup.count(location) > 0) - { - pixels[pixelLookup[location]].Combine(newPixel); - } - else - { - pixels.push_back(PixelType(newPixel)); - pixelLookup.insert(std::pair(location, - (unsigned int) (pixels.size() - - 1))); - } - } - - bool IsInUse() const - { - return inUse; - } - - void SetInUse() - { - inUse = true; - } - - void Release() - { - inUse = false; - } - - void SendQuantity(net::Net* net, proc_t destination) - { - count = (int) pixels.size(); - log::Logger::Log("Sending pixel count of %i", count); - net->RequestSendR(count, destination); - } - - void ReceiveQuantity(net::Net* net, proc_t source) - { - net->RequestReceiveR(count,source); - } - - void SendPixels(net::Net* net, proc_t destination) - { - if (pixels.size() > 0) - { - log::Logger::Log("Sending %i pixels to proc %i", - (int) pixels.size(), - (int) destination); - net->RequestSendV(pixels, destination); - } - } - - void ReceivePixels(net::Net* net, proc_t source) - { - if (count > 0) - { - // First make sure the vector will be large enough to hold the incoming pixels. - log::Logger::Log("Receiving %i pixels from proc %i", - count, - (int) source); - pixels.resize(count); - net->RequestReceiveV(pixels, source); - } - } - - size_t GetPixelCount() const - { - return pixels.size(); - } - - const std::vector& GetPixels() const - { - return pixels; - } - - void Clear() - { - pixelLookup.clear(); - pixels.clear(); - } - - private: - std::map pixelLookup; - std::vector pixels; - int count; - bool inUse; - }; - } -} - -#endif // HEMELB_VIS_PIXELSET_H diff --git a/Code/vis/PixelSetStore.h b/Code/vis/PixelSetStore.h deleted file mode 100644 index bf5e2a4ff..000000000 --- a/Code/vis/PixelSetStore.h +++ /dev/null @@ -1,71 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_PIXELSETSTORE_H -#define HEMELB_VIS_PIXELSETSTORE_H - -#include -#include "log/Logger.h" - -namespace hemelb -{ - namespace vis - { - template - class PixelSetStore - { - public: - PixelSetType* GetUnusedPixelSet() - { - if (!store.empty()) - { - // Take the first pixel set (and put it on the back of the queue for reuse later). - PixelSetType* candidate = store.front(); - - store.pop(); - store.push(candidate); - - // If we can use this one to write, return it. - if (!candidate->IsInUse()) - { - candidate->SetInUse(); - candidate->Clear(); - return candidate; - } - } - - // We've not got any inUse pixel sets, so create a new one, store it and return it. - PixelSetType* ret = new PixelSetType(); - - store.push(ret); - - ret->SetInUse(); - return ret; - } - - ~PixelSetStore() - { - while (!store.empty()) - { - if (store.front()->IsInUse()) - { - log::Logger::Log("This could be a problem: we've just " - "cleared out a pixel set which appears to still be in use. If pixel sets are being " - "managed properly there shouldn't be more than a few of these messages per core."); - } - - delete store.front(); - store.pop(); - } - } - - private: - std::queue store; - }; - } -} - -#endif // HEMELB_VIS_PIXELSETSTORE_H diff --git a/Code/vis/Rendering.cc b/Code/vis/Rendering.cc deleted file mode 100644 index 4c13ba0ea..000000000 --- a/Code/vis/Rendering.cc +++ /dev/null @@ -1,137 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "vis/Rendering.h" - -namespace hemelb -{ - namespace vis - { - Rendering::Rendering(PixelSet* glyph, - PixelSet* ray, - PixelSet* streak) : - glyphResult(glyph), rayResult(ray), streakResult(streak) - { - - } - - void Rendering::ReleaseAll() - { - if (glyphResult != NULL) - { - glyphResult-> Release(); - } - - if (rayResult != NULL) - { - rayResult->Release(); - } - - if (streakResult != NULL) - { - streakResult-> Release(); - } - } - - void Rendering::ReceivePixelCounts(net::Net* inNet, proc_t source) - { - if (glyphResult != NULL) - { - glyphResult->ReceiveQuantity(inNet, source); - } - if (rayResult != NULL) - { - rayResult->ReceiveQuantity(inNet, source); - } - if (streakResult != NULL) - { - streakResult->ReceiveQuantity(inNet, source); - } - } - - void Rendering::ReceivePixelData(net::Net* inNet, proc_t source) - { - if (glyphResult != NULL) - { - glyphResult->ReceivePixels(inNet, source); - } - if (rayResult != NULL) - { - rayResult->ReceivePixels(inNet, source); - } - if (streakResult != NULL) - { - streakResult->ReceivePixels(inNet, source); - } - } - - void Rendering::SendPixelCounts(net::Net* inNet, proc_t destination) - { - if (glyphResult != NULL) - { - glyphResult->SendQuantity(inNet, destination); - } - if (rayResult != NULL) - { - rayResult->SendQuantity(inNet, destination); - } - if (streakResult != NULL) - { - streakResult->SendQuantity(inNet, destination); - } - } - - void Rendering::SendPixelData(net::Net* inNet, proc_t destination) - { - if (glyphResult != NULL) - { - glyphResult->SendPixels(inNet, destination); - } - if (rayResult != NULL) - { - rayResult->SendPixels(inNet, destination); - } - if (streakResult != NULL) - { - streakResult->SendPixels(inNet, destination); - } - } - - void Rendering::Combine(const Rendering& other) - { - if (glyphResult != NULL) - { - glyphResult->Combine(*other.glyphResult); - } - if (rayResult != NULL) - { - rayResult->Combine(*other.rayResult); - } - if (streakResult != NULL) - { - streakResult->Combine(*other.streakResult); - } - } - - void Rendering::PopulateResultSet(PixelSet* resultSet) - { - if (glyphResult != NULL) - { - AddPixelsToResultSet(resultSet, glyphResult->GetPixels()); - } - - if (rayResult != NULL) - { - AddPixelsToResultSet(resultSet, rayResult->GetPixels()); - } - - if (streakResult != NULL) - { - AddPixelsToResultSet(resultSet, streakResult->GetPixels()); - } - } - } -} diff --git a/Code/vis/Rendering.h b/Code/vis/Rendering.h deleted file mode 100644 index 200c43f94..000000000 --- a/Code/vis/Rendering.h +++ /dev/null @@ -1,61 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RENDERING_H -#define HEMELB_VIS_RENDERING_H - -#include "vis/BasicPixel.h" -#include "vis/PixelSet.h" -#include "vis/rayTracer/RayDataNormal.h" -#include "vis/ResultPixel.h" -#include "vis/streaklineDrawer/StreakPixel.h" - -namespace hemelb -{ - namespace vis - { - /** - * Rendering: A class that acts as the interface between the visualisation controller and the - * drawn renderings from each component drawer. - */ - class Rendering - { - public: - Rendering(PixelSet* glyph, - PixelSet* ray, - PixelSet* streak); - void ReleaseAll(); - - void ReceivePixelCounts(net::Net* inNet, proc_t source); - - void ReceivePixelData(net::Net* inNet, proc_t source); - - void SendPixelCounts(net::Net* inNet, proc_t destination); - - void SendPixelData(net::Net* inNet, proc_t destination); - void Combine(const Rendering& other); - void PopulateResultSet(PixelSet* resultSet); - - private: - template - void AddPixelsToResultSet(PixelSet* resultSet, - const std::vector& inPixels) - { - for (typename std::vector::const_iterator it = inPixels.begin(); it - != inPixels.end(); it++) - { - resultSet ->AddPixel(ResultPixel(&*it)); - } - } - - PixelSet* glyphResult; - PixelSet* rayResult; - PixelSet* streakResult; - }; - } -} - -#endif /* HEMELB_VIS_RENDERING_H */ diff --git a/Code/vis/ResultPixel.cc b/Code/vis/ResultPixel.cc deleted file mode 100644 index 22b570b79..000000000 --- a/Code/vis/ResultPixel.cc +++ /dev/null @@ -1,234 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "vis/ResultPixel.h" - -namespace hemelb -{ - namespace vis - { - ResultPixel::ResultPixel(const BasicPixel* glyph) : - BasicPixel(glyph->GetI(), glyph->GetJ()), hasGlyph(true), normalRayPixel(NULL), streakPixel(NULL) - { - - } - - ResultPixel::ResultPixel(const raytracer::RayDataNormal* ray) : - BasicPixel(ray->GetI(), ray->GetJ()), hasGlyph(false), normalRayPixel(ray), streakPixel(NULL) - { - - } - - ResultPixel::ResultPixel(const streaklinedrawer::StreakPixel* streak) : - BasicPixel(streak->GetI(), streak->GetJ()), hasGlyph(false), normalRayPixel(NULL), streakPixel(streak) - { - - } - - const raytracer::RayDataNormal* ResultPixel::GetRayPixel() const - { - return normalRayPixel; - } - - void ResultPixel::Combine(const ResultPixel& other) - { - if (other.hasGlyph) - { - hasGlyph = true; - } - - if (other.normalRayPixel != NULL) - { - normalRayPixel = other.normalRayPixel; - } - - if (other.streakPixel != NULL) - { - streakPixel = other.streakPixel; - } - } - - void ResultPixel::WritePixel(unsigned* pixel_index, - unsigned char rgb_data[12], - const DomainStats& iDomainStats, - const VisSettings& visSettings) const - { - const int bits_per_char = sizeof(char) * 8; - *pixel_index = (i << (2 * bits_per_char)) + j; - - if (normalRayPixel != NULL) - { - // store velocity volume rendering colour - normalRayPixel->GetVelocityColour(rgb_data, visSettings, iDomainStats); - - float stress = normalRayPixel->GetNearestStress(); - - if (visSettings.mStressType != lb::ShearStress) - { - normalRayPixel->GetStressColour(&rgb_data[3], visSettings, iDomainStats); - } - else if (stress < (float) NO_VALUE) - { - float stress_col[3]; - PickColour(stress, stress_col); - - // store wall shear stress colour - MakePixelColour(int(255.0F * stress_col[0]), - int(255.0F * stress_col[1]), - int(255.0F * stress_col[2]), - &rgb_data[3]); - } - else - { - rgb_data[3] = rgb_data[4] = rgb_data[5] = 0; - } - } - else - { - for (int ii = 0; ii < 6; ++ii) - { - rgb_data[ii] = 255; - } - } - - float density = normalRayPixel == NULL - ? 0.0F - : normalRayPixel->GetNearestDensity(); - float stress = normalRayPixel == NULL - ? 0.0F - : normalRayPixel->GetNearestStress(); - - if (visSettings.mStressType != lb::ShearStress && visSettings.mode == VisSettings::ISOSURFACES) - { - float density_col[3], stress_col[3]; - PickColour(density, density_col); - PickColour(stress, stress_col); - - // store wall pressure colour - MakePixelColour(int(255.0F * density_col[0]), - int(255.0F * density_col[1]), - int(255.0F * density_col[2]), - &rgb_data[6]); - - // store von Mises stress colour - MakePixelColour(int(255.0F * stress_col[0]), - int(255.0F * stress_col[1]), - int(255.0F * stress_col[2]), - &rgb_data[9]); - - } - else if (visSettings.mStressType != lb::ShearStress && visSettings.mode == VisSettings::ISOSURFACESANDGLYPHS) - { - float density_col[3], stress_col[3]; - PickColour(density, density_col); - PickColour(stress, stress_col); - - if (normalRayPixel != NULL) - { - if (!hasGlyph) - { - density_col[0] += 1.0F; - density_col[1] += 1.0F; - density_col[2] += 1.0F; - - stress_col[0] += 1.0F; - stress_col[1] += 1.0F; - stress_col[2] += 1.0F; - } - - // store wall pressure (+glyph) colour - MakePixelColour(int(127.5F * density_col[0]), - int(127.5F * density_col[1]), - int(127.5F * density_col[2]), - &rgb_data[6]); - - // store von Mises stress (+glyph) colour - MakePixelColour(int(127.5F * stress_col[0]), - int(127.5F * stress_col[1]), - int(127.5F * stress_col[2]), - &rgb_data[9]); - } - else - { - for (int ii = 6; ii < 12; ++ii) - { - rgb_data[ii] = 0; - } - } - - } - else if (streakPixel != NULL) - { - float scaled_vel = (float) (streakPixel->GetParticleVelocity() * iDomainStats.velocity_threshold_max_inv); - float particle_col[3]; - PickColour(scaled_vel, particle_col); - - // store particle colour - MakePixelColour(int(255.0F * particle_col[0]), - int(255.0F * particle_col[1]), - int(255.0F * particle_col[2]), - &rgb_data[6]); - - for (int ii = 9; ii < 12; ++ii) - { - rgb_data[ii] = rgb_data[ii - 3]; - } - } - else - { - // store pressure colour - rgb_data[6] = rgb_data[7] = rgb_data[8] - = (unsigned char) util::NumericalFunctions::enforceBounds(int(127.5F * density), 0, 127); - - // store shear stress or von Mises stress - if (stress < ((float) NO_VALUE)) - { - rgb_data[9] = rgb_data[10] = rgb_data[11] - = (unsigned char) util::NumericalFunctions::enforceBounds(int(127.5F * stress), 0, 127); - } - else - { - rgb_data[9] = rgb_data[10] = rgb_data[11] = 0; - } - } - } - - void ResultPixel::PickColour(float value, float colour[3]) - { - colour[0] = util::NumericalFunctions::enforceBounds(4.F * value - 2.F, 0.F, 1.F); - colour[1] = util::NumericalFunctions::enforceBounds(2.F - 4.F * (float) fabs(value - 0.5F), 0.F, 1.F); - colour[2] = util::NumericalFunctions::enforceBounds(2.F - 4.F * value, 0.F, 1.F); - } - - void ResultPixel::MakePixelColour(int rawRed, int rawGreen, int rawBlue, unsigned char* dest) - { - dest[0] = (unsigned char) util::NumericalFunctions::enforceBounds(rawRed, 0, 255); - dest[1] = (unsigned char) util::NumericalFunctions::enforceBounds(rawGreen, 0, 255); - dest[2] = (unsigned char) util::NumericalFunctions::enforceBounds(rawBlue, 0, 255); - } - - void ResultPixel::LogDebuggingInformation() const - { - log::Logger::Log("Pixel at (%i,%i) with (ray,streak,glyph)=(%i,%i,%i)", - GetI(), - GetJ(), - normalRayPixel != NULL, - streakPixel != NULL, - hasGlyph); - - if (normalRayPixel != NULL) - { - normalRayPixel->LogDebuggingInformation(); - } - - if (streakPixel != NULL) - { - streakPixel->LogDebuggingInformation(); - } - } - - } -} diff --git a/Code/vis/ResultPixel.h b/Code/vis/ResultPixel.h deleted file mode 100644 index bd6082c60..000000000 --- a/Code/vis/ResultPixel.h +++ /dev/null @@ -1,59 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RESULTPIXEL_H -#define HEMELB_VIS_RESULTPIXEL_H - -#include - -#include "util/utilityFunctions.h" -#include "vis/BasicPixel.h" -#include "vis/rayTracer/RayDataNormal.h" -#include "vis/streaklineDrawer/StreakPixel.h" -#include "vis/VisSettings.h" -#include "vis/DomainStats.h" - -namespace hemelb -{ - namespace vis - { - class ResultPixel : public BasicPixel - { - public: - ResultPixel(const BasicPixel* glyph); - - ResultPixel(const raytracer::RayDataNormal* ray); - - ResultPixel(const streaklinedrawer::StreakPixel* streak); - - const raytracer::RayDataNormal* GetRayPixel() const; - - void Combine(const ResultPixel& other); - - void WritePixel(unsigned* pixel_index, - unsigned char rgb_data[12], - const DomainStats& iDomainStats, - const VisSettings& visSettings) const; - - /* - * Debugging function to output details about the pixel to the logger. - */ - void LogDebuggingInformation() const; - - private: - - static void PickColour(float value, float colour[3]); - - static void MakePixelColour(int rawRed, int rawGreen, int rawBlue, unsigned char* dest); - - bool hasGlyph; - const raytracer::RayDataNormal* normalRayPixel; - const streaklinedrawer::StreakPixel* streakPixel; - }; - } -} - -#endif /* HEMELB_VIS_RESULTPIXEL_H */ diff --git a/Code/vis/Screen.cc b/Code/vis/Screen.cc deleted file mode 100644 index 6ec4fed66..000000000 --- a/Code/vis/Screen.cc +++ /dev/null @@ -1,103 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/IOCommunicator.h" -#include "util/utilityFunctions.h" -#include "vis/Screen.h" - -namespace hemelb -{ - namespace vis - { - - Screen::Screen() - { - } - - Screen::~Screen() - { - } - - void Screen::Set(float maxX, - float maxY, - int pixelsX, - int pixelsY, - float rad, - const Viewpoint* viewpoint) - { - MaxXValue = maxX; - MaxYValue = maxY; - - mPixelUnitVectorProjectionX - = viewpoint->RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(MaxXValue, - 0.0F, - 0.0F)); - mPixelUnitVectorProjectionY - = viewpoint-> RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(0.0F, - MaxYValue, - 0.0F)); - - Resize(pixelsX, pixelsY); - - mPixelsPerUnitX = (float) GetPixelsX() / (2.F * MaxXValue); - mPixelsPerUnitY = (float) GetPixelsY() / (2.F * MaxYValue); - - util::Vector3D - lCameraToLocalCentreVector = - viewpoint->RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(0.F, - 0.F, - -viewpoint->GetDistanceFromCameraToScreen())); - - util::Vector3D lMiddleCentreToMiddleRightOfScreen = - viewpoint->RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(MaxXValue, - 0.0F, - 0.0F)); - - util::Vector3D lLowerCentreToTopCentreOfScreen = - viewpoint->RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(0.0F, - MaxYValue, - 0.0F)); - - mCameraToBottomLeftOfScreen = (lCameraToLocalCentreVector - - lMiddleCentreToMiddleRightOfScreen) - lLowerCentreToTopCentreOfScreen; - - mPixelUnitVectorProjectionX = lMiddleCentreToMiddleRightOfScreen * (2.F - / (float) GetPixelsX()); - - mPixelUnitVectorProjectionY = lLowerCentreToTopCentreOfScreen * (2.F / (float) GetPixelsY()); - } - - void Screen::Resize(unsigned int newPixelsX, unsigned int newPixelsY) - { - if (newPixelsX * newPixelsY <= COLOURED_PIXELS_MAX) - { - xPixels = newPixelsX; - yPixels = newPixelsY; - } - } - - const util::Vector3D& Screen::GetCameraToBottomLeftOfScreenVector() const - { - return mCameraToBottomLeftOfScreen; - } - const util::Vector3D& Screen::GetPixelUnitVectorProjectionX() const - { - return mPixelUnitVectorProjectionX; - } - const util::Vector3D& Screen::GetPixelUnitVectorProjectionY() const - { - return mPixelUnitVectorProjectionY; - } - int Screen::GetPixelsX() const - { - return xPixels; - } - int Screen::GetPixelsY() const - { - return yPixels; - } - } -} diff --git a/Code/vis/Screen.h b/Code/vis/Screen.h deleted file mode 100644 index 71624d40e..000000000 --- a/Code/vis/Screen.h +++ /dev/null @@ -1,78 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_SCREEN_H -#define HEMELB_VIS_SCREEN_H - -#include "io/writers/Writer.h" -#include "util/Vector3D.h" -#include "vis/Viewpoint.h" -#include "vis/VisSettings.h" - -namespace hemelb -{ - namespace vis - { - class Screen - { - public: - static const unsigned int COLOURED_PIXELS_MAX = 2048 * 2048; - - Screen(); - ~Screen(); - - void Set(float maxX, - float maxY, - int pixelsX, - int pixelsY, - float rad, - const Viewpoint* viewpoint); - - void Resize(unsigned int pixelsX, unsigned int pixelsY); - - /** - * Does a transform from input array into output array. This function - * will still work if the two arrays point to the same location in - * memory. It only operates on the first two elements of input. - * - * @param input - * @param output - */ - template - XYCoordinates TransformScreenToPixelCoordinates(const XYCoordinates& iXYIn) const - { - return XYCoordinates (static_cast (mPixelsPerUnitX * (iXYIn.x + MaxXValue)), - static_cast (mPixelsPerUnitY * (iXYIn.y + MaxYValue))); - } - - const util::Vector3D& GetCameraToBottomLeftOfScreenVector() const; - const util::Vector3D& GetPixelUnitVectorProjectionX() const; - const util::Vector3D& GetPixelUnitVectorProjectionY() const; - int GetPixelsX() const; - int GetPixelsY() const; - - private: - int xPixels, yPixels; - - //The number of pixels per unit of X or Y in screen coordinates - float mPixelsPerUnitX; - float mPixelsPerUnitY; - - //The extent of the screen in screen coordintes - //(from -MaxXValue to MaxXValue) - float MaxXValue; - float MaxYValue; - - util::Vector3D mCameraToBottomLeftOfScreen; - - // Projection of unit vectors along screen axes into normal space. - util::Vector3D mPixelUnitVectorProjectionX; - util::Vector3D mPixelUnitVectorProjectionY; - }; - } -} - -#endif /* HEMELB_VIS_SCREEN_H */ diff --git a/Code/vis/Viewpoint.cc b/Code/vis/Viewpoint.cc deleted file mode 100644 index e58e27332..000000000 --- a/Code/vis/Viewpoint.cc +++ /dev/null @@ -1,155 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include - -#include "log/Logger.h" -#include "util/Vector3D.h" -#include "vis/Viewpoint.h" -#include "vis/XYCoordinates.h" - -namespace hemelb -{ - namespace vis - { - Viewpoint::Viewpoint() : - mViewpointLocationInWorldCoordinates(0.0F) - { - } - - util::Vector3D Viewpoint::RotateCameraCoordinatesToWorldCoordinates(const util::Vector3D< - float>& iVector) const - { - // A rotation of iThetaX clockwise looking up the x-axis (increasing) - // Followed by a rotation of iThetaY anticlockwise looking up the y-axis. - return Rotate(mSinLatitude, mCosLatitude, mSinLongitude, mCosLongitude, iVector); - } - - util::Vector3D Viewpoint::RotateWorldToCameraCoordinates(const util::Vector3D& iVector) const - { - // The reverse of the above - return UnRotate(mSinLatitude, mCosLatitude, mSinLongitude, mCosLongitude, iVector); - } - - util::Vector3D Viewpoint::Rotate(float iSinThetaX, - float iCosThetaX, - float iSinThetaY, - float iCosThetaY, - const util::Vector3D& iVector) - { - // A rotation of iThetaX clockwise looking down the x-axis - // Followed by a rotation of iThetaY anticlockwise looking down the y-axis. - // In matrices: - // (cos(iThetaY) 0 sin(iThetaY)) (1 0 0 ) - // Out = (0 1 0 ) (0 cos(-iThetaX) -sin(-iThetaX)) In - // (-sin(iThetaY) 0 cos(iThetaY)) (0 sin(-iThetaX) cos(-iThetaX) ) - // - // (Xcos(iThetaY) + Zsin(iThetaY)cos(iThetaX) - Ysin(iThetaY)sin(iThetaX)) - // Out = (Ycos(iThetaX) + Zsin(iThetaX) ) - // (Zcos(iThetaX)cos(iThetaY) - Ysin(iThetaX)cos(iThetaY) - Xsin(iThetaY)) - - const float lTemp = iVector.z * iCosThetaX - iVector.y * iSinThetaX; - - return util::Vector3D(lTemp * iSinThetaY + iVector.x * iCosThetaY, - iVector.z * iSinThetaX + iVector.y * iCosThetaX, - lTemp * iCosThetaY - iVector.x * iSinThetaY); - } - - util::Vector3D Viewpoint::UnRotate(float iSinThetaX, - float iCosThetaX, - float iSinThetaY, - float iCosThetaY, - const util::Vector3D& iVector) - { - // A rotation of iThetaY aniclockwise looking down the y-axis - // Followed by a rotation of iThetaX clockwise looking down the x-axis. - // In matrices: - // (1 0 0 )(cos(-iThetaY) 0 sin(-iThetaY)) - // Out = (0 cos(iThetaX) -sin(iThetaX))(0 1 0 ) In - // (0 sin(iThetaX) cos(iThetaX) )(-sin(-iThetaY) 0 cos(-iThetaY)) - // - // This is the Rotation matrix inversed / transposted - // ie (AB)^-1 = (AB)^t = B^t A^T - - const float lTemp = iVector.x * iSinThetaY + iVector.z * iCosThetaY; - - return util::Vector3D(iVector.x * iCosThetaY - iVector.z * iSinThetaY, - -lTemp * iSinThetaX + iVector.y * iCosThetaX, - lTemp * iCosThetaX + iVector.y * iSinThetaX); - } - - util::Vector3D Viewpoint::Project(const util::Vector3D& iWorldLocation) const - { - util::Vector3D lLocationCamCoordinates = - GetLocationInCameraCoordinates(iWorldLocation); - - //Carry out a perspective projection on an infinite spanning screen - //between the camera and the subject. - //Reverse the sign such that depth is positive (I believe). - return util::Vector3D(mDistanceFromCameraToScreen / (-lLocationCamCoordinates.z) - * lLocationCamCoordinates.x, - mDistanceFromCameraToScreen / (-lLocationCamCoordinates.z) - * lLocationCamCoordinates.y, - -lLocationCamCoordinates.z); - } - - XYCoordinates Viewpoint::FlatProject(const util::Vector3D& iWorldLocation) const - { - util::Vector3D lLocationCamCoordinates = - GetLocationInCameraCoordinates(iWorldLocation); - - return XYCoordinates (mDistanceFromCameraToScreen / (-lLocationCamCoordinates.z) - * lLocationCamCoordinates.x, - mDistanceFromCameraToScreen / (-lLocationCamCoordinates.z) - * lLocationCamCoordinates.y); - } - - void Viewpoint::SetViewpointPosition(float iLongitude, - float iLatitude, - const util::Vector3D& iLocalCentre, - float iRadius, - float iDistanceFromCameraToScreen) - { - - mSinLongitude = sinf(iLongitude); - mCosLongitude = cosf(iLongitude); - - mSinLatitude = sinf(iLatitude); - mCosLatitude = cosf(iLatitude); - - //The camera is located at 0,0,radius from the world centre in camera co-ordinates - mViewpointLocationInWorldCoordinates - = RotateCameraCoordinatesToWorldCoordinates(util::Vector3D(0., 0., iRadius)); - - //Translate the camera location to allow it to point at - //a local centre rather than the world centre - mViewpointLocationInWorldCoordinates += iLocalCentre; - - mDistanceFromCameraToScreen = iDistanceFromCameraToScreen; - } - - const util::Vector3D& Viewpoint::GetViewpointLocation() const - { - return mViewpointLocationInWorldCoordinates; - } - - float Viewpoint::GetDistanceFromCameraToScreen() const - { - return mDistanceFromCameraToScreen; - } - - util::Vector3D Viewpoint::GetLocationInCameraCoordinates(const util::Vector3D& iWorldLocation) const - { - //Calculate the location of the point relative to the viewpoint - util::Vector3D lLocationRelativeToViewPoint = iWorldLocation - - mViewpointLocationInWorldCoordinates; - - //Rotate the location vector in the opposite manner to that of the camera - return RotateWorldToCameraCoordinates(lLocationRelativeToViewPoint); - } - - } -} diff --git a/Code/vis/Viewpoint.h b/Code/vis/Viewpoint.h deleted file mode 100644 index 4697ee15f..000000000 --- a/Code/vis/Viewpoint.h +++ /dev/null @@ -1,109 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_VIEWPOINT_H -#define HEMELB_VIS_VIEWPOINT_H - -#include "util/Vector3D.h" -#include "vis/XYCoordinates.h" - -namespace hemelb -{ - namespace vis - { - /** - * Holds the position of the viewpoint or camera location and performs - * projection of points in world coordinates into 3D screen co-ordinates - */ - class Viewpoint - { - //See http://en.wikipedia.org/wiki/3D_projection#Perspective_projection - public: - Viewpoint(); - - /** - * Changes a location in camera coordinates into world coordinates - * Note that both co-ordinate systems share the same (0,0,0) point - * and the camera is at (0,0,radius) in camera coordinates - * - */ - util::Vector3D RotateCameraCoordinatesToWorldCoordinates(const util::Vector3D& iVector) const; - - /** - * Does the reverse of the above - * - */ - util::Vector3D RotateWorldToCameraCoordinates(const util::Vector3D& iVector) const; - - /** - * Projects a location in world coordinates onto the infinite screen, - * by translating and rotating such as to give coordinates relative - * to the camera, and then applying a perspective projection - */ - util::Vector3D Project(const util::Vector3D& p1) const; - - /** - * Same as project but doesn't return a z component - * - */ - XYCoordinates FlatProject(const util::Vector3D& iWorldLocation) const; - - /** - * Sets the position of the Camera or Viewpoint - * - * In world coordinates, the camera is located at radius from - * the local centre, and pointing at an angle indicated by the - * latitude and longitude. - * - * @param iLongitude - longitude in radians. - * @param iLatitude - latitude in radians. - * @param iLocalCentre - where the camera should point in world-coordinates - * @param iRadius - the distance of the camera from this local centre - * @param iDistanceFromCameraToScreen - the distance of the infinite screen - * from the viewer. Allows for zoom - */ - void SetViewpointPosition(float iLongitude, - float iLatitude, - const util::Vector3D& iLocalCentre, - float iRadius, - float iDistanceFromCameraToScreen); - - const util::Vector3D& GetViewpointLocation() const; - - float GetDistanceFromCameraToScreen() const; - - private: - util::Vector3D GetLocationInCameraCoordinates(const util::Vector3D& iWorldLocation) const; - - //Performs a vector rotation using stored - //Sin and Cosine Values - static util::Vector3D Rotate(float iSinThetaX, - float iCosThetaX, - float iSinThetaY, - float iCosThetaY, - const util::Vector3D& iVector); - - //Reverses a vector rotation of the above - static util::Vector3D UnRotate(float iSinThetaX, - float iCosThetaX, - float iSinThetaY, - float iCosThetaY, - const util::Vector3D& iVector); - - float mSinLongitude; - float mCosLongitude; - float mSinLatitude; - float mCosLatitude; - - //Stores the viewpoint Location in world co-ordinate - util::Vector3D mViewpointLocationInWorldCoordinates; - - float mDistanceFromCameraToScreen; - }; - } -} - -#endif /* HEMELB_VIS_VIEWPOINT_H */ diff --git a/Code/vis/VisSettings.h b/Code/vis/VisSettings.h deleted file mode 100644 index 0f7a0ae8a..000000000 --- a/Code/vis/VisSettings.h +++ /dev/null @@ -1,48 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_VISSETTINGS_H -#define HEMELB_VIS_VISSETTINGS_H - -#include "lb/LbmParameters.h" - -namespace hemelb -{ - namespace vis - { - struct VisSettings - { - enum Mode - { - // 0 - Only display the isosurfaces (wall pressure and stress) - ISOSURFACES = 0, - // 1 - Isosurface and glyphs - ISOSURFACESANDGLYPHS = 1, - // 2 - Wall pattern streak lines - WALLANDSTREAKLINES = 2 - }; - - // better public member vars than globals! - Mode mode; - - float ctr_x, ctr_y, ctr_z; - float streaklines_per_simulation, streakline_length; - double mouse_pressure, mouse_stress; - float brightness; - float glyphLength; - - //Maximum distance - used in the enhanced ray tracer to handle - //dept cuing - float maximumDrawDistance; - - lb::StressTypes mStressType; - - int mouse_x, mouse_y; - }; - } -} - -#endif /* HEMELB_VIS_VISSETTINGS_H */ diff --git a/Code/vis/XYCoordinates.h b/Code/vis/XYCoordinates.h deleted file mode 100644 index 245c3f2d1..000000000 --- a/Code/vis/XYCoordinates.h +++ /dev/null @@ -1,137 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_XYCOORDINATES_H -#define HEMELB_VIS_XYCOORDINATES_H - -#include "constants.h" -#include "util/utilityFunctions.h" - -namespace hemelb -{ - namespace vis - { - template - class XYCoordinates - { - public: - T x, y; - - XYCoordinates() - { - } - ; - - XYCoordinates(T iX, T iY) : - x(iX), y(iY) - { - } - - XYCoordinates(T iN) : - x(iN), y(iN) - { - } - - //Copy constructor - can be used to perform type converstion - template - XYCoordinates(const XYCoordinates & iOldXYCoordinates) - { - x = static_cast(iOldXYCoordinates.x); - y = static_cast(iOldXYCoordinates.y); - } - - //Equality - bool operator==(const XYCoordinates right) - { - if (x != right.x) - { - return false; - } - if (y != right.y) - { - return false; - } - return true; - } - - //Vector addition - XYCoordinates operator+(const XYCoordinates right) const - { - return XYCoordinates(x + right.x, y + right.y); - } - - //Vector addition - XYCoordinates& operator+=(const XYCoordinates right) - { - x += right.x; - y += right.y; - - return *this; - } - - //Vector subtraction - XYCoordinates operator-(const XYCoordinates right) const - { - return XYCoordinates(x - right.x, y - right.y); - } - - //Scalar multiplication - template - XYCoordinates operator*(const MultiplierT multiplier) const - { - return XYCoordinates(x * multiplier, y * multiplier); - } - - //Updates the XYCoordinates with the smallest of each - //of the x and y co-ordinatess independently of both XYCoordinates - void UpdatePointwiseMin(const XYCoordinates& iCompareLocation) - { - x = util::NumericalFunctions::min(x, iCompareLocation.x); - - y = util::NumericalFunctions::min(y, iCompareLocation.y); - } - - //Updates the XYCoordinates with the largest of each - //of the x and y co-ordinates independently of both XYCoordinates - void UpdatePointwiseMax(const XYCoordinates& iCompareLocation) - { - x = util::NumericalFunctions::max(x, iCompareLocation.x); - - y = util::NumericalFunctions::max(y, iCompareLocation.y); - } - - static XYCoordinates MaxLimit() - { - return XYCoordinates(std::numeric_limits::max()); - } - - static XYCoordinates MinLimit() - { - return XYCoordinates(std::numeric_limits::min()); - } - }; - - template - XYCoordinates operator+(const XYCoordinates left, const XYCoordinates right) - { - return left + right; - } - - template - XYCoordinates operator-(const XYCoordinates left, const XYCoordinates right) - { - return left - right; - } - - template - XYCoordinates operator*(const TLeft left, const XYCoordinates right) - { - return right * left; - } - } -} - -#endif // HEMELB_VIS_XYCOORDINATES_H diff --git a/Code/vis/rayTracer/Cluster.h b/Code/vis/rayTracer/Cluster.h deleted file mode 100644 index 51122635a..000000000 --- a/Code/vis/rayTracer/Cluster.h +++ /dev/null @@ -1,168 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTER_H -#define HEMELB_VIS_RAYTRACER_CLUSTER_H - -#include - -#include "util/Vector3D.h" - -#include "vis/rayTracer/SiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - //The cluster structure stores data relating to the clusters - //used by the RayTracer, in an optimal format - //Cluster are produced by the ClusterSharedFactory - //Caution: the data within the flow field is altered by means - //of pointers obtained from the GetClusterSharedVoxelDataPointer - //method - template - class Cluster - { - public: - Cluster(unsigned short xBlockCount, - unsigned short yBlockCount, - unsigned short zBlockCount, - const util::Vector3D& minimalSite, - const util::Vector3D& maximalSite, - const util::Vector3D& minimalSiteOnMinimalBlock, - const util::Vector3D& minimalBlock) : - blocksX(xBlockCount), blocksY(yBlockCount), blocksZ(zBlockCount), minSite(minimalSite), maxSite(maximalSite), leastSiteOnLeastBlockInImage(minimalSiteOnMinimalBlock), minBlock(minimalBlock) - { - } - - unsigned int GetBlockIdFrom3DBlockLocation(const util::Vector3D& iLocation) const - { - return iLocation.x * blocksY * blocksZ + iLocation.y * blocksZ + iLocation.z; - } - - const util::Vector3D* GetWallData(site_t iBlockNumber, site_t iSiteNumber) const - { - return ((const Derived*) (this))->DoGetWallData(iBlockNumber, iSiteNumber); - } - - void SetWallData(site_t iBlockNumber, site_t iSiteNumber, const util::Vector3D& iData) - { - return ((Derived*) (this))->DoSetWallData(iBlockNumber, iSiteNumber, iData); - } - - static bool NeedsWallNormals() - { - return Derived::DoNeedsWallNormals(); - } - - const std::vector > GetCorners() const - { - std::vector > lCorners; - - lCorners.push_back(util::Vector3D(minSite.x, minSite.y, minSite.z)); - - lCorners.push_back(util::Vector3D(minSite.x, minSite.y, maxSite.z)); - - lCorners.push_back(util::Vector3D(minSite.x, maxSite.y, minSite.z)); - - lCorners.push_back(util::Vector3D(minSite.x, maxSite.y, maxSite.z)); - - lCorners.push_back(util::Vector3D(maxSite.x, minSite.y, minSite.z)); - - lCorners.push_back(util::Vector3D(maxSite.x, minSite.y, maxSite.z)); - - lCorners.push_back(util::Vector3D(maxSite.x, maxSite.y, minSite.z)); - - lCorners.push_back(util::Vector3D(maxSite.x, maxSite.y, maxSite.z)); - - return lCorners; - } - - /** - * True if the cluster type requires wall normals. - * - * This can be overridden by deriving classes. - * @return - */ - static bool DoNeedsWallNormals() - { - return false; - } - - unsigned short GetBlocksX() const - { - return blocksX; - } - - unsigned short GetBlocksY() const - { - return blocksY; - } - - unsigned short GetBlocksZ() const - { - return blocksZ; - } - - const util::Vector3D& GetMinSite() const - { - return minSite; - } - - const util::Vector3D& GetMaxSite() const - { - return maxSite; - } - - const util::Vector3D& GetLeastSiteOnLeastBlockInImage() const - { - return leastSiteOnLeastBlockInImage; - } - - /** - * Returns the block coordinates of the block with minimal x, y and z - * coordinates in the cluster. - * - * @return The minimal block coordinates - */ - const util::Vector3D& GetMinBlockLocation() const - { - return minBlock; - } - - private: - /** - * The size of the cluster in terms of the number of blocks - */ - unsigned short blocksX; - unsigned short blocksY; - unsigned short blocksZ; - - /** - * The min and maximum site location, in site units - * relative to the centre of the lattice - */ - util::Vector3D minSite; - util::Vector3D maxSite; - - /** - * The lowest x, y and z block location of the ClusterShared - * in terms of site units relative to the centre location - */ - util::Vector3D leastSiteOnLeastBlockInImage; - - /** - * The coordinates of the block with minimal x, y and z components. - */ - util::Vector3D minBlock; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTER_H diff --git a/Code/vis/rayTracer/ClusterBuilder.h b/Code/vis/rayTracer/ClusterBuilder.h deleted file mode 100644 index 8a441469f..000000000 --- a/Code/vis/rayTracer/ClusterBuilder.h +++ /dev/null @@ -1,340 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTERBUILDER_H -#define HEMELB_VIS_RAYTRACER_CLUSTERBUILDER_H - -#include -#include - -#include "geometry/BlockTraverserWithVisitedBlockTracker.h" -#include "geometry/LatticeData.h" -#include "geometry/SiteTraverser.h" -#include "lb/LbmParameters.h" -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" -#include "vis/rayTracer/ClusterBuilder.h" -#include "vis/rayTracer/ClusterTraverser.h" -#include "vis/rayTracer/RayTracer.h" -#include "vis/rayTracer/SiteData.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - template - class ClusterBuilder - { - public: - ClusterBuilder(const geometry::LatticeData* latticeData, int localRank_) : - mBlockTraverser(*latticeData), localRank(localRank_) - { - mLatticeData = latticeData; - - //Each block is assigned a cluster id once it has been - //assigned to a cluster - mClusterIdOfBlock = new short int[mLatticeData->GetBlockCount()]; - for (site_t lId = 0; lId < mLatticeData->GetBlockCount(); lId++) - { - mClusterIdOfBlock[lId] = NOTASSIGNEDTOCLUSTER; - } - } - - ~ClusterBuilder() - { - delete[] mClusterIdOfBlock; - } - - void BuildClusters() - { - //Initially locate all clusters, locating their - //range by block span and site span - LocateClusters(); - - // Process the flow field for every cluster - for (unsigned int lThisClusterId = 0; lThisClusterId < mClusters.size(); lThisClusterId++) - { - ProcessCluster(lThisClusterId); - } - } - - std::vector& GetClusters() - { - return mClusters; - } - - private: - // Locates all the clusters in the lattice structure and the - void LocateClusters() - { - // Run through all unvisited blocks finding clusters - do - { - //Mark the block visited - mBlockTraverser.MarkCurrentBlockVisited(); - - //If there are sites assigned to the local processor, search for the - //cluster of connected sites - if (AreSitesAssignedToLocalProcessorRankInBlock(mBlockTraverser.GetCurrentBlockData())) - { - FindNewCluster(); - } - } - while (mBlockTraverser.GoToNextUnvisitedBlock()); - } - - //Locates all the clusters in the lattice structure and stores - //their locations - void FindNewCluster() - { - //These locations will eventually contain the bounds of the - //rectangular cluster, both in terms of block number and - //site numbers - util::Vector3D clusterBlockMin = util::Vector3D::MaxLimit(); - util::Vector3D clusterBlockMax = util::Vector3D::MinLimit(); - util::Vector3D clusterSiteMin = util::Vector3D::MaxLimit(); - util::Vector3D clusterSiteMax = util::Vector3D::MinLimit(); - - //To discover the cluster, we continually visit the neighbours - //of sequential blocks - //We keep a stack of all the sites that must be processed - //and sequentially add neighbours to it - std::stack > blocksToProcess; - - //Set up the initial condition - blocksToProcess.push(mBlockTraverser.GetCurrentLocation()); - - //Loop over the cluster via neighbours until - //all blocks have been processed - while (!blocksToProcess.empty()) - { - //Get location off the top of the stack - //(we could actually take anything off the stack) - util::Vector3D lCurrentLocation = blocksToProcess.top(); - blocksToProcess.pop(); - - if (AreSitesAssignedToLocalProcessorRankInBlock(mBlockTraverser.GetBlockDataForLocation(lCurrentLocation))) - { - //Update block range of the cluster - clusterBlockMin.UpdatePointwiseMin(lCurrentLocation); - clusterBlockMax.UpdatePointwiseMax(lCurrentLocation); - - //Update the cluster id of the given block - site_t blockId = mBlockTraverser.GetIndexFromLocation(lCurrentLocation); - mClusterIdOfBlock[blockId] = (short int) mClusters.size(); - - //Loop through all the sites on the block, to - //update the site bounds on the cluster - geometry::SiteTraverser siteTraverser = mBlockTraverser.GetSiteTraverser(); - do - { - //If the site is not a solid - if (!mBlockTraverser.GetBlockDataForLocation(lCurrentLocation).SiteIsSolid(siteTraverser.GetCurrentIndex())) - { - clusterSiteMin.UpdatePointwiseMin(siteTraverser.GetCurrentLocation() - + lCurrentLocation * mBlockTraverser.GetBlockSize()); - - clusterSiteMax.UpdatePointwiseMax(siteTraverser.GetCurrentLocation() - + lCurrentLocation * mBlockTraverser.GetBlockSize()); - } - } - while (siteTraverser.TraverseOne()); - - //Check all the neighbouring blocks to see if they need visiting. Add them to the stack. - AddNeighbouringBlocks(lCurrentLocation, blocksToProcess); - } - } - - AddCluster(clusterBlockMin, clusterBlockMax, clusterSiteMin, clusterSiteMax); - } - - //Adds neighbouring blocks of the input location to the input stack - void AddNeighbouringBlocks(util::Vector3D iCurrentLocation, - std::stack >& oBlocksToProcess) - { - // Loop over all neighbouring blocks - for (int l = 0; l < 26; l++) - { - util::Vector3D lNeighbouringBlock = iCurrentLocation + mNeighbours[l]; - - //The neighouring block location might not exist - //eg negative co-ordinates - if (mBlockTraverser.IsValidLocation(lNeighbouringBlock)) - { - //Ensure that the block hasn't been visited before - if (!mBlockTraverser.IsBlockVisited(lNeighbouringBlock)) - { - //Add to the stack - oBlocksToProcess.push(lNeighbouringBlock); - - //We must mark this locatoin as visited so it only - //gets processed once - mBlockTraverser.MarkBlockVisited(lNeighbouringBlock); - } - } - } - } - - //Returns true if there are sites in the given block associated with the - //local processor rank - bool AreSitesAssignedToLocalProcessorRankInBlock(const geometry::Block& block) - { - if (block.IsEmpty()) - { - return false; - } - - for (site_t siteId = 0; siteId < mLatticeData->GetSitesPerBlockVolumeUnit(); siteId++) - { - if (localRank == block.GetProcessorRankForSite(siteId)) - { - return true; - } - } - return false; - } - - //Adds a new cluster by taking in the required data in interger format - //and converting it to that used by the raytracer - //NB: Futher processing is required on the cluster before it can be used - //by the ray tracer, which is handled by the ProcessCluster method - void AddCluster(util::Vector3D clusterBlockMin, - util::Vector3D clusterBlockMax, - util::Vector3D clusterVoxelMin, - util::Vector3D clusterVoxelMax) - { - const util::Vector3D halfLatticeSiteCount = util::Vector3D(mLatticeData->GetSiteDimensions()) - * 0.5F; - - //The friendly locations must be turned into a format usable by the ray tracer - ClusterType lNewCluster((unsigned short) (1 + clusterBlockMax.x - clusterBlockMin.x), - (unsigned short) (1 + clusterBlockMax.y - clusterBlockMin.y), - (unsigned short) (1 + clusterBlockMax.z - clusterBlockMin.z), - util::Vector3D(clusterVoxelMin) - halfLatticeSiteCount, - util::Vector3D(clusterVoxelMax + util::Vector3D(1)) - - halfLatticeSiteCount, - util::Vector3D(clusterBlockMin * mLatticeData->GetBlockSize()) - - halfLatticeSiteCount, - clusterBlockMin); - - mClusters.push_back(lNewCluster); - - //We need to store the cluster block minimum in - //order to process the cluster - mClusterBlockMins.push_back(clusterBlockMin); - } - - //Adds "flow-field" data to the cluster - void ProcessCluster(unsigned int clusterId) - { - ClusterType& cluster = mClusters[clusterId]; - - ClusterTraverser clusterTraverser(cluster); - - do - { - util::Vector3D blockCoordinates = clusterTraverser.GetCurrentLocation() - + mClusterBlockMins[clusterId]; - - site_t blockId = mLatticeData->GetBlockIdFromBlockCoords(blockCoordinates); - - if (mClusterIdOfBlock[blockId] == (short) clusterId) - { - UpdateSiteData(blockId, clusterTraverser.GetCurrentIndex(), cluster); - } - } - while (clusterTraverser.TraverseOne()); - } - - void UpdateSiteData(site_t blockId, site_t blockIndexWithinCluster, ClusterType& cluster) - { - geometry::SiteTraverser siteTraverser(*mLatticeData); - do - { - UpdateSiteDataAtSite(blockId, blockIndexWithinCluster, cluster, siteTraverser.GetCurrentIndex()); - } - while (siteTraverser.TraverseOne()); - } - - virtual void UpdateSiteDataAtSite(site_t blockId, - site_t blockIndexWithinCluster, - ClusterType& cluster, - site_t siteIdOnBlock) - { - const geometry::Block& block = mLatticeData->GetBlock(blockId); - - //If site not a solid and on the current processor [net.cc] - if (!block.SiteIsSolid(siteIdOnBlock)) - { - if (ClusterType::NeedsWallNormals()) - { - UpdateWallNormalAtSite(block, blockIndexWithinCluster, cluster, siteIdOnBlock); - } - } - } - - void UpdateWallNormalAtSite(const geometry::Block& block, - site_t blockNum, - ClusterType& cluster, - site_t siteIdOnBlock) - { - site_t localIndex = block.GetLocalContiguousIndexForSite(siteIdOnBlock); - - const geometry::Site site = mLatticeData->GetSite(localIndex); - - if (site.IsWall()) - { - cluster.SetWallData(blockNum, siteIdOnBlock, site.GetWallNormal()); - } - } - - //Caution: the data within mClusters is altered by means - //of pointers obtained from the GetClusterVoxelDataPointer - //method. No insertion or copying must therefore take place - //on mClusters once building is complete - std::vector mClusters; - - const geometry::LatticeData* mLatticeData; - - geometry::BlockTraverserWithVisitedBlockTracker mBlockTraverser; - - std::vector > mClusterBlockMins; - - short int *mClusterIdOfBlock; - - int localRank; - - static const short int NOTASSIGNEDTOCLUSTER = -1; - - static const util::Vector3D mNeighbours[26]; - } - ; - - template - const util::Vector3D ClusterBuilder::mNeighbours[26] = { - util::Vector3D(-1, -1, -1), util::Vector3D(-1, -1, 0), util::Vector3D(-1, -1, 1), - util::Vector3D(-1, 0, -1), util::Vector3D(-1, 0, 0), util::Vector3D(-1, 0, 1), - util::Vector3D(-1, 1, -1), util::Vector3D(-1, 1, 0), - util::Vector3D(-1, 1, 1), - util::Vector3D(0, -1, -1), - util::Vector3D(0, -1, 0), - util::Vector3D(0, -1, 1), - util::Vector3D(0, 0, -1), - // 0 0 0 is same site - util::Vector3D(0, 0, 1), util::Vector3D(0, 1, -1), util::Vector3D(0, 1, 0), - util::Vector3D(0, 1, 1), util::Vector3D(1, -1, -1), util::Vector3D(1, -1, 0), - util::Vector3D(1, -1, 1), util::Vector3D(1, 0, -1), util::Vector3D(1, 0, 0), - util::Vector3D(1, 0, 1), util::Vector3D(1, 1, -1), util::Vector3D(1, 1, 0), - util::Vector3D(1, 1, 1) }; - - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTERBUILDER_H diff --git a/Code/vis/rayTracer/ClusterNormal.cc b/Code/vis/rayTracer/ClusterNormal.cc deleted file mode 100644 index 1bb3eedc0..000000000 --- a/Code/vis/rayTracer/ClusterNormal.cc +++ /dev/null @@ -1,44 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "geometry/LatticeData.h" -#include "vis/rayTracer/ClusterNormal.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - ClusterNormal::ClusterNormal(unsigned short xBlockCount, - unsigned short yBlockCount, - unsigned short zBlockCount, - const util::Vector3D& minimalSite, - const util::Vector3D& maximalSite, - const util::Vector3D& minimalSiteOnMinimalBlock, - const util::Vector3D& minimalBlock) : - Cluster(xBlockCount, - yBlockCount, - zBlockCount, - minimalSite, - maximalSite, - minimalSiteOnMinimalBlock, - minimalBlock) - { - } - - const util::Vector3D* ClusterNormal::DoGetWallData(site_t iBlockNumber, site_t iSiteNumber) const - { - return NULL; - } - - void ClusterNormal::DoSetWallData(site_t iBlockNumber, site_t iSiteNumber, const util::Vector3D& iData) - { - } - - } - } -} diff --git a/Code/vis/rayTracer/ClusterNormal.h b/Code/vis/rayTracer/ClusterNormal.h deleted file mode 100644 index a9855142b..000000000 --- a/Code/vis/rayTracer/ClusterNormal.h +++ /dev/null @@ -1,39 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTERNORMAL_H -#define HEMELB_VIS_RAYTRACER_CLUSTERNORMAL_H - -#include "vis/rayTracer/Cluster.h" -#include "vis/rayTracer/SiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - class ClusterNormal : public Cluster - { - public: - ClusterNormal(unsigned short xBlockCount, - unsigned short yBlockCount, - unsigned short zBlockCount, - const util::Vector3D& minimalSite, - const util::Vector3D& maximalSite, - const util::Vector3D& minimalSiteOnMinimalBlock, - const util::Vector3D& minimalBlock); - - const util::Vector3D* DoGetWallData(site_t iBlockNumber, site_t iSiteNumber) const; - - void DoSetWallData(site_t iBlockNumber, site_t iSiteNumber, const util::Vector3D& iData); - }; - - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTERNORMAL_H diff --git a/Code/vis/rayTracer/ClusterRayTracer.h b/Code/vis/rayTracer/ClusterRayTracer.h deleted file mode 100644 index 953fd7a6e..000000000 --- a/Code/vis/rayTracer/ClusterRayTracer.h +++ /dev/null @@ -1,761 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTERRAYTRACER_H -#define HEMELB_VIS_RAYTRACER_CLUSTERRAYTRACER_H - -#include -#include -#include - -#include "geometry/SiteTraverser.h" -#include "lb/MacroscopicPropertyCache.h" -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" -#include "vis/DomainStats.h" -#include "vis/PixelSet.h" -#include "vis/rayTracer/Cluster.h" -#include "vis/rayTracer/ClusterTraverser.h" -#include "vis/rayTracer/Ray.h" -#include "vis/Screen.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - template - class ClusterRayTracer - { - public: - ClusterRayTracer(const Viewpoint& iViewpoint, - Screen& iScreen, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings, - const hemelb::geometry::LatticeData& iLatticeData, - const lb::MacroscopicPropertyCache& propertyCache) : - viewpoint(iViewpoint), screen(iScreen), domainStats(iDomainStats), visSettings(iVisSettings), latticeData(iLatticeData), propertyCache(propertyCache) - { - // TODO: This is absolutely horrible, but neccessary until RayDataNormal is - // removed. - RayDataNormal::mDomainStats = &iDomainStats; - } - - void RenderCluster(const ClusterType& iCluster, PixelSet& pixels) - { - mLowerSiteCordinatesOfClusterRelativeToViewpoint = iCluster.GetLeastSiteOnLeastBlockInImage() - - viewpoint.GetViewpointLocation(); - - //Calculate the projection of the cluster on the screen - //refered to as the subimage - CalculateSubImage(iCluster); - - //If the entire sub-image is off the screen, - //no rendering is needed - if (SubImageOffScreen()) - { - return; - } - - CropSubImageToScreen(); - - CalculateVectorsToClusterSpanAndLowerLeftPixel(iCluster); - - CastRaysForEachPixel(iCluster, pixels); - } - - private: - void GetRayUnitsFromViewpointToCluster(const Ray & iRay, - float & oMaximumRayUnits, - float & oMinimumRayUnits) - { - // (Remember that iRay.mDirection is normalised) - float lMaxUnitRaysBasedOnX; - float lMinUnitRaysBasedOnX; - if (iRay.GetDirection().x > 0.0F) - { - lMaxUnitRaysBasedOnX = mViewpointCentreToMaxSite.x * iRay.GetInverseDirection().x; - - lMinUnitRaysBasedOnX = mViewpointCentreToMinSite.x * iRay.GetInverseDirection().x; - } - else if (iRay.GetDirection().x < 0.0F) - { - lMaxUnitRaysBasedOnX = mViewpointCentreToMinSite.x * iRay.GetInverseDirection().x; - lMinUnitRaysBasedOnX = mViewpointCentreToMaxSite.x * iRay.GetInverseDirection().x; - } - else - { - lMaxUnitRaysBasedOnX = std::numeric_limits::max(); - lMinUnitRaysBasedOnX = 0.0F; - } - - float lMaxUnitRaysBasedOnY; - float lMinUnitRaysBasedOnY; - if (iRay.GetDirection().y > 0.0F) - { - lMaxUnitRaysBasedOnY = mViewpointCentreToMaxSite.y * iRay.GetInverseDirection().y; - - lMinUnitRaysBasedOnY = mViewpointCentreToMinSite.y * iRay.GetInverseDirection().y; - } - else if (iRay.GetDirection().y < 0.0F) - { - lMaxUnitRaysBasedOnY = mViewpointCentreToMinSite.y * iRay.GetInverseDirection().y; - lMinUnitRaysBasedOnY = mViewpointCentreToMaxSite.y * iRay.GetInverseDirection().y; - } - else - { - lMaxUnitRaysBasedOnY = std::numeric_limits::max(); - lMinUnitRaysBasedOnY = 0.0F; - } - - float lMaxUnitRaysBasedOnZ; - float lMinUnitRaysBasedOnZ; - if (iRay.GetDirection().z > 0.0F) - { - lMaxUnitRaysBasedOnZ = mViewpointCentreToMaxSite.z * iRay.GetInverseDirection().z; - - lMinUnitRaysBasedOnZ = mViewpointCentreToMinSite.z * iRay.GetInverseDirection().z; - } - else if (iRay.GetDirection().z < 0.0F) - { - lMaxUnitRaysBasedOnZ = mViewpointCentreToMinSite.z * iRay.GetInverseDirection().z; - lMinUnitRaysBasedOnZ = mViewpointCentreToMaxSite.z * iRay.GetInverseDirection().z; - } - else - { - lMaxUnitRaysBasedOnZ = std::numeric_limits::max(); - lMinUnitRaysBasedOnZ = 0.0F; - } - - //Maximum ray units from viewpoint to cluster - //We want the minimum number - since at this point the ray is - //completely out - oMaximumRayUnits = std::min(std::min(lMaxUnitRaysBasedOnX, lMaxUnitRaysBasedOnY), lMaxUnitRaysBasedOnZ); - - //Maximum ray units to get us into the cluster - //We want the maximum number - since only at this point - // is the ray completely in - oMinimumRayUnits = std::max(std::max(lMinUnitRaysBasedOnX, lMinUnitRaysBasedOnY), lMinUnitRaysBasedOnZ); - - } - - void CastRay(const ClusterType& iCluster, - Ray& ioRay, - float iMaximumRayUnits, - float iMinimumRayUnits) - { - //It's possible for the ray to totally miss the cluster - //This is because the sub-image is square while the cluster - // projection won't be in most circumstances - if (iMaximumRayUnits < iMinimumRayUnits) - { - return; - } - - ioRay.SetRayLengthTraversedToCluster(iMinimumRayUnits); - - util::Vector3D fromLowerSiteToFirstRayClusterIntersection = ioRay.GetDirection() * iMinimumRayUnits - - mLowerSiteCordinatesOfClusterRelativeToViewpoint; - - TraverseBlocks(iCluster, fromLowerSiteToFirstRayClusterIntersection, ioRay); - } - - void CalculateSubImage(const ClusterType& iCluster) - { - //The extent of the cluster when projected (ie the subimage) - //is determined by projecting all eight vertices of the cuboid - XYCoordinates lSubImageLowerLeft = XYCoordinates::MaxLimit(); - XYCoordinates lSubImageUpperRight = XYCoordinates::MinLimit(); - - const std::vector > lCorners = iCluster.GetCorners(); - - for (std::vector >::const_iterator lIt = lCorners.begin(); lIt != lCorners.end(); - lIt++) - { - UpdateSubImageExtentForCorner(*lIt, lSubImageLowerLeft, lSubImageUpperRight); - } - - lowerLeftPixelCoordinatesOfSubImage = - screen.template TransformScreenToPixelCoordinates(lSubImageLowerLeft); - - // We add a unit vector here because the transformation will round down from float - // to int. - upperRightPixelCoordinatesOfSubImage = - screen.template TransformScreenToPixelCoordinates(lSubImageUpperRight); - } - - void UpdateSubImageExtentForCorner(const util::Vector3D& iCorner, - XYCoordinates& ioSubImageLowerLeft, - XYCoordinates& ioSubImageUpperRight) - { - XYCoordinates lCornerProjection = viewpoint.FlatProject(iCorner); - - ioSubImageLowerLeft.UpdatePointwiseMin(lCornerProjection); - ioSubImageUpperRight.UpdatePointwiseMax(lCornerProjection); - } - - bool SubImageOffScreen() - { - return (lowerLeftPixelCoordinatesOfSubImage.x >= screen.GetPixelsX() - || upperRightPixelCoordinatesOfSubImage.x < 0 - || lowerLeftPixelCoordinatesOfSubImage.y >= screen.GetPixelsY() - || upperRightPixelCoordinatesOfSubImage.y < 0); - } - - void CropSubImageToScreen() - { - lowerLeftPixelCoordinatesOfSubImage.x = util::NumericalFunctions::max(lowerLeftPixelCoordinatesOfSubImage.x, - 0); - - upperRightPixelCoordinatesOfSubImage.x = - util::NumericalFunctions::min(upperRightPixelCoordinatesOfSubImage.x, screen.GetPixelsX() - 1); - - lowerLeftPixelCoordinatesOfSubImage.y = util::NumericalFunctions::max(lowerLeftPixelCoordinatesOfSubImage.y, - 0); - - upperRightPixelCoordinatesOfSubImage.y = - util::NumericalFunctions::min(upperRightPixelCoordinatesOfSubImage.y, screen.GetPixelsY() - 1); - } - - void CalculateVectorsToClusterSpanAndLowerLeftPixel(const ClusterType& iCluster) - { - mViewpointCentreToMaxSite = iCluster.GetMaxSite() - viewpoint.GetViewpointLocation(); - - mViewpointCentreToMinSite = iCluster.GetMinSite() - viewpoint.GetViewpointLocation(); - - //Obtaining the vector from the camera to the lower left pixel - fromCameraToBottomLeftPixelOfSubImage = screen.GetCameraToBottomLeftOfScreenVector() - + screen.GetPixelUnitVectorProjectionX() * (float) lowerLeftPixelCoordinatesOfSubImage.x - + screen.GetPixelUnitVectorProjectionY() * (float) lowerLeftPixelCoordinatesOfSubImage.y; - } - - void CastRaysForEachPixel(const ClusterType& iCluster, PixelSet& pixels) - { - XYCoordinates lPixel; - - //Loop over all the pixels - util::Vector3D lCameraToBottomRow = fromCameraToBottomLeftPixelOfSubImage; - for (lPixel.x = lowerLeftPixelCoordinatesOfSubImage.x; lPixel.x <= upperRightPixelCoordinatesOfSubImage.x; - ++lPixel.x) - { - util::Vector3D lCameraToPixel = lCameraToBottomRow; - for (lPixel.y = lowerLeftPixelCoordinatesOfSubImage.y; lPixel.y <= upperRightPixelCoordinatesOfSubImage.y; - ++lPixel.y) - { - CastRayForPixel(iCluster, lPixel, lCameraToPixel, pixels); - - lCameraToPixel += screen.GetPixelUnitVectorProjectionY(); - } - - lCameraToBottomRow += screen.GetPixelUnitVectorProjectionX(); - } - } - - virtual void CastRayForPixel(const ClusterType& iCluster, - const XYCoordinates& iPixelCoordinates, - const util::Vector3D& iRayDirection, - PixelSet& pixels) - { - Ray lRay(iRayDirection, iPixelCoordinates.x, iPixelCoordinates.y); - - //These tell us how many ray units get us into the cluster - //and after how many ray units we are out - float lMaximumRayUnits; - float lMinimumRayUnits; - GetRayUnitsFromViewpointToCluster(lRay, lMaximumRayUnits, lMinimumRayUnits); - - CastRay(iCluster, lRay, lMaximumRayUnits, lMinimumRayUnits); - - //Make sure the ray hasn't reached infinity - if (!lRay.CollectedNoData()) - { - pixels.AddPixel(lRay.GetRayData()); - } - } - - void TraverseRayThroughBlock(const util::Vector3D& fromFirstRayClusterIntersectionToLowerSiteOfCurrentBlock, - const util::Vector3D& iLocationInBlock, - const ClusterType& iCluster, - const util::Vector3D& blockLocation, - const site_t blockNumberOnCluster, - float euclideanClusterLengthTraversedByRay, - Ray& ioRay) - { - //Work out which site we're currently in - const util::Vector3D truncatedLocationInBlock = RoundToNearestVoxel(iLocationInBlock); - - geometry::SiteTraverser siteTraverser(latticeData); - siteTraverser.SetCurrentLocation(truncatedLocationInBlock); - - // In order to trace the rays through the voxels, we need to keep track of how far the - // ray can travel to the next voxel in each of the three directions in ray units - util::Vector3D rayUnitsUntilNextSite = - CalculateRayUnitsBeforeNextSite(fromFirstRayClusterIntersectionToLowerSiteOfCurrentBlock, - util::Vector3D(truncatedLocationInBlock), - ioRay); - - while (siteTraverser.CurrentLocationValid()) - { - // Firstly, work out in which direction we - // can travel the least ray units before reaching - // a vortex side - const util::Direction::Direction directionOfLeastTravel = DirectionOfLeastTravel(rayUnitsUntilNextSite); - - // Find out how far the ray can move - const float manhattanRayLengthThroughVoxel = rayUnitsUntilNextSite.GetByDirection(directionOfLeastTravel); - - const geometry::Block& block = latticeData.GetBlock(latticeData.GetBlockIdFromBlockCoords(blockLocation)); - - if (!block.IsEmpty()) // Ensure fluid site - { - if (!block.SiteIsSolid(siteTraverser.GetCurrentIndex())) - { - const site_t localContiguousId = - block.GetLocalContiguousIndexForSite(siteTraverser.GetCurrentIndex()); - - SiteData_t siteData; - siteData.density = propertyCache.densityCache.Get(localContiguousId); - siteData.velocity = propertyCache.velocityCache.Get(localContiguousId).GetMagnitude(); - - if (visSettings.mStressType == lb::ShearStress) - { - siteData.stress = propertyCache.wallShearStressMagnitudeCache.Get(localContiguousId); - } - else - { - siteData.stress = propertyCache.vonMisesStressCache.Get(localContiguousId); - } - - const util::Vector3D* lWallData = iCluster.GetWallData(blockNumberOnCluster, - siteTraverser.GetCurrentIndex()); - - if (lWallData == NULL || lWallData->x == NO_VALUE) - { - ioRay.UpdateDataForNormalFluidSite(siteData, - manhattanRayLengthThroughVoxel - - euclideanClusterLengthTraversedByRay, // Manhattan Ray-length through the voxel - euclideanClusterLengthTraversedByRay, // euclidean ray units spent in cluster - domainStats, - visSettings); - } - else - { - ioRay.UpdateDataForWallSite(siteData, - manhattanRayLengthThroughVoxel - euclideanClusterLengthTraversedByRay, - euclideanClusterLengthTraversedByRay, - domainStats, - visSettings, - lWallData); - } - } - else - { - ioRay.ProcessSolidSite(); - } - } - else - { - ioRay.ProcessSolidSite(); - } - - //Update ray length traversed so far - euclideanClusterLengthTraversedByRay = manhattanRayLengthThroughVoxel; - - //Update the block location and RayUnitsBeforeNextVoxel - //in each direction - switch (directionOfLeastTravel) - { - case util::Direction::X: - if (ioRay.XIncreasing()) - { - siteTraverser.IncrementX(); - rayUnitsUntilNextSite.x += ioRay.GetInverseDirection().x; - } - else - { - siteTraverser.DecrementX(); - rayUnitsUntilNextSite.x -= ioRay.GetInverseDirection().x; - } - - break; - - case util::Direction::Y: - if (ioRay.YIncreasing()) - { - siteTraverser.IncrementY(); - rayUnitsUntilNextSite.y += ioRay.GetInverseDirection().y; - } - else - { - siteTraverser.DecrementY(); - rayUnitsUntilNextSite.y -= ioRay.GetInverseDirection().y; - } - break; - - case util::Direction::Z: - if (ioRay.ZIncreasing()) - { - siteTraverser.IncrementZ(); - rayUnitsUntilNextSite.z += ioRay.GetInverseDirection().z; - } - else - { - siteTraverser.DecrementZ(); - rayUnitsUntilNextSite.z -= ioRay.GetInverseDirection().z; - } - break; - } - - } - } - - util::Vector3D CalculateRayUnitsBeforeNextSite(const util::Vector3D& iFirstRayClusterIntersectionToBlockLowerSite, - const util::Vector3D& iTruncatedLocationInBlock, - const Ray& iRay) const - { - util::Vector3D lRayUnits; - - //The ray has already travelled iFirstRayClusterIntersectionToBlockLowerSite - //If the ray is increasing in the co-ordinte it can travel as far - //as the truncated location + 1, otherwise just the truncated location - //for each co-ordinate - - //If the ray has zero in any direction, set the ray units to max - - if (iRay.GetDirection().x == 0.0F) - { - lRayUnits.x = std::numeric_limits::max(); - } - else - { - lRayUnits.x = iFirstRayClusterIntersectionToBlockLowerSite.x + (float) (iTruncatedLocationInBlock.x); - if (iRay.XIncreasing()) - { - lRayUnits.x += 1.0F; - } - //Convert from site units into ray units - lRayUnits.x *= iRay.GetInverseDirection().x; - } - - if (iRay.GetDirection().y == 0.0F) - { - lRayUnits.y = std::numeric_limits::max(); - } - else - { - lRayUnits.y = iFirstRayClusterIntersectionToBlockLowerSite.y + (float) (iTruncatedLocationInBlock.y); - if (iRay.YIncreasing()) - { - lRayUnits.y += 1.0F; - } - lRayUnits.y *= iRay.GetInverseDirection().y; - } - - if (iRay.GetDirection().z == 0.0F) - { - lRayUnits.z = std::numeric_limits::max(); - } - else - { - lRayUnits.z = iFirstRayClusterIntersectionToBlockLowerSite.z + (float) (iTruncatedLocationInBlock.z); - if (iRay.ZIncreasing()) - { - lRayUnits.z += 1.0F; - } - lRayUnits.z *= iRay.GetInverseDirection().z; - } - - return lRayUnits; - } - - util::Vector3D RoundToNearestVoxel(const util::Vector3D& iUnboundLocation) const - { - util::Vector3D lVoxelLocationInBlock; - - //Due to rounding errors, it's possible for the site location within a block - //to be outside the wrong block - lVoxelLocationInBlock.x = util::NumericalFunctions::enforceBounds((site_t) iUnboundLocation.x, - 0, - latticeData.GetBlockSize() - 1); - - lVoxelLocationInBlock.y = util::NumericalFunctions::enforceBounds((site_t) iUnboundLocation.y, - 0, - latticeData.GetBlockSize() - 1); - - lVoxelLocationInBlock.z = util::NumericalFunctions::enforceBounds((site_t) iUnboundLocation.z, - 0, - latticeData.GetBlockSize() - 1); - - return lVoxelLocationInBlock; - } - - util::Direction::Direction DirectionOfLeastTravel(const util::Vector3D& iRayUnitsBeforeNextVoxelOrBlock) const - { - - if (iRayUnitsBeforeNextVoxelOrBlock.x < iRayUnitsBeforeNextVoxelOrBlock.y) - { - //X is less than Y - if (iRayUnitsBeforeNextVoxelOrBlock.x < iRayUnitsBeforeNextVoxelOrBlock.z) - { - //X is less than Y and Z - return util::Direction::X; - } - else - { - //X is less than Y - //Z is less Than X (And Y) - return util::Direction::Z; - } - } - else - { - // Y is less than X - if (iRayUnitsBeforeNextVoxelOrBlock.y < iRayUnitsBeforeNextVoxelOrBlock.z) - { - //Y is less than X and Z - return util::Direction::Y; - } - else - { - //Y is less than X - //Z is less than Y (and so X) - return util::Direction::Z; - } - - } - } - - void TraverseBlocks(const ClusterType& iCluster, - const util::Vector3D& fromLowerClusterSiteToFirstRayIntersection, - Ray& ioRay) - { - float blockSizeAsFloat = (float) (latticeData.GetBlockSize()); - - //Calculate the coordinates of the block within the cluster where - //the ray first intersects - const util::Vector3D blockHoldingFirstIntersection = - GetBlockCoordinatesOfFirstIntersectionBlock(iCluster, fromLowerClusterSiteToFirstRayIntersection); - - //The Cluster Traverser keeps track of which block we're at - //in the cluster - ClusterTraverser clusterTraverser(iCluster); - - clusterTraverser.SetCurrentLocation(blockHoldingFirstIntersection); - - // The number of ray units in a block in each direction are cached. - const util::Vector3D rayUnitsAlongEachBlockSize = ioRay.GetInverseDirection() * blockSizeAsFloat; - - //For every block that is traversed, a vector is needed from - //where the ray first hits the cluster to the lower site ie site - //(0,0,0) within the block. This is required to locate how - //far the ray has travelled and where each part is in relation to - //voxel sites - util::Vector3D fromFirstIntersectionToLowerSiteOfCurrentBlock = - util::Vector3D(blockHoldingFirstIntersection) * blockSizeAsFloat - - fromLowerClusterSiteToFirstRayIntersection; - - //We need to know how many ray units can be traversed before - //a new block is hit. The initial value is calculated based on - //the location of first intersection - util::Vector3D totalRayUnitsToNextBlockFromFirstIntersection = - CalculateMinimalTotalRayUnitsToBlocksBehindCurrentOne(fromFirstIntersectionToLowerSiteOfCurrentBlock, - ioRay); - - // We need to track how far the ray has travelled - float siteUnitsTraversed = 0.0F; - - while (clusterTraverser.CurrentLocationValid()) - { - // The location of the ray within the block. - util::Vector3D siteLocationWithinBlock = (ioRay.GetDirection()) * siteUnitsTraversed - - fromFirstIntersectionToLowerSiteOfCurrentBlock; - - TraverseRayThroughBlock(fromFirstIntersectionToLowerSiteOfCurrentBlock, - siteLocationWithinBlock, - iCluster, - iCluster.GetMinBlockLocation() + clusterTraverser.GetCurrentLocation(), - clusterTraverser.GetCurrentIndex(), - siteUnitsTraversed, - ioRay); - - // The direction of least travel is the direction of - // the next block that will be hit by the ray - // relative to the current block. - util::Direction::Direction lDirectionOfLeastTravel = - DirectionOfLeastTravel(totalRayUnitsToNextBlockFromFirstIntersection); - - //Move to the next block based on the direction - //of least travel and update variables accordingly - siteUnitsTraversed = - totalRayUnitsToNextBlockFromFirstIntersection.GetByDirection(lDirectionOfLeastTravel); - - switch (lDirectionOfLeastTravel) - { - case util::Direction::X: - if (ioRay.XIncreasing()) - { - clusterTraverser.IncrementX(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.x += blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.x += rayUnitsAlongEachBlockSize.x; - } - else - { - clusterTraverser.DecrementX(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.x -= blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.x -= rayUnitsAlongEachBlockSize.x; - } - break; - - case util::Direction::Y: - if (ioRay.YIncreasing()) - { - clusterTraverser.IncrementY(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.y += blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.y += rayUnitsAlongEachBlockSize.y; - } - else - { - clusterTraverser.DecrementY(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.y -= blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.y -= rayUnitsAlongEachBlockSize.y; - } - break; - - case util::Direction::Z: - if (ioRay.ZIncreasing()) - { - clusterTraverser.IncrementZ(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.z += blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.z += rayUnitsAlongEachBlockSize.z; - } - else - { - clusterTraverser.DecrementZ(); - fromFirstIntersectionToLowerSiteOfCurrentBlock.z -= blockSizeAsFloat; - totalRayUnitsToNextBlockFromFirstIntersection.z -= rayUnitsAlongEachBlockSize.z; - } - break; - } - - } - - } - - util::Vector3D GetBlockCoordinatesOfFirstIntersectionBlock(const ClusterType& iCluster, - const util::Vector3D& iLowerSiteToFirstRayClusterIntersection) - { - util::Vector3D lBlockCoordinatesOfFirstIntersectionBlock; - - //Perform the truncated division and ensure that the - //coordinates are valid to allow for numerical errors - const util::Vector3D exactBlockCoordsOfFirstIntersectingBlock = - iLowerSiteToFirstRayClusterIntersection * (1.0F / (float) latticeData.GetBlockSize()); - - lBlockCoordinatesOfFirstIntersectionBlock.x = - (site_t) util::NumericalFunctions::enforceBounds((site_t) exactBlockCoordsOfFirstIntersectingBlock.x, - 0, - iCluster.GetBlocksX() - 1); - - lBlockCoordinatesOfFirstIntersectionBlock.y = - (site_t) util::NumericalFunctions::enforceBounds((site_t) exactBlockCoordsOfFirstIntersectingBlock.y, - 0, - iCluster.GetBlocksY() - 1); - - lBlockCoordinatesOfFirstIntersectionBlock.z = - (site_t) util::NumericalFunctions::enforceBounds((site_t) exactBlockCoordsOfFirstIntersectingBlock.z, - 0, - iCluster.GetBlocksZ() - 1); - - return lBlockCoordinatesOfFirstIntersectionBlock; - } - - util::Vector3D CalculateMinimalTotalRayUnitsToBlocksBehindCurrentOne(const util::Vector3D& lFirstIntersectionToBlockLowerSite, - const Ray& iRay) const - { - util::Vector3D lRayUnits; - - //The ray is currently at the first intersection - //The number of ray units for a given co-ordinate is the - //distance in sites to the next block divided by - //the ray unit distance projected in that direction - - if (iRay.GetDirection().x == 0.0F) - { - lRayUnits.x = std::numeric_limits::max(); - } - else - { - lRayUnits.x = lFirstIntersectionToBlockLowerSite.x; - //If the ray is increasing in this coordinate, we want the - //distance to the next block - if (iRay.XIncreasing()) - { - lRayUnits.x += (float) (latticeData.GetBlockSize()); - } - //Turn this from site units into ray units - lRayUnits.x *= iRay.GetInverseDirection().x; - } - - if (iRay.GetDirection().y == 0.0F) - { - lRayUnits.y = std::numeric_limits::max(); - } - else - { - lRayUnits.y = lFirstIntersectionToBlockLowerSite.y; - if (iRay.YIncreasing()) - { - lRayUnits.y += (float) (latticeData.GetBlockSize()); - } - lRayUnits.y *= iRay.GetInverseDirection().y; - } - - if (iRay.GetDirection().z == 0.0F) - { - lRayUnits.z = std::numeric_limits::max(); - } - else - { - lRayUnits.z = lFirstIntersectionToBlockLowerSite.z; - if (iRay.ZIncreasing()) - { - lRayUnits.z += (float) (latticeData.GetBlockSize()); - } - lRayUnits.z *= iRay.GetInverseDirection().z; - } - - return lRayUnits; - } - - const Viewpoint& viewpoint; - const Screen& screen; - const DomainStats& domainStats; - const VisSettings& visSettings; - const hemelb::geometry::LatticeData& latticeData; - /** - * The cache of macroscopic fluid properties at each local fluid site. - */ - const lb::MacroscopicPropertyCache& propertyCache; - - util::Vector3D fromCameraToBottomLeftPixelOfSubImage; - - util::Vector3D mLowerSiteCordinatesOfClusterRelativeToViewpoint; - - XYCoordinates lowerLeftPixelCoordinatesOfSubImage; - XYCoordinates upperRightPixelCoordinatesOfSubImage; - - //Vectors from the viewpoint centre - //to the maximum and minimum site span - //locations respectively - util::Vector3D mViewpointCentreToMaxSite; - util::Vector3D mViewpointCentreToMinSite; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTERRAYTRACER_H diff --git a/Code/vis/rayTracer/ClusterTraverser.h b/Code/vis/rayTracer/ClusterTraverser.h deleted file mode 100644 index cc1b96621..000000000 --- a/Code/vis/rayTracer/ClusterTraverser.h +++ /dev/null @@ -1,55 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTERTRAVERSER_H -#define HEMELB_VIS_RAYTRACER_CLUSTERTRAVERSER_H - -#include "geometry/VolumeTraverser.h" -#include "vis/rayTracer/Cluster.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - //ClusterTraverser is used to traverse the cluster - template - class ClusterTraverser : public geometry::VolumeTraverser - { - public: - ClusterTraverser(const ClusterType& iCluster) : - mCluster(iCluster) - { - } - - virtual ~ClusterTraverser() - { - } - - site_t GetXCount() const - { - return mCluster.GetBlocksX(); - } - - site_t GetYCount() const - { - return mCluster.GetBlocksY(); - } - - site_t GetZCount() const - { - return mCluster.GetBlocksZ(); - } - - private: - const ClusterType& mCluster; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTERTRAVERSER_H diff --git a/Code/vis/rayTracer/ClusterWithWallNormals.cc b/Code/vis/rayTracer/ClusterWithWallNormals.cc deleted file mode 100644 index 9f12dfd83..000000000 --- a/Code/vis/rayTracer/ClusterWithWallNormals.cc +++ /dev/null @@ -1,64 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "geometry/LatticeData.h" -#include "vis/rayTracer/ClusterWithWallNormals.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - ClusterWithWallNormals::ClusterWithWallNormals(unsigned short xBlockCount, - unsigned short yBlockCount, - unsigned short zBlockCount, - const util::Vector3D& minimalSite, - const util::Vector3D& maximalSite, - const util::Vector3D& minimalSiteOnMinimalBlock, - const util::Vector3D& minimalBlock) : - Cluster(xBlockCount, - yBlockCount, - zBlockCount, - minimalSite, - maximalSite, - minimalSiteOnMinimalBlock, - minimalBlock) - { - WallNormals.resize(GetBlocksX() * GetBlocksY() * GetBlocksZ()); - } - - const util::Vector3D* ClusterWithWallNormals::DoGetWallData(site_t blockNumber, site_t siteNumber) const - { - if (siteNumber < (site_t) WallNormals[blockNumber].size()) - { - return WallNormals[blockNumber][siteNumber]; - } - else - { - return NULL; - } - } - - void ClusterWithWallNormals::DoSetWallData(site_t blockNumber, - site_t siteNumber, - const util::Vector3D& data) - { - if (WallNormals[blockNumber].size() <= (size_t) siteNumber) - { - WallNormals[blockNumber].resize(siteNumber + 1, NULL); - } - - WallNormals[blockNumber][siteNumber] = &data; - } - - bool ClusterWithWallNormals::DoNeedsWallNormals() - { - return true; - } - } - } -} diff --git a/Code/vis/rayTracer/ClusterWithWallNormals.h b/Code/vis/rayTracer/ClusterWithWallNormals.h deleted file mode 100644 index f823bf10f..000000000 --- a/Code/vis/rayTracer/ClusterWithWallNormals.h +++ /dev/null @@ -1,44 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_CLUSTERWITHWALLNORMALS_H -#define HEMELB_VIS_RAYTRACER_CLUSTERWITHWALLNORMALS_H - -#include "vis/rayTracer/Cluster.h" -#include "vis/rayTracer/SiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - class ClusterWithWallNormals : public Cluster - { - public: - ClusterWithWallNormals(unsigned short xBlockCount, - unsigned short yBlockCount, - unsigned short zBlockCount, - const util::Vector3D& minimalSite, - const util::Vector3D& maximalSite, - const util::Vector3D& minimalSiteOnMinimalBlock, - const util::Vector3D& minimalBlock); - - const util::Vector3D* DoGetWallData(site_t iBlockNumber, site_t iSiteNumber) const; - - void DoSetWallData(site_t iBlockNumber, site_t iSiteNumber, const util::Vector3D& iData); - - static bool DoNeedsWallNormals(); - - private: - std::vector*> > WallNormals; - }; - - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_CLUSTERWITHWALLNORMALS_H diff --git a/Code/vis/rayTracer/HSLToRGBConverter.cc b/Code/vis/rayTracer/HSLToRGBConverter.cc deleted file mode 100644 index 1e4252bbd..000000000 --- a/Code/vis/rayTracer/HSLToRGBConverter.cc +++ /dev/null @@ -1,127 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#if HEMELB_HAVE_CSTDINT -# include -#else -# include -#endif - -#include "vis/rayTracer/HSLToRGBConverter.h" -#include "log/Logger.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - //See http://en.wikipedia.org/wiki/HSL_and_HSV for the - //formulae used - //TODO: Possibly replace truncation with rounding? - void HSLToRGBConverter::Convert(float iHue, - float iSaturation, - float iLightness, - unsigned char oRGBColour[3]) - { - //All values are stored as 32-bit unsigned integers - //stored within 16-bits, to allow multiplication between - //two values - - //All values are scaled up to be between 0 and the - //maximum value of a 16-bit interger - - //Scales a value between 0 and 360 to a range between 0 - //and the maximum value of a 16-bit interger - static const uint32_t DegreesScalar = std::numeric_limits::max() / 360; - - //Scales a value between 0 and 1 to a range between 0 - //and the maximum value of a 16-bit interger - static const uint32_t OtherScalar = std::numeric_limits::max(); - - //Cast the inputs accodingly - uint32_t lHue = (uint32_t)(iHue * (float) (DegreesScalar)); - uint32_t lSaturation = (uint32_t)(iSaturation * (float) (OtherScalar)); - uint32_t lLightness = (uint32_t)(iLightness * (float) (OtherScalar)); - - //Calculate the Chroma - a division by OtherScalar is - //required for the Chroma to remain between 0 - //and the maximum value of a 16-bit interger - int32_t lTemp = 2 * (int32_t) (lLightness) - (int32_t) (OtherScalar); - uint32_t lChroma = ( (OtherScalar - abs(lTemp)) * lSaturation) / OtherScalar; - - uint32_t lHuePrime = lHue / 60; - - lTemp = (int) (lHuePrime % (2 * DegreesScalar)) - (int) (DegreesScalar); - - //Calculate the "Intermediate" - a division by - //DegreesScalar is required for the Chroma to - //remain between 0 and the maximum value of a 16-bit interger - uint32_t lIntermediate = (lChroma * (DegreesScalar - abs(lTemp))) / DegreesScalar; - - uint32_t red=0, green=0, blue=0; - - //Map the hue to value to six cases - uint32_t lHueInt = lHuePrime / DegreesScalar; - switch (lHueInt) - { - case 0: - red = lChroma; - green = lIntermediate; - blue = 0.0F; - break; - - case 1: - red = lIntermediate; - green = lChroma; - blue = 0.0F; - break; - - case 2: - red = 0.0F; - green = lChroma; - blue = lIntermediate; - break; - - case 3: - red = 0.0F; - green = lIntermediate; - blue = lChroma; - break; - - case 4: - red = lIntermediate; - green = 0.0F; - blue = lChroma; - break; - - case 5: - red = lChroma; - green = 0.0F; - blue = lIntermediate; - break; - - default: - log::Logger::Log("Failed while using hue in HslToRgbConvertor."); - } - - // A value to divide the results by to map them - // between 0 and the maximum value of an unsigned - //char - static const uint32_t CharScalar = OtherScalar / std::numeric_limits::max(); - - int32_t lMatcher = (int) (lLightness) - (int) (lChroma) / 2; - - oRGBColour[0] = (unsigned char) ( (red + lMatcher) / CharScalar); - oRGBColour[1] = (unsigned char) ( (green + lMatcher) / CharScalar); - oRGBColour[2] = (unsigned char) ( (blue + lMatcher) / CharScalar); - } - } - } -} diff --git a/Code/vis/rayTracer/HSLToRGBConverter.h b/Code/vis/rayTracer/HSLToRGBConverter.h deleted file mode 100644 index 94830a1cf..000000000 --- a/Code/vis/rayTracer/HSLToRGBConverter.h +++ /dev/null @@ -1,34 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_HSLTORGBCONVERTER_H -#define HEMELB_VIS_RAYTRACER_HSLTORGBCONVERTER_H - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - class HSLToRGBConverter - { - public: - /** - * Converts a colour in HSL coordinates to - * RGB coordinates (between 0 and 255) - * iHue must be between 0.0F and 360.0F in degrees - * and the other two between 0.0F and 1.0F - */ - static void Convert(float iHue, - float iSaturation, - float iLightness, - unsigned char oRGBColour[3]); - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_HSLTORGBCONVERTER_H diff --git a/Code/vis/rayTracer/Ray.h b/Code/vis/rayTracer/Ray.h deleted file mode 100644 index 991fd3790..000000000 --- a/Code/vis/rayTracer/Ray.h +++ /dev/null @@ -1,165 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_RAY_H -#define HEMELB_VIS_RAYTRACER_RAY_H - -#include -#include - -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" -#include "vis/rayTracer/SiteData.h" -#include "vis/rayTracer/RayDataNormal.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - template - class Ray - { - public: - Ray(util::Vector3D iDirection, int i, int j) : - - mDirection(iDirection.Normalise()), - mInverseDirection(1.0F / mDirection.x, 1.0F / mDirection.y, 1.0F / mDirection.z), - mInWall(false), mPassedThroughNormalFluidSite(false), mRayData(i, j) - { - } - - const util::Vector3D& GetDirection() const - { - return mDirection; - } - - const util::Vector3D& GetInverseDirection() const - { - return mInverseDirection; - } - - bool XIncreasing() const - { - return GetDirection().x > 0.0F; - } - - bool YIncreasing() const - { - return GetDirection().y > 0.0F; - } - - bool ZIncreasing() const - { - return GetDirection().z > 0.0F; - } - - void UpdateDataForWallSite(const SiteData_t& iSiteData, - const float iRayLengthInVoxel, - const float iRayUnitsInCluster, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings, - const util::Vector3D* iWallNormal) - { - //Have we just entered the wall? - if (!mInWall) - { - mRayData.UpdateDataForWallSite(iSiteData, - GetDirection(), - iRayLengthInVoxel, - iRayUnitsInCluster + mRayUnitsTraversedToCluster, - iDomainStats, - iVisSettings, - iWallNormal); - //We're in the wall - mInWall = true; - } - else - { - //We've already processed a wall site - process as a normal site - mRayData.UpdateDataForNormalFluidSite(iSiteData, - GetDirection(), - iRayLengthInVoxel, - iRayUnitsInCluster - + mRayUnitsTraversedToCluster, - iDomainStats, - iVisSettings); - } - } - - void UpdateDataForNormalFluidSite(const SiteData_t& iSiteData, - const float manhattanRayLengthThroughVoxel, - const float euclideanRayUnitsSpentInCluster, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings) - { - //Set mInWall to false in case we've just left a wall - mInWall = false; - - //We know we've passed through normal fluid - mPassedThroughNormalFluidSite = true; - - mRayData.UpdateDataForNormalFluidSite(iSiteData, - GetDirection(), - manhattanRayLengthThroughVoxel, - euclideanRayUnitsSpentInCluster - + mRayUnitsTraversedToCluster, - iDomainStats, - iVisSettings); - } - - void ProcessSolidSite() - { - //Special case - tangenting the vessel and never reaching - //normal fluid sites - if (mInWall && !mPassedThroughNormalFluidSite) - { - mRayData.ProcessTangentingVessel(); - } - - //We're out the wall - mInWall = false; - } - - RayDataType GetRayData() - { - return mRayData; - } - - bool CollectedNoData() - { - return !mRayData.ContainsRayData(); - } - - void SetRayLengthTraversedToCluster(float iRayUnitsTraversedToCluster) - { - mRayUnitsTraversedToCluster = iRayUnitsTraversedToCluster; - } - - private: - const util::Vector3D mDirection; - const util::Vector3D mInverseDirection; - - //mInWall indicates whether the ray is in a wall or not - //- if a wall site has just been processed - bool mInWall; - - //mPassedThroughNormalFluid indicates whether the ray has passed - //through normal fluid since entering a wall. If this remains false - //and the ray hits a non-fluid site, the ray has just tangented - //the surface of the vessel - bool mPassedThroughNormalFluidSite; - - float mRayUnitsTraversedToCluster; - - RayDataType mRayData; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_RAY_H diff --git a/Code/vis/rayTracer/RayData.h b/Code/vis/rayTracer/RayData.h deleted file mode 100644 index 8155bc84c..000000000 --- a/Code/vis/rayTracer/RayData.h +++ /dev/null @@ -1,248 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_RAYDATA_H -#define HEMELB_VIS_RAYTRACER_RAYDATA_H - -#include "constants.h" -#include "net/mpi.h" -#include "vis/BasicPixel.h" -#include "vis/DomainStats.h" -#include "vis/VisSettings.h" -#include "vis/rayTracer/SiteData.h" -#include "lb/LbmParameters.h" -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - /** - * The abstract base type for RayData, using the CRTP pattern - * Contains functionality common to all derived types - */ - - template - class RayData : public BasicPixel - { - public: - RayData(int i, int j) : - BasicPixel(i, j) - { - // A cheap way of indicating no ray data - mCumulativeLengthInFluid = 0.0F; - } - - RayData() - { - - } - - // Used to process the ray data for a normal (non-wall) fluid site - // Calls the method common to all ray data processing classes - // and then the derived class method - void UpdateDataForNormalFluidSite(const raytracer::SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const float iAbsoluteDistanceFromViewpoint, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings) - { - UpdateRayDataCommon(iSiteData, - iRayDirection, - iRayLengthInVoxel, - iAbsoluteDistanceFromViewpoint, - iDomainStats, - iVisSettings); - - ((Derived*) (this))->DoUpdateDataForNormalFluidSite(iSiteData, - iRayDirection, - iRayLengthInVoxel, - iVisSettings); - } - - // Processes the ray data for a normal (non wall) fluid site - void UpdateDataForWallSite(const raytracer::SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const float iAbsoluteDistanceFromViewpoint, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings, - const util::Vector3D* iWallNormal) - { - UpdateRayDataCommon(iSiteData, - iRayDirection, - iRayLengthInVoxel, - iAbsoluteDistanceFromViewpoint, - iDomainStats, - iVisSettings); - - ((Derived*) (this))->DoUpdateDataForWallSite(iSiteData, - iRayDirection, - iRayLengthInVoxel, - iVisSettings, - iWallNormal); - } - - void ProcessTangentingVessel() - { - ((Derived*) (this))->DoProcessTangentingVessel(); - } - - // Merges in the data from another segment of ray (from another core) - void Combine(const Derived& iOtherRayData) - { - // Carry out the merging specific to the derived class - ((Derived*) (this))->DoCombine(iOtherRayData); - - // Sum length in fluid - SetCumulativeLengthInFluid(GetCumulativeLengthInFluid() + iOtherRayData.GetCumulativeLengthInFluid()); - - // Update data relating to site nearest to viewpoint - if (iOtherRayData.GetLengthBeforeRayFirstCluster() < this->GetLengthBeforeRayFirstCluster()) - { - SetLengthBeforeRayFirstCluster(iOtherRayData.GetLengthBeforeRayFirstCluster()); - - SetNearestDensity(iOtherRayData.GetNearestDensity()); - SetNearestStress(iOtherRayData.GetNearestStress()); - } - } - - // Obtains the colour representing the velocity ray trace - void GetVelocityColour(unsigned char oColour[3], - const VisSettings& iVisSettings, - const DomainStats& iDomainStats) const - { - ((const Derived*) (this))->DoGetVelocityColour(oColour, - GetLengthBeforeRayFirstCluster() - / iVisSettings.maximumDrawDistance, - iDomainStats); - } - - // Obtains the colour representing the stress ray trace - void GetStressColour(unsigned char oColour[3], - const VisSettings& iVisSettings, - const DomainStats& iDomainStats) const - { - ((const Derived*) (this))->DoGetStressColour(oColour, - GetLengthBeforeRayFirstCluster() - / iVisSettings.maximumDrawDistance, - iDomainStats); - } - - // Whether or not the data contained in the instance has valid - // ray data, or if it has just been constructed - bool ContainsRayData() const - { - return (GetCumulativeLengthInFluid() != 0.0F); - } - - float GetNearestStress() const - { - return mStressAtNearestPoint; - } - - float GetNearestDensity() const - { - return mDensityAtNearestPoint; - } - - float GetCumulativeLengthInFluid() const - { - return mCumulativeLengthInFluid; - } - - float GetLengthBeforeRayFirstCluster() const - { - return mLengthBeforeRayFirstCluster; - } - - void LogDebuggingInformation() const - { - log::Logger::Log("Ray data at (%i,%i) with " - "(lengthToFirstCluster, lengthInFluid, nearestDensity, nearest stress) = (%f, %f, %f, %f)", - GetI(), - GetJ(), - GetLengthBeforeRayFirstCluster(), - GetCumulativeLengthInFluid(), - GetNearestDensity(), - GetNearestStress()); - } - - protected: - static const float mLongestDistanceInVoxelInverse; - - static void PickColour(float value, float colour[3]) - { - colour[0] = util::NumericalFunctions::enforceBounds(4.F * value - 2.F, 0.F, 1.F); - colour[1] - = util::NumericalFunctions::enforceBounds(2.F - 4.F * (float) fabs(value - 0.5F), 0.F, 1.F); - colour[2] = util::NumericalFunctions::enforceBounds(2.F - 4.F * value, 0.F, 1.F); - } - - float mLengthBeforeRayFirstCluster; - float mCumulativeLengthInFluid; - - float mDensityAtNearestPoint; - float mStressAtNearestPoint; - - private: - // Perform processing of the ray data common to all derived - // types, ie cumulative length in the fluid, nearest stress - // and density. - void UpdateRayDataCommon(const raytracer::SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const float iAbsoluteDistanceFromViewpoint, - const DomainStats& iDomainStats, - const VisSettings& iVisSettings) - { - if (GetCumulativeLengthInFluid() == 0.0F) - { - SetLengthBeforeRayFirstCluster(iAbsoluteDistanceFromViewpoint); - - // Keep track of the density nearest to the viewpoint - SetNearestDensity( (iSiteData.density - (float) iDomainStats.density_threshold_min) - * (float) iDomainStats.density_threshold_minmax_inv); - - if (iVisSettings.mStressType == lb::VonMises || iVisSettings.mStressType == lb::ShearStress) - { - // Keep track of the stress nearest to the viewpoint - SetNearestStress(iSiteData.stress * (float) (iDomainStats.stress_threshold_max_inv)); - } - } - - SetCumulativeLengthInFluid(GetCumulativeLengthInFluid() + iRayLengthInVoxel); - } - - void SetNearestStress(float iStress) - { - mStressAtNearestPoint = iStress; - } - - void SetNearestDensity(float iDensity) - { - mDensityAtNearestPoint = iDensity; - } - - void SetCumulativeLengthInFluid(float iLength) - { - mCumulativeLengthInFluid = iLength; - } - - void SetLengthBeforeRayFirstCluster(float iLength) - { - mLengthBeforeRayFirstCluster = iLength; - } - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_RAYDATA_H diff --git a/Code/vis/rayTracer/RayDataEnhanced.cc b/Code/vis/rayTracer/RayDataEnhanced.cc deleted file mode 100644 index 07796bc0b..000000000 --- a/Code/vis/rayTracer/RayDataEnhanced.cc +++ /dev/null @@ -1,59 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "vis/rayTracer/RayDataEnhanced.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - namespace DepthCuing - { - const float Mist::SurfaceNormalLightnessRange = 0.3F; - const float Mist::ParallelSurfaceAttenuation = 0.5F; - const float Mist::LowestLightness = 0.3F; - const float Mist::VelocityHueMin = 240.0F; - const float Mist::VelocityHueRange = 120.0F; - const float Mist::VelocitySaturation = 1.0F; - const float Mist::StressHue = 230.0F; - const float Mist::StressSaturationRange = 0.5F; - const float Mist::StressSaturationMin = 0.5F; - - const float None::SurfaceNormalLightnessRange = 0.5F; - const float None::ParallelSurfaceAttenuation = 0.75F; - const float None::LowestLightness = 0.3F; - const float None::VelocityHueMin = 240.0F; - const float None::VelocityHueRange = 120.0F; - const float None::VelocitySaturation = 1.0F; - const float None::StressHue = 230.0F; - const float None::StressSaturationRange = 0.5F; - const float None::StressSaturationMin = 0.5F; - - const float Darkness::SurfaceNormalLightnessRange = 0.3F; - const float Darkness::ParallelSurfaceAttenuation = 0.5F; - const float Darkness::LowestLightness = 0.0F; - const float Darkness::VelocityHueMin = 240.0F; - const float Darkness::VelocityHueRange = 120.0F; - const float Darkness::VelocitySaturation = 1.0F; - const float Darkness::StressHue = 230.0F; - const float Darkness::StressSaturationRange = 0.5F; - const float Darkness::StressSaturationMin = 0.5F; - } - } - } - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() - { - MPI_Datatype ret = vis::raytracer::RayDataEnhanced::GetMpiType(); - HEMELB_MPI_CALL(MPI_Type_commit, (&ret)); - return ret; - } - } -} diff --git a/Code/vis/rayTracer/RayDataEnhanced.h b/Code/vis/rayTracer/RayDataEnhanced.h deleted file mode 100644 index c0f3c4313..000000000 --- a/Code/vis/rayTracer/RayDataEnhanced.h +++ /dev/null @@ -1,294 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_RAYDATAENHANCED_H -#define HEMELB_VIS_RAYTRACER_RAYDATAENHANCED_H - -#include "net/mpi.h" -#include "vis/DomainStats.h" -#include "vis/rayTracer/HSLToRGBConverter.h" -#include "vis/rayTracer/RayData.h" -#include "vis/VisSettings.h" -#include "util/Vector3D.h" -#include "lb/LbmParameters.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - namespace DepthCuing - { - class Mist - { - static const float SurfaceNormalLightnessRange; - static const float ParallelSurfaceAttenuation; - static const float LowestLightness; - static const float VelocityHueMin; - static const float VelocityHueRange; - static const float VelocitySaturation; - static const float StressHue; - static const float StressSaturationRange; - static const float StressSaturationMin; - - //Obtain the lightness value for the ray, based on - //the lightness obtained through surface normals - //and optional depth cuing - static float GetLightnessValue(const float normalisedDistance, - const float surfaceNormalLightness) - { - //Set the smallest lightness value to between - //the mimimum lightness and 1.0F based on the normalised distance between - //the viewpoint and the first cluster hit - //Add onto this the surface normal lightness - float lightnessValue = LowestLightness + (1.0F - LowestLightness) - * normalisedDistance + surfaceNormalLightness * SurfaceNormalLightnessRange; - - if (lightnessValue > 1.0F) - { - return 1.0F; - } - return lightnessValue; - } - }; - - class None - { - static const float SurfaceNormalLightnessRange; - static const float ParallelSurfaceAttenuation; - static const float LowestLightness; - static const float VelocityHueMin; - static const float VelocityHueRange; - static const float VelocitySaturation; - static const float StressHue; - static const float StressSaturationRange; - static const float StressSaturationMin; - - static float GetLightnessValue(const float normalisedDistance, - const float surfaceNormalLightness) - { - return LowestLightness + surfaceNormalLightness * SurfaceNormalLightnessRange; - } - }; - - class Darkness - { - static const float SurfaceNormalLightnessRange; - static const float ParallelSurfaceAttenuation; - static const float LowestLightness; - static const float VelocityHueMin; - static const float VelocityHueRange; - static const float VelocitySaturation; - static const float StressHue; - static const float StressSaturationRange; - static const float StressSaturationMin; - - static float GetLightnessValue(const float normalisedDistance, - const float surfaceNormalLightness) - { - //Set the maximum lightness to be between 0.8F and mLowestLighness - //based on the normalised distance and take off the surface normal - //lightness - float lightnessValue = 0.8F * (1.0F - normalisedDistance) + (surfaceNormalLightness - - 1.0F) * SurfaceNormalLightnessRange; - - if (lightnessValue < LowestLightness) - { - return LowestLightness; - } - return lightnessValue; - } - }; - } - - /** - * RayDataEnhanced - sums velocity and stress data of the ray trace and provides - * with surface normal highlighting and optional depth cuing to enhance the - * 3D perception. - * NB functions prefixed Do should only be called by the base class - */ - class RayDataEnhanced : public RayData - { - public: - RayDataEnhanced(int i, int j) : - RayData (i, j), mSurfaceNormalLightness(1.0F), mVelocitySum(0.0F), - mStressSum(0.0F) - { - } - - RayDataEnhanced() - { - - } - - //Processes the ray data for a normal (non wall) fluid site - void DoUpdateDataForNormalFluidSite(const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings) - { - //Add the velocity multiplied by the ray length in each voxel, - mVelocitySum += iSiteData.velocity * iRayLengthInVoxel; - - if (iVisSettings.mStressType == lb::VonMises) - { - //Update the volume rendering of the von Mises stress flow field - mStressSum = iSiteData.stress * iRayLengthInVoxel; - } - } - - //Processes the data for wall sites - template - void DoUpdateDataForWallSite(const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings, - const util::Vector3D* iWallNormal) - { - //Do everything that would be done for a normal fluid site - DoUpdateDataForNormalFluidSite(iSiteData, - iRayDirection, - iRayLengthInVoxel, - iVisSettings); - - double lDotProduct = iRayDirection.Dot(*iWallNormal); - - // Scale the surface normal lightness between mParallelSurfaceAttenuation - // and 1.0F - // Keep a copy for the special case - mLastSurfaceNormalLightnessMultiplier = (depthCuing::ParallelSurfaceAttenuation + (1.0F - - depthCuing::ParallelSurfaceAttenuation) * fabs(lDotProduct)); - - mSurfaceNormalLightness *= mLastSurfaceNormalLightnessMultiplier; - } - - void DoProcessTangentingVessel() - { - mSurfaceNormalLightness *= mLastSurfaceNormalLightnessMultiplier; - } - - //Obtains the colour representing the velocity ray trace - - template - void DoGetVelocityColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const - { - float lVelocityHue = depthCuing::VelocityHueRange * GetAverageVelocity() - * (float) (iDomainStats.velocity_threshold_max_inv) + depthCuing::VelocityHueMin; - - if (lVelocityHue >= 360.0F) - { - lVelocityHue = 0.0F; - } - - HSLToRGBConverter::Convert(lVelocityHue, - depthCuing::VelocitySaturation, - depthCuing::GetLightnessValue(iNormalisedDistanceToFirstCluster, - GetSurfaceNormalLightness()), - oColour); - } - - //Obtains the colour representing the stress ray trace - template - void DoGetStressColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const - { - float - lStressSaturation = - util::NumericalFunctions::enforceBounds(depthCuing::StressSaturationMin - + depthCuing::StressSaturationRange - * GetAverageStress() - * (float) (iDomainStats.stress_threshold_max_inv), - 0.0F, - 1.0F); - - HSLToRGBConverter::Convert(depthCuing::StressHue, - lStressSaturation, - depthCuing::GetLightnessValue(iNormalisedDistanceToFirstCluster, - GetSurfaceNormalLightness()), - oColour); - } - - //Carries out the merging of the ray data in this - //inherited type, for different segments of the same ray - void DoCombine(const RayDataEnhanced& iOtherRayData) - { - //Add together velocities and stress sums - mVelocitySum += iOtherRayData.GetVelocitySum(); - mStressSum += iOtherRayData.GetStressSum(); - //Multiply the surface lightness - mSurfaceNormalLightness *= iOtherRayData.GetSurfaceNormalLightness(); - } - - float GetVelocitySum() const - { - return mVelocitySum; - } - - float GetStressSum() const - { - return mStressSum; - } - - float GetSurfaceNormalLightness() const - { - return mSurfaceNormalLightness; - } - - float GetAverageVelocity() const - { - return GetVelocitySum() / RayData::GetCumulativeLengthInFluid(); - } - - float GetAverageStress() const - { - return GetStressSum() / RayData::GetCumulativeLengthInFluid(); - } - - static MPI_Datatype GetMpiType() - { - HEMELB_MPI_TYPE_BEGIN(type, RayDataEnhanced, 9); - - HEMELB_MPI_TYPE_ADD_MEMBER(i); - HEMELB_MPI_TYPE_ADD_MEMBER(j); - HEMELB_MPI_TYPE_ADD_MEMBER(mLengthBeforeRayFirstCluster); - HEMELB_MPI_TYPE_ADD_MEMBER(mCumulativeLengthInFluid); - HEMELB_MPI_TYPE_ADD_MEMBER(mDensityAtNearestPoint); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressAtNearestPoint); - HEMELB_MPI_TYPE_ADD_MEMBER(mSurfaceNormalLightness); - HEMELB_MPI_TYPE_ADD_MEMBER(mVelocitySum); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressSum); - - HEMELB_MPI_TYPE_END(type, RayDataEnhanced); - - return type; - } - - private: - float mSurfaceNormalLightness; - float mVelocitySum; - float mStressSum; - - // The last surface normal lightness multiplier - // is retained for the case of tangenting a vessel - // ie only passing through walls sites - // NB: Keep this last as it isn't sent over MPI - float mLastSurfaceNormalLightnessMultiplier; - }; - } - } - - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); - } -} - -#endif // HEMELB_VIS_RAYTRACER_RAYDATAENHANCED_H diff --git a/Code/vis/rayTracer/RayDataNormal.cc b/Code/vis/rayTracer/RayDataNormal.cc deleted file mode 100644 index 5d6b9d684..000000000 --- a/Code/vis/rayTracer/RayDataNormal.cc +++ /dev/null @@ -1,162 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include - -#include "util/Vector3D.h" -#include "vis/DomainStats.h" -#include "vis/rayTracer/RayDataNormal.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - - RayDataNormal::RayDataNormal(int i, int j) : - RayData (i, j) - { - mVelR = 0.0F; - mVelG = 0.0F; - mVelB = 0.0F; - - mStressR = 0.0F; - mStressG = 0.0F; - mStressB = 0.0F; - } - - RayDataNormal::RayDataNormal() - { - - } - - void RayDataNormal::DoUpdateDataForNormalFluidSite( - const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings) - { - float lPalette[3]; - - // update the volume rendering of the velocity flow field - PickColour(iSiteData.velocity * (float) mDomainStats->velocity_threshold_max_inv, lPalette); - - UpdateVelocityColour(iRayLengthInVoxel, lPalette); - - if (iVisSettings.mStressType != lb::ShearStress) - { - // update the volume rendering of the von Mises stress flow field - float lScaledStress = iSiteData.stress * (float) mDomainStats->stress_threshold_max_inv; - - PickColour(lScaledStress, lPalette); - - UpdateStressColour(iRayLengthInVoxel, lPalette); - } - } - - void RayDataNormal::DoUpdateDataForWallSite(const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings, - const util::Vector3D* iWallNormal) - { - DoUpdateDataForNormalFluidSite(iSiteData, iRayDirection, iRayLengthInVoxel, iVisSettings); - } - - void RayDataNormal::DoGetVelocityColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const - { - MakeColourComponent(mVelR * 255.0F, oColour[0]); - MakeColourComponent(mVelG * 255.0F, oColour[1]); - MakeColourComponent(mVelB * 255.0F, oColour[2]); - } - - void RayDataNormal::DoGetStressColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const - { - MakeColourComponent(mStressR, oColour[0]); - MakeColourComponent(mStressG, oColour[1]); - MakeColourComponent(mStressB, oColour[2]); - } - - void RayDataNormal::MakeColourComponent(float value, unsigned char& colour) const - { - colour - = util::NumericalFunctions::enforceBounds((unsigned char) (value - / GetCumulativeLengthInFluid()), - 0, - 255); - } - - void RayDataNormal::DoCombine(const RayDataNormal& iOtherRayData) - { - mVelR += iOtherRayData.mVelR; - mVelG += iOtherRayData.mVelG; - mVelB += iOtherRayData.mVelB; - - mStressR += iOtherRayData.mStressR; - mStressG += iOtherRayData.mStressG; - mStressB += iOtherRayData.mStressB; - } - - void RayDataNormal::UpdateVelocityColour(float iDt, const float iPalette[3]) - { - mVelR += iDt * iPalette[0]; - mVelG += iDt * iPalette[1]; - mVelB += iDt * iPalette[2]; - } - - void RayDataNormal::UpdateStressColour(float iDt, const float iPalette[3]) - { - mStressR += iDt * iPalette[0]; - mStressG += iDt * iPalette[1]; - mStressB += iDt * iPalette[2]; - } - - void RayDataNormal::DoProcessTangentingVessel() - { - } - - MPI_Datatype RayDataNormal::GetMPIType() - { - HEMELB_MPI_TYPE_BEGIN(type, RayDataNormal, 12); - - HEMELB_MPI_TYPE_ADD_MEMBER(i); - HEMELB_MPI_TYPE_ADD_MEMBER(j); - HEMELB_MPI_TYPE_ADD_MEMBER(mLengthBeforeRayFirstCluster); - HEMELB_MPI_TYPE_ADD_MEMBER(mCumulativeLengthInFluid); - HEMELB_MPI_TYPE_ADD_MEMBER(mDensityAtNearestPoint); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressAtNearestPoint); - HEMELB_MPI_TYPE_ADD_MEMBER(mVelR); - HEMELB_MPI_TYPE_ADD_MEMBER(mVelG); - HEMELB_MPI_TYPE_ADD_MEMBER(mVelB); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressR); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressG); - HEMELB_MPI_TYPE_ADD_MEMBER(mStressB); - - HEMELB_MPI_TYPE_END(type, RayDataNormal); - return type; - } - - const DomainStats* RayDataNormal::mDomainStats = NULL; - } - } - - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() - { - MPI_Datatype ret = vis::raytracer::RayDataNormal::GetMPIType(); - HEMELB_MPI_CALL(MPI_Type_commit, (&ret)); - return ret; - } - } -} - diff --git a/Code/vis/rayTracer/RayDataNormal.h b/Code/vis/rayTracer/RayDataNormal.h deleted file mode 100644 index 4131df225..000000000 --- a/Code/vis/rayTracer/RayDataNormal.h +++ /dev/null @@ -1,83 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_RAYDATANORMAL_H -#define HEMELB_VIS_RAYTRACER_RAYDATANORMAL_H - -#include "vis/DomainStats.h" -#include "vis/rayTracer/RayData.h" -#include "vis/VisSettings.h" -#include "util/Vector3D.h" -#include "lb/LbmParameters.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - // NB functions prefixed Do should only be called by the base class - class RayDataNormal : public RayData - { - public: - RayDataNormal(int i, int j); - RayDataNormal(); - - // Used to process the ray data for a normal (non-wall) fluid site - void DoUpdateDataForNormalFluidSite(const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings); - - // Used to process the ray data for wall site - void DoUpdateDataForWallSite(const SiteData_t& iSiteData, - const util::Vector3D& iRayDirection, - const float iRayLengthInVoxel, - const VisSettings& iVisSettings, - const util::Vector3D* iWallNormal); - - // Carries out the merging of the ray data in this - // inherited type, for different segments of the same ray - void DoCombine(const RayDataNormal& iOtherRayData); - - //Obtains the colour representing the velocity ray trace - void DoGetVelocityColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const; - - // Obtains the colour representing the stress ray trace - void DoGetStressColour(unsigned char oColour[3], - const float iNormalisedDistanceToFirstCluster, - const DomainStats& iDomainStats) const; - - void DoProcessTangentingVessel(); - - static MPI_Datatype GetMPIType(); - - // We need this because RayDataNormal uses it for every voxel update - static const DomainStats* mDomainStats; - - private: - void UpdateVelocityColour(float iDt, const float iPalette[3]); - - void UpdateStressColour(float iDt, const float iPalette[3]); - - void MakeColourComponent(float value, unsigned char& colour) const; - - float mVelR, mVelG, mVelB; - float mStressR, mStressG, mStressB; - }; - } - } - - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); - } -} - -#endif // HEMELB_VIS_RAYTRACER_RAYDATANORMAL_H diff --git a/Code/vis/rayTracer/RayTracer.h b/Code/vis/rayTracer/RayTracer.h deleted file mode 100644 index 7d1f8f896..000000000 --- a/Code/vis/rayTracer/RayTracer.h +++ /dev/null @@ -1,100 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_RAYTRACER_H -#define HEMELB_VIS_RAYTRACER_RAYTRACER_H - -#include -#include -#include -#include -#include -#include - -#include "constants.h" -#include "debug/Debugger.h" -#include "geometry/LatticeData.h" -#include "lb/LbmParameters.h" -#include "log/Logger.h" -#include "net/IOCommunicator.h" -#include "util/utilityFunctions.h" -#include "util/Vector3D.h" -#include "vis/DomainStats.h" -#include "vis/PixelSet.h" -#include "vis/PixelSetStore.h" -#include "vis/Screen.h" -#include "vis/Viewpoint.h" -#include "vis/VisSettings.h" -#include "vis/XYCoordinates.h" -#include "vis/rayTracer/Cluster.h" -#include "vis/rayTracer/ClusterBuilder.h" -#include "vis/rayTracer/ClusterRayTracer.h" -#include "vis/rayTracer/Ray.h" -#include "vis/rayTracer/RayTracer.h" -#include "vis/rayTracer/SiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - template - class RayTracer : public PixelSetStore > - { - public: - // Constructor and destructor do all the usual stuff. - RayTracer(const geometry::LatticeData* iLatDat, - const DomainStats* iDomainStats, - Screen* iScreen, - Viewpoint* iViewpoint, - VisSettings* iVisSettings) : - mClusterBuilder(iLatDat, iLatDat->GetLocalRank()), mLatDat(iLatDat), mDomainStats(iDomainStats), - mScreen(iScreen), mViewpoint(iViewpoint), mVisSettings(iVisSettings) - { - mClusterBuilder.BuildClusters(); - } - - ~RayTracer() - { - } - - // Render the current state into an image. - PixelSet* Render(const lb::MacroscopicPropertyCache& propertyCache) - { - PixelSet* pixels = - PixelSetStore >::GetUnusedPixelSet(); - pixels->Clear(); - - ClusterRayTracer lClusterRayTracer(*mViewpoint, - *mScreen, - *mDomainStats, - *mVisSettings, - *mLatDat, - propertyCache); - - for (unsigned int clusterId = 0; clusterId < mClusterBuilder.GetClusters().size(); clusterId++) - { - lClusterRayTracer.RenderCluster(mClusterBuilder.GetClusters()[clusterId], *pixels); - } - - return pixels; - } - - private: - ClusterBuilder mClusterBuilder; - const geometry::LatticeData* mLatDat; - - const DomainStats* mDomainStats; - Screen* mScreen; - Viewpoint* mViewpoint; - VisSettings* mVisSettings; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_RAYTRACER_H diff --git a/Code/vis/rayTracer/SiteData.h b/Code/vis/rayTracer/SiteData.h deleted file mode 100644 index f698e0b60..000000000 --- a/Code/vis/rayTracer/SiteData.h +++ /dev/null @@ -1,28 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_RAYTRACER_SITEDATA_H -#define HEMELB_VIS_RAYTRACER_SITEDATA_H - -namespace hemelb -{ - namespace vis - { - namespace raytracer - { - //Stores the data about an individual voxel - struct SiteData_t - { - public: - float density; - float velocity; - float stress; - }; - } - } -} - -#endif // HEMELB_VIS_RAYTRACER_SITEDATA_H diff --git a/Code/vis/streaklineDrawer/NeighbouringProcessor.cc b/Code/vis/streaklineDrawer/NeighbouringProcessor.cc deleted file mode 100644 index b3abadc4d..000000000 --- a/Code/vis/streaklineDrawer/NeighbouringProcessor.cc +++ /dev/null @@ -1,167 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include - -#include "constants.h" -#include "vis/streaklineDrawer/NeighbouringProcessor.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - - NeighbouringProcessor::NeighbouringProcessor() - { - } - - NeighbouringProcessor::NeighbouringProcessor(proc_t neighbourRankIn) : - neighbourRank(neighbourRankIn) - { - } - - void NeighbouringProcessor::AddParticleToSend(const Particle& particle) - { - particlesToSend.push_back(particle); - } - - bool NeighbouringProcessor::ParticlesToBeRetrieved() - { - return (particlesToReceive.size() > 0); - } - - const Particle& NeighbouringProcessor::PopNextReceivedParticle() - { - const Particle& lParticle = particlesToReceive.back(); - particlesToReceive.pop_back(); - return lParticle; - } - - void NeighbouringProcessor::ExchangeParticleCounts(net::Net& net) - { - numberOfParticlesToSend = particlesToSend.size(); - net.RequestSendR(numberOfParticlesToSend, neighbourRank); - net.RequestReceiveR(numberOfParticlesToReceive, neighbourRank); - } - - void NeighbouringProcessor::ClearParticleSendingList() - { - particlesToSend.clear(); - } - - void NeighbouringProcessor::ExchangeParticles(net::Net& net) - { - if (numberOfParticlesToReceive > 0) - { - particlesToReceive.resize(numberOfParticlesToReceive); - - net.RequestReceiveV(particlesToReceive, - neighbourRank); - } - - if (particlesToSend.size() > 0) - { - net.RequestSendV(particlesToSend, neighbourRank); //Request - } - } - - void NeighbouringProcessor::AddSiteToRequestVelocityDataFor(site_t siteI, - site_t siteJ, - site_t siteK) - { - siteCoordsRequestedByThisCore.push_back(util::Vector3D(siteI, siteJ, siteK)); - } - - void NeighbouringProcessor::ExchangeSiteIdCounts(net::Net& net) - { - numberOfSitesRequestedByThisCore = siteCoordsRequestedByThisCore.size(); - - net.RequestReceiveR(numberOfSiteBeingRequestedByNeighbour, neighbourRank); - net.RequestSendR(numberOfSitesRequestedByThisCore, neighbourRank); - } - - void NeighbouringProcessor::ExchangeSiteIds(net::Net& net) - { - if (numberOfSiteBeingRequestedByNeighbour > 0) - { - siteCoordsRequestedByNeighbour.resize(numberOfSiteBeingRequestedByNeighbour); - velocityFieldDataForNeighbour.resize(numberOfSiteBeingRequestedByNeighbour); - - net.RequestReceive(&siteCoordsRequestedByNeighbour[0], - (int) numberOfSiteBeingRequestedByNeighbour, - neighbourRank); - } - - if (numberOfSitesRequestedByThisCore > 0) - { - velocityFieldDataFromNeighbour.resize(numberOfSitesRequestedByThisCore); - - net.RequestSendV(siteCoordsRequestedByThisCore, - neighbourRank); - } - } - - void NeighbouringProcessor::ExchangeVelocitiesForRequestedSites(net::Net& net) - { - if (numberOfSitesRequestedByThisCore > 0) - { - velocityFieldDataFromNeighbour.resize(numberOfSitesRequestedByThisCore); - - net.RequestReceiveV(velocityFieldDataFromNeighbour, - neighbourRank); - } - - if (numberOfSiteBeingRequestedByNeighbour > 0) - { - velocityFieldDataForNeighbour.resize(numberOfSiteBeingRequestedByNeighbour); - - net.RequestSendV(velocityFieldDataForNeighbour, - neighbourRank); - } - } - - site_t NeighbouringProcessor::GetNumberOfSitesRequestedByNeighbour() const - { - return siteCoordsRequestedByNeighbour.size(); - } - - const util::Vector3D& NeighbouringProcessor::GetReceivedVelocityField(const site_t receivedIndex) const - { - return velocityFieldDataFromNeighbour[receivedIndex]; - } - - const util::Vector3D& NeighbouringProcessor::GetSiteCoordsBeingRequestedByNeighbour(const site_t receivedIndex) const - { - return siteCoordsRequestedByNeighbour[receivedIndex]; - } - - void NeighbouringProcessor::SetVelocityFieldToSend(const site_t sendIndex, - const util::Vector3D& velocityFieldToSend) - { - velocityFieldDataForNeighbour[sendIndex] = velocityFieldToSend; - } - - site_t NeighbouringProcessor::GetNumberOfSitesRequestedByThisCore() const - { - return siteCoordsRequestedByThisCore.size(); - } - - const util::Vector3D& NeighbouringProcessor::GetSendingSiteCoorinates(site_t sendIndex) const - { - return siteCoordsRequestedByThisCore[sendIndex]; - } - - void NeighbouringProcessor::ClearListOfRequestedSites() - { - siteCoordsRequestedByThisCore.clear(); - } - - } - } -} diff --git a/Code/vis/streaklineDrawer/NeighbouringProcessor.h b/Code/vis/streaklineDrawer/NeighbouringProcessor.h deleted file mode 100644 index d1b406f82..000000000 --- a/Code/vis/streaklineDrawer/NeighbouringProcessor.h +++ /dev/null @@ -1,77 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_NEIGHBOURINGPROCESSOR_H -#define HEMELB_VIS_STREAKLINEDRAWER_NEIGHBOURINGPROCESSOR_H - -#include - -#include "net/net.h" -#include "util/Vector3D.h" -#include "vis/streaklineDrawer/Particle.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - class NeighbouringProcessor - { - public: - // Constructors - NeighbouringProcessor(); - NeighbouringProcessor(proc_t iID); - - // Functions for communicating particles. - void AddParticleToSend(const Particle& iParticle); - bool ParticlesToBeRetrieved(); - const Particle& PopNextReceivedParticle(); - void ClearParticleSendingList(); - - void ExchangeParticleCounts(net::Net& net); - void ExchangeParticles(net::Net& net); - - // Functions for communicating velocity data. - void AddSiteToRequestVelocityDataFor(site_t, site_t, site_t); - site_t GetNumberOfSitesRequestedByNeighbour() const; - const util::Vector3D& GetReceivedVelocityField(const site_t receivedIndex) const; - const util::Vector3D - & GetSiteCoordsBeingRequestedByNeighbour(const site_t receivedIndex) const; - void SetVelocityFieldToSend(const site_t sendIndex, - const util::Vector3D& velocityFieldToSend); - - const util::Vector3D& GetSendingSiteCoorinates(site_t sendIndex) const; - site_t GetNumberOfSitesRequestedByThisCore() const; - void ClearListOfRequestedSites(); - - void ExchangeSiteIdCounts(net::Net& net); - void ExchangeSiteIds(net::Net& net); - void ExchangeVelocitiesForRequestedSites(net::Net& net); - - private: - site_t numberOfParticlesToSend; - std::vector particlesToSend; - - site_t numberOfParticlesToReceive; - std::vector particlesToReceive; - - site_t numberOfSiteBeingRequestedByNeighbour; - site_t numberOfSitesRequestedByThisCore; - - std::vector > siteCoordsRequestedByThisCore; - std::vector > siteCoordsRequestedByNeighbour; - - std::vector > velocityFieldDataForNeighbour; - std::vector > velocityFieldDataFromNeighbour; - - proc_t neighbourRank; - }; - } - } -} - -#endif // HEMELB_VIS_STREAKLINEDRAWER_NEIGHBOURINGPROCESSOR_H diff --git a/Code/vis/streaklineDrawer/Particle.cc b/Code/vis/streaklineDrawer/Particle.cc deleted file mode 100644 index a0b2570b8..000000000 --- a/Code/vis/streaklineDrawer/Particle.cc +++ /dev/null @@ -1,47 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "net/mpi.h" -#include "vis/streaklineDrawer/Particle.h" -#include "constants.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - Particle::Particle() : - position(NO_VALUE), velocity(NO_VALUE), vel(NO_VALUE), inletID() - { - - } - - Particle::Particle(float iX, float iY, float iZ, unsigned int iInletId) : - position(iX, iY, iZ), velocity(0), vel(0), inletID(iInletId) - { - } - } - } - - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() - { - HEMELB_MPI_TYPE_BEGIN(type, vis::streaklinedrawer::Particle, 3); - - HEMELB_MPI_TYPE_ADD_MEMBER(position); - HEMELB_MPI_TYPE_ADD_MEMBER(vel); - HEMELB_MPI_TYPE_ADD_MEMBER(inletID); - - HEMELB_MPI_TYPE_END(type, vis::streaklinedrawer::Particle); - - HEMELB_MPI_CALL(MPI_Type_commit, (&type)); - return type; - } - } -} diff --git a/Code/vis/streaklineDrawer/Particle.h b/Code/vis/streaklineDrawer/Particle.h deleted file mode 100644 index b48a8b0bd..000000000 --- a/Code/vis/streaklineDrawer/Particle.h +++ /dev/null @@ -1,40 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_PARTICLE_H -#define HEMELB_VIS_STREAKLINEDRAWER_PARTICLE_H - -#include "util/Vector3D.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - struct Particle - { - public: - Particle(); - - Particle(float iX, float iY, float iZ, unsigned int iInletId); - - util::Vector3D position; - util::Vector3D velocity; - float vel; - unsigned int inletID; - }; - - } - } - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); - } -} - -#endif // HEMELB_VIS_STREAKLINEDRAWER_PARTICLE_H diff --git a/Code/vis/streaklineDrawer/ParticleManager.cc b/Code/vis/streaklineDrawer/ParticleManager.cc deleted file mode 100644 index 2dc3cc3b2..000000000 --- a/Code/vis/streaklineDrawer/ParticleManager.cc +++ /dev/null @@ -1,121 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include - -#include "vis/streaklineDrawer/ParticleManager.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - ParticleManager::ParticleManager(std::map& iNeighbouringProcessors) : - neighbouringProcessors(iNeighbouringProcessors) - { - } - - void ParticleManager::AddParticle(const Particle& iParticle) - { - particles.push_back(iParticle); - } - - std::vector& ParticleManager::GetParticles() - { - return particles; - } - - size_t ParticleManager::GetNumberOfLocalParticles() const - { - return particles.size(); - } - - void ParticleManager::DeleteParticle(site_t iIndex) - { - assert(particles.size() > static_cast (iIndex)); - - //Move the particle at the end to position - particles[iIndex] = particles.back(); - - //Delete the now duplicated particle at the end - particles.pop_back(); - } - - void ParticleManager::DeleteAll() - { - particles.clear(); - } - - void ParticleManager::ProcessParticleMovement() - { - for (unsigned int i = 0; i < GetNumberOfLocalParticles(); i++) - { - // particle coords updating (dt = 1) - particles[i].position += particles[i].velocity; - } - } - - // Communicate the particles' current state to other processors. - void ParticleManager::CommunicateParticles(net::Net& streakNet, - const geometry::LatticeData& latticeData, - VelocityField& velocityField) - { - unsigned int particles_temp = GetNumberOfLocalParticles(); - - proc_t thisRank = streakNet.Rank(); - - for (int n = (int) (particles_temp - 1); n >= 0; n--) - { - VelocitySiteData* siteVelocityData = - velocityField.GetVelocitySiteData(latticeData, - util::Vector3D(particles[n].position)); - - // TODO can we get rid of the first test? - if (siteVelocityData == NULL || siteVelocityData->proc_id == -1) - { - continue; - } - else if (thisRank == siteVelocityData->proc_id) - { - continue; - } - - neighbouringProcessors[siteVelocityData->proc_id].AddParticleToSend(particles[n]); - DeleteParticle(n); - } - - for (std::map::iterator proc = - neighbouringProcessors.begin(); proc != neighbouringProcessors.end(); ++proc) - { - (*proc).second.ExchangeParticleCounts(streakNet); - } - streakNet.Dispatch(); - - for (std::map::iterator proc = - neighbouringProcessors.begin(); proc != neighbouringProcessors.end(); ++proc) - { - (*proc).second.ExchangeParticles(streakNet); - } - - streakNet.Dispatch(); - - for (std::map::iterator proc = - neighbouringProcessors.begin(); proc != neighbouringProcessors.end(); ++proc) - { - NeighbouringProcessor& neighbourProc = (*proc).second; - neighbourProc.ClearParticleSendingList(); - - while (neighbourProc.ParticlesToBeRetrieved()) - { - AddParticle(neighbourProc.PopNextReceivedParticle()); - } - } - } - - } - } -} diff --git a/Code/vis/streaklineDrawer/ParticleManager.h b/Code/vis/streaklineDrawer/ParticleManager.h deleted file mode 100644 index f20bc9f4b..000000000 --- a/Code/vis/streaklineDrawer/ParticleManager.h +++ /dev/null @@ -1,60 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_PARTICLEMANAGER_H -#define HEMELB_VIS_STREAKLINEDRAWER_PARTICLEMANAGER_H - -#include -#include - -#include "constants.h" -#include "net/mpi.h" - -#include "geometry/LatticeData.h" -#include "net/IOCommunicator.h" - -#include "vis/streaklineDrawer/NeighbouringProcessor.h" -#include "vis/streaklineDrawer/Particle.h" -#include "vis/streaklineDrawer/VelocityField.h" -#include "vis/streaklineDrawer/VelocitySiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - class ParticleManager - { - public: - // Constructor - ParticleManager(std::map& iNeighbouringProcessors); - - // Functions for manipulating the particle store - void AddParticle(const Particle& iParticle); - std::vector& GetParticles(); - size_t GetNumberOfLocalParticles() const; - void DeleteParticle(site_t iIndex); - void DeleteAll(); - - // Function for updating the particles' positions. - void ProcessParticleMovement(); - - // Function for moving the particles between cores. - void CommunicateParticles(net::Net& streakNet, - const geometry::LatticeData& iLatDat, - VelocityField& iVelocityField); - - private: - std::vector particles; - std::map& neighbouringProcessors; - - }; - } - } -} - -#endif // HEMELB_VIS_STREAKLINEDRAWER_PARTICLEMANAGER_H diff --git a/Code/vis/streaklineDrawer/StreakPixel.cc b/Code/vis/streaklineDrawer/StreakPixel.cc deleted file mode 100644 index 7c1dc43d4..000000000 --- a/Code/vis/streaklineDrawer/StreakPixel.cc +++ /dev/null @@ -1,21 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include "vis/streaklineDrawer/StreakPixel.h" - -namespace hemelb -{ - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType() - { - MPI_Datatype ret = vis::streaklinedrawer::StreakPixel::GetMPIType(); - MPI_Type_commit(&ret); - return ret; - } - } -} diff --git a/Code/vis/streaklineDrawer/StreakPixel.h b/Code/vis/streaklineDrawer/StreakPixel.h deleted file mode 100644 index 6575dde92..000000000 --- a/Code/vis/streaklineDrawer/StreakPixel.h +++ /dev/null @@ -1,101 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_STREAKPIXEL_H -#define HEMELB_VIS_STREAKLINEDRAWER_STREAKPIXEL_H - -#include "log/Logger.h" -#include "vis/BasicPixel.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - class StreakPixel : public BasicPixel - { - public: - StreakPixel() - { - - } - - StreakPixel(int i, int j, float particleVelocity, float particleZ, int particleInlet) : - BasicPixel(i, j), particle_vel(particleVelocity), particle_z(particleZ), particle_inlet_id(particleInlet) - { - - } - - void Combine(const StreakPixel& inPixel) - { - if (inPixel.particle_z < particle_z) - { - particle_z = inPixel.particle_z; - particle_vel = inPixel.particle_vel; - particle_inlet_id = inPixel.particle_inlet_id; - } - } - - float GetParticleVelocity() const - { - return particle_vel; - } - - /** - * Produces an MPI Datatype object but doesn't commit it or manage its memory. - * @return - */ - static MPI_Datatype GetMPIType() - { - const int typeCount = 5; - int blocklengths[typeCount] = { 1, 1, 1, 1, 1 }; - - MPI_Datatype types[typeCount] = { MPI_INT, MPI_INT, MPI_FLOAT, MPI_FLOAT, MPI_INT }; - - StreakPixel example; - - MPI_Aint displacements[typeCount]; - - MPI_Get_address(&example.i, &displacements[0]); - MPI_Get_address(&example.j, &displacements[1]); - MPI_Get_address(&example.particle_vel, &displacements[2]); - MPI_Get_address(&example.particle_z, &displacements[3]); - MPI_Get_address(&example.particle_inlet_id, &displacements[4]); - - for (int ii = typeCount - 1; ii >= 0; --ii) - { - displacements[ii] -= displacements[0]; - } - - MPI_Datatype ret; - - MPI_Type_create_struct(typeCount, blocklengths, displacements, types, &ret); - - return ret; - } - - void LogDebuggingInformation() const - { - log::Logger::Log("Streak pixel at (%i,%i) with " - "(source inlet, velocity, z) = (%d, %f, %f)", GetI(), GetJ(), particle_inlet_id, particle_vel, particle_z); - } - - private: - float particle_vel; - float particle_z; - int particle_inlet_id; - }; - } - } - namespace net - { - template<> - MPI_Datatype MpiDataTypeTraits::RegisterMpiDataType(); - } -} - -#endif /* HEMELB_VIS_STREAKLINEDRAWER_STREAKPIXEL_H */ diff --git a/Code/vis/streaklineDrawer/StreaklineDrawer.cc b/Code/vis/streaklineDrawer/StreaklineDrawer.cc deleted file mode 100644 index ed0316fdc..000000000 --- a/Code/vis/streaklineDrawer/StreaklineDrawer.cc +++ /dev/null @@ -1,351 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include - -#include "geometry/BlockTraverser.h" -#include "geometry/SiteTraverser.h" -#include "util/utilityFunctions.h" -#include "vis/Control.h" -#include "vis/streaklineDrawer/StreaklineDrawer.h" -#include "vis/streaklineDrawer/StreakPixel.h" -#include "vis/XYCoordinates.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - // Constructor, populating fields from lattice data objects. - StreaklineDrawer::StreaklineDrawer(const geometry::LatticeData& iLatDat, - const Screen& iScreen, - const Viewpoint& iViewpoint, - const VisSettings& iVisSettings, - const lb::MacroscopicPropertyCache& propertyCache, - const net::MpiCommunicator& comms) : - latDat(iLatDat), screen(iScreen), viewpoint(iViewpoint), visSettings(iVisSettings), - // propertyCache(propertyCache), - particleManager(neighbouringProcessors), - velocityField(comms.Rank(), neighbouringProcessors, propertyCache), - streakNet(new net::Net(comms)) - { - velocityField.BuildVelocityField(iLatDat); - ChooseSeedParticles(); - } - - // Destructor - StreaklineDrawer::~StreaklineDrawer() - { - delete streakNet; - } - - // Reset the streakline drawer. - void StreaklineDrawer::Restart() - { - particleManager.DeleteAll(); - } - - // Create streakline particles and move them. - void StreaklineDrawer::ProgressStreaklines(unsigned long time_steps, unsigned long total_time_steps) - { - // Set the particle creation period to be every time step, unless there are >=10000 - // timesteps - unsigned int particle_creation_period = - util::NumericalFunctions::max(1, (unsigned int) (total_time_steps / 5000)); - - int timestepsBetweenStreaklinesRounded = (int) (0.5F - + (float) total_time_steps / visSettings.streaklines_per_simulation); - - if ((float) (time_steps % timestepsBetweenStreaklinesRounded) - <= (visSettings.streakline_length / 100.0F) - * ((float) total_time_steps / visSettings.streaklines_per_simulation) - && time_steps % particle_creation_period == 0) - { - CreateParticlesFromSeeds(); - } - - velocityField.InvalidateAllCalculatedVelocities(); - - // Decide which sites we will need velocity data for in order to move particles - WorkOutVelocityDataNeededForParticles(); - - // Communicate this with other processors. - CommunicateSiteIds(); - - // Recalculate velocities at the sites requested from this rank. - UpdateVelocityFieldForCommunicatedSites(); - - // Communicate the velocities back to the sites that requested them. - CommunicateVelocities(); - - // Update our local velocity field is sufficient to update all particles and prune - // those that are stationary. - UpdateVelocityFieldForAllParticlesAndPrune(); - - // Process the particles' movement. - particleManager.ProcessParticleMovement(); - - // Communicate any particles that have crossed into the territory of another rank. - particleManager.CommunicateParticles(*streakNet, latDat, velocityField); - } - - // Render the streaklines - PixelSet* StreaklineDrawer::Render() - { - int pixels_x = screen.GetPixelsX(); - int pixels_y = screen.GetPixelsY(); - - const std::vector& particles = particleManager.GetParticles(); - - PixelSet* set = GetUnusedPixelSet(); - set->Clear(); - - for (unsigned int n = 0; n < particles.size(); n++) - { - util::Vector3D p1 = particles[n].position - util::Vector3D(latDat.GetSiteDimensions() / 2); - - util::Vector3D p2 = viewpoint.Project(p1); - - XYCoordinates x = screen.TransformScreenToPixelCoordinates(XYCoordinates(p2.x, p2.y)); - - if (! (x.x < 0 || x.x >= pixels_x || x.y < 0 || x.y >= pixels_y)) - { - StreakPixel pixel(x.x, x.y, particles[n].vel, p2.z, particles[n].inletID); - set->AddPixel(pixel); - } - } - - return set; - } - - void StreaklineDrawer::ChooseSeedParticles() - { - site_t inlet_sites = 0; - - geometry::BlockTraverser blockTraverser(latDat); - do - { - const geometry::Block& block = blockTraverser.GetCurrentBlockData(); - - if (block.IsEmpty()) - { - continue; - } - - geometry::SiteTraverser siteTraverser(latDat); - do - { - if (this->streakNet->Rank() - != block.GetProcessorRankForSite(siteTraverser.GetCurrentIndex())) - { - continue; - } - - const geometry::Site site = - latDat.GetSite(block.GetLocalContiguousIndexForSite(siteTraverser.GetCurrentIndex())); - - // if the lattice site is not an inlet - if (site.GetSiteType() != geometry::INLET_TYPE) - { - continue; - } - ++inlet_sites; - - // TODO this is a problem on multiple cores. - if (inlet_sites % 50 != 0) - { - continue; - } - - particleSeeds.push_back(Particle(static_cast(blockTraverser.GetX() * blockTraverser.GetBlockSize() - + siteTraverser.GetX()), - static_cast(blockTraverser.GetY() * blockTraverser.GetBlockSize() - + siteTraverser.GetY()), - static_cast(blockTraverser.GetZ() * blockTraverser.GetBlockSize() - + siteTraverser.GetZ()), - site.GetIoletId())); - - } - while (siteTraverser.TraverseOne()); - } - while (blockTraverser.TraverseOne()); - } - - // Create seed particles to begin the streaklines. - void StreaklineDrawer::CreateParticlesFromSeeds() - { - for (unsigned int n = 0; n < particleSeeds.size(); n++) - { - particleManager.AddParticle(particleSeeds[n]); - } - } - - void StreaklineDrawer::WorkOutVelocityDataNeededForParticles() - { - std::vector &particles = particleManager.GetParticles(); - - // Note that we iterate through the array back to front because at the end of this - // loop we delete a particle. Iterating back to front ensures that we end up - // visiting each particle. - for (int n = (int) particles.size() - 1; n >= 0; --n) - { - Particle& particle = particles[n]; - - for (int unitGridI = 0; unitGridI <= 1; ++unitGridI) - { - site_t neighbourI = (site_t) particle.position.x + unitGridI; - - for (int unitGridJ = 0; unitGridJ <= 1; ++unitGridJ) - { - site_t neighbourJ = (site_t) particle.position.y + unitGridJ; - - for (int unitGridK = 0; unitGridK <= 1; ++unitGridK) - { - site_t neighbourK = (site_t) particle.position.z + unitGridK; - - proc_t sourceProcessor; - - if (velocityField.NeededFromNeighbour(util::Vector3D(neighbourI, neighbourJ, neighbourK), - latDat, - &sourceProcessor)) - { - neighbouringProcessors[sourceProcessor].AddSiteToRequestVelocityDataFor(neighbourI, - neighbourJ, - neighbourK); - } - } - } - } - } - } - - void StreaklineDrawer::UpdateVelocityFieldForAllParticlesAndPrune() - { - std::vector &particles = particleManager.GetParticles(); - - // Note that we iterate through the array back to front because at the end of this - // loop we delete a particle. Iterating back to front ensures that we end up - // visiting each particle. - for (int n = (int) particles.size() - 1; n >= 0; --n) - { - util::Vector3D localVelocityField[2][2][2]; - - velocityField.GetVelocityFieldAroundPoint(util::Vector3D(particles[n].position), - latDat, - localVelocityField); - - util::Vector3D interp_v = velocityField.InterpolateVelocityForPoint(particles[n].position, - localVelocityField); - - float vel = interp_v.Dot(interp_v); - - if (vel > 1.0F) - { - particles[n].vel = 1.0F; - particles[n].velocity = interp_v * float(1.0 / sqrtf(vel)); - } - else if (vel > 1.0e-8) - { - particles[n].vel = sqrtf(vel); - particles[n].velocity = interp_v; - } - else - { - particleManager.DeleteParticle(n); - } - } - } - - void StreaklineDrawer::UpdateVelocityFieldForCommunicatedSites() - { - for (std::map::const_iterator neighProc = neighbouringProcessors.begin(); - neighProc != neighbouringProcessors.end(); ++neighProc) - { - const NeighbouringProcessor& proc = (*neighProc).second; - - for (site_t sendingVelocityIndex = 0; sendingVelocityIndex < proc.GetNumberOfSitesRequestedByNeighbour(); - sendingVelocityIndex++) - { - const util::Vector3D& siteCoords = - proc.GetSiteCoordsBeingRequestedByNeighbour(sendingVelocityIndex); - - velocityField.UpdateLocalField(siteCoords, latDat); - } - } - } - - // Communicate site ids to other processors. - void StreaklineDrawer::CommunicateSiteIds() - { - for (std::map::iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); ++proc) - { - (*proc).second.ExchangeSiteIdCounts(*streakNet); - } - - streakNet->Dispatch(); - - for (std::map::iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); proc++) - { - (*proc).second.ExchangeSiteIds(*streakNet); - } - - streakNet->Dispatch(); - } - - // Communicate velocities to other processors. - void StreaklineDrawer::CommunicateVelocities() - { - - for (std::map::iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); ++proc) - { - (*proc).second.ExchangeVelocitiesForRequestedSites(*streakNet); - } - - for (std::map::iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); ++proc) - { - NeighbouringProcessor& neighbourProc = (*proc).second; - - for (site_t n = 0; n < neighbourProc.GetNumberOfSitesRequestedByNeighbour(); ++n) - { - const util::Vector3D siteCoords = neighbourProc.GetSiteCoordsBeingRequestedByNeighbour(n); - - const VelocitySiteData* velocityDataForSite = velocityField.GetVelocitySiteData(latDat, siteCoords); - - neighbourProc.SetVelocityFieldToSend(n, velocityDataForSite->velocity); - } - } - - streakNet->Dispatch(); - - for (std::map::const_iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); ++proc) - { - const NeighbouringProcessor& neighbourProc = (*proc).second; - - for (site_t n = 0; n < neighbourProc.GetNumberOfSitesRequestedByThisCore(); n++) - { - const util::Vector3D &coords = neighbourProc.GetSendingSiteCoorinates(n); - velocityField.GetVelocitySiteData(latDat, coords)->velocity = neighbourProc.GetReceivedVelocityField(n); - } - } - - for (std::map::iterator proc = neighbouringProcessors.begin(); - proc != neighbouringProcessors.end(); ++proc) - { - (*proc).second.ClearListOfRequestedSites(); - } - } - - } - } -} diff --git a/Code/vis/streaklineDrawer/StreaklineDrawer.h b/Code/vis/streaklineDrawer/StreaklineDrawer.h deleted file mode 100644 index a47136bbe..000000000 --- a/Code/vis/streaklineDrawer/StreaklineDrawer.h +++ /dev/null @@ -1,90 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_STREAKLINEDRAWER_H -#define HEMELB_VIS_STREAKLINEDRAWER_STREAKLINEDRAWER_H - -#include -#include - -#include "constants.h" -#include "net/mpi.h" - -#include "debug/Debugger.h" -#include "geometry/LatticeData.h" -#include "net/IOCommunicator.h" -#include "vis/PixelSet.h" -#include "vis/PixelSetStore.h" -#include "vis/streaklineDrawer/NeighbouringProcessor.h" -#include "vis/streaklineDrawer/ParticleManager.h" -#include "vis/streaklineDrawer/VelocityField.h" -#include "vis/streaklineDrawer/VelocitySiteData.h" -#include "vis/Screen.h" -#include "vis/streaklineDrawer/StreakPixel.h" -#include "vis/Viewpoint.h" -#include "vis/VisSettings.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - /** - * Class that controls the drawing of streaklines - lines that trace - * the path of an imaginary particle were it dropped into the fluid. - */ - class StreaklineDrawer : public PixelSetStore > - { - public: - // Constructor and destructor. - StreaklineDrawer(const geometry::LatticeData& iLatDat, - const Screen& iScreen, - const Viewpoint& iViewpoint, - const VisSettings& iVisSettings, - const lb::MacroscopicPropertyCache& propertyCache, - const net::MpiCommunicator& comms); - ~StreaklineDrawer(); - - // Method to reset streakline drawer - void Restart(); - - // Drawing methods. - void ProgressStreaklines(unsigned long time_steps, unsigned long total_time_steps); - PixelSet* Render(); - - private: - // Function for updating the velocity field and the particles in it. - void UpdateVelocityFieldForAllParticlesAndPrune(); - void UpdateVelocityFieldForCommunicatedSites(); - - // Private functions for the creation / deletion of particles. - void ChooseSeedParticles(); - void CreateParticlesFromSeeds(); - - // Private functions for inter-proc communication. - void CommunicateSiteIds(); - void CommunicateVelocities(); - void WorkOutVelocityDataNeededForParticles(); - - const geometry::LatticeData& latDat; - const Screen& screen; - const Viewpoint& viewpoint; - const VisSettings& visSettings; - //const lb::MacroscopicPropertyCache& propertyCache; - - std::map neighbouringProcessors; - ParticleManager particleManager; - VelocityField velocityField; - - std::vector particleSeeds; - net::Net* streakNet; - }; - } - } -} - -#endif // HEMELB_VIS_STREAKLINEDRAWER_STREAKLINEDRAWER_H diff --git a/Code/vis/streaklineDrawer/VelocityField.cc b/Code/vis/streaklineDrawer/VelocityField.cc deleted file mode 100644 index 2b396bde4..000000000 --- a/Code/vis/streaklineDrawer/VelocityField.cc +++ /dev/null @@ -1,347 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include - -#include "debug/Debugger.h" -#include "geometry/BlockTraverser.h" -#include "geometry/SiteTraverser.h" -#include "vis/streaklineDrawer/VelocityField.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - VelocityField::VelocityField(proc_t localRank_, - std::map& neighbouringProcessorsIn, - const lb::MacroscopicPropertyCache& propertyCache) : - counter(0), localRank(localRank_), neighbouringProcessors(neighbouringProcessorsIn), propertyCache(propertyCache) - { - } - - void VelocityField::BuildVelocityField(const geometry::LatticeData& latDat) - { - velocityField.resize(latDat.GetBlockCount()); - - // Iterate over each block with some sites on this rank. - geometry::BlockTraverser blockTraverser(latDat); - do - { - const geometry::Block& block = blockTraverser.GetCurrentBlockData(); - - if (block.IsEmpty()) - { - continue; - } - - geometry::SiteTraverser siteTraverser(latDat); - do - { - // Only interested if the site lives on this rank. - if (localRank != block.GetProcessorRankForSite(siteTraverser.GetCurrentIndex())) - { - continue; - } - - // Calculate the bounds of the unit cube around the current site (within the lattice) - const site_t startI = util::NumericalFunctions::max(0, - blockTraverser.GetX() - * blockTraverser.GetBlockSize() - + siteTraverser.GetX() - 1); - - const site_t startJ = util::NumericalFunctions::max(0, - blockTraverser.GetY() - * blockTraverser.GetBlockSize() - + siteTraverser.GetY() - 1); - - const site_t startK = util::NumericalFunctions::max(0, - blockTraverser.GetZ() - * blockTraverser.GetBlockSize() - + siteTraverser.GetZ() - 1); - - const site_t endI = util::NumericalFunctions::min(latDat.GetSiteDimensions().x - 1, - blockTraverser.GetX() - * blockTraverser.GetBlockSize() - + siteTraverser.GetX() + 1); - - const site_t endJ = util::NumericalFunctions::min(latDat.GetSiteDimensions().y - 1, - blockTraverser.GetY() - * blockTraverser.GetBlockSize() - + siteTraverser.GetY() + 1); - - const site_t endK = util::NumericalFunctions::min(latDat.GetSiteDimensions().z - 1, - blockTraverser.GetZ() - * blockTraverser.GetBlockSize() - + siteTraverser.GetZ() + 1); - - // Iterate over the sites in the unit cube. - for (site_t neighbourI = startI; neighbourI <= endI; neighbourI++) - { - for (site_t neighbourJ = startJ; neighbourJ <= endJ; neighbourJ++) - { - for (site_t neighbourK = startK; neighbourK <= endK; neighbourK++) - { - // Get the rank that the neighbour lives on. - const proc_t neigh_proc_id = latDat.GetProcIdFromGlobalCoords(util::Vector3D(neighbourI, - neighbourJ, - neighbourK)); - - // If we have data for it, we should initialise a block in the velocity field - // for the neighbour site. - if (neigh_proc_id == SITE_OR_BLOCK_SOLID) - { - continue; - } - - InitializeVelocityFieldBlock(latDat, - util::Vector3D(neighbourI, neighbourJ, neighbourK), - neigh_proc_id); - - // If the neighbour is on this rank, ignore it. - if (localRank == neigh_proc_id) - { - continue; - } - - if (neighbouringProcessors.count(neigh_proc_id) == 0) - { - NeighbouringProcessor newProc(neigh_proc_id); - neighbouringProcessors[neigh_proc_id] = newProc; - } - } - } - } - } - while (siteTraverser.TraverseOne()); - } - while (blockTraverser.TraverseOne()); - - // Iterate over the blocks, updating the value of the counter variable wherever - // there is velocity field data. - for (site_t block = 0; block < latDat.GetBlockCount(); block++) - { - if (velocityField[block].empty()) - { - continue; - } - - if (latDat.GetBlock(block).IsEmpty()) - { - continue; - } - - // Update the site id on each velocity field unit as required. - for (site_t localSiteId = 0; localSiteId < latDat.GetSitesPerBlockVolumeUnit(); localSiteId++) - { - velocityField[block][localSiteId].site_id = - latDat.GetBlock(block).GetLocalContiguousIndexForSite(localSiteId); - } - } - } - - bool VelocityField::BlockContainsData(size_t blockNumber) const - { - return !velocityField[blockNumber].empty(); - } - - VelocitySiteData& VelocityField::GetSiteData(site_t blockNumber, site_t siteNumber) - { - return velocityField[blockNumber][siteNumber]; - } - - // Returns the velocity site data for a given index, or NULL if the index isn't valid / has - // no data. - VelocitySiteData* VelocityField::GetVelocitySiteData(const geometry::LatticeData& latDat, - const util::Vector3D& location) - { - if (!latDat.IsValidLatticeSite(location)) - { - return NULL; - } - - util::Vector3D blockCoords, siteCoords; - latDat.GetBlockAndLocalSiteCoords(location, blockCoords, siteCoords); - - site_t block_id = latDat.GetBlockIdFromBlockCoords(blockCoords); - - if (!BlockContainsData(static_cast(block_id))) - { - return NULL; - } - - site_t site_id = latDat.GetLocalSiteIdFromLocalSiteCoords(siteCoords); - - return &GetSiteData(block_id, site_id); - } - - // Function to initialise the velocity field at given coordinates. - void VelocityField::InitializeVelocityFieldBlock(const geometry::LatticeData& latDat, - const util::Vector3D location, - const proc_t proc_id) - { - util::Vector3D blockCoords, siteCoords; - latDat.GetBlockAndLocalSiteCoords(location, blockCoords, siteCoords); - - site_t blockId = latDat.GetBlockIdFromBlockCoords(blockCoords); - - if (!BlockContainsData(blockId)) - { - for (site_t localSiteId = 0; localSiteId < latDat.GetSitesPerBlockVolumeUnit(); ++localSiteId) - { - velocityField[blockId].push_back(VelocitySiteData()); - } - } - - site_t localSiteId = latDat.GetLocalSiteIdFromLocalSiteCoords(siteCoords); - velocityField[blockId][localSiteId].proc_id = proc_id; - } - - // Populate the matrix v with all the velocity field data at each index. - // Returns true if this the area resides entirely on this core. - void VelocityField::GetVelocityFieldAroundPoint(const util::Vector3D location, - const geometry::LatticeData& latDat, - util::Vector3D localVelocityField[2][2][2]) - { - for (int unitGridI = 0; unitGridI <= 1; ++unitGridI) - { - site_t neighbourI = location.x + unitGridI; - - for (int unitGridJ = 0; unitGridJ <= 1; ++unitGridJ) - { - site_t neighbourJ = location.y + unitGridJ; - - for (int unitGridK = 0; unitGridK <= 1; ++unitGridK) - { - site_t neighbourK = location.z + unitGridK; - - util::Vector3D neighbour(neighbourI, neighbourJ, neighbourK); - - if (!latDat.IsValidLatticeSite(neighbour)) - { - // it is a solid site and the velocity is - // assumed to be zero - localVelocityField[unitGridI][unitGridJ][unitGridK] = util::Vector3D::Zero(); - continue; - } - - VelocitySiteData *vel_site_data_p = GetVelocitySiteData(latDat, - util::Vector3D(neighbourI, - neighbourJ, - neighbourK)); - - if (vel_site_data_p == NULL || vel_site_data_p->proc_id == -1) - { - // it is a solid site and the velocity is - // assumed to be zero - localVelocityField[unitGridI][unitGridJ][unitGridK] = util::Vector3D::Zero(); - continue; - } - - if (vel_site_data_p->counter != counter) - { - UpdateLocalField(vel_site_data_p, latDat); - } - - localVelocityField[unitGridI][unitGridJ][unitGridK] = vel_site_data_p->velocity; - } - } - } - } - - bool VelocityField::NeededFromNeighbour(const util::Vector3D location, - const geometry::LatticeData& latDat, - proc_t* sourceProcessor) - { - if (!latDat.IsValidLatticeSite(location)) - { - return false; - } - - VelocitySiteData *vel_site_data_p = GetVelocitySiteData(latDat, location); - - if (vel_site_data_p == NULL || vel_site_data_p->proc_id == -1 || vel_site_data_p->proc_id == localRank - || vel_site_data_p->counter == counter) - { - return false; - } - - vel_site_data_p->counter = counter; - *sourceProcessor = vel_site_data_p->proc_id; - - return true; - } - - void VelocityField::UpdateLocalField(const util::Vector3D& position, const geometry::LatticeData& latDat) - { - VelocitySiteData *localVelocitySiteData = GetVelocitySiteData(latDat, position); - - if (log::Logger::ShouldDisplay()) - { - if (localRank != localVelocitySiteData->proc_id) - { - log::Logger::Log("Got a request for velocity data " - "that actually seems to be on rank %i", - localVelocitySiteData->proc_id); - } - } - - UpdateLocalField(localVelocitySiteData, latDat); - } - - void VelocityField::UpdateLocalField(VelocitySiteData* localVelocitySiteData, const geometry::LatticeData& latDat) - { - // the local counter is set equal to the global one - // and the local velocity is calculated - localVelocitySiteData->counter = counter; - - const util::Vector3D& velocity = propertyCache.velocityCache.Get(localVelocitySiteData->site_id); - localVelocitySiteData->velocity.x = (float) velocity.x; - localVelocitySiteData->velocity.y = (float) velocity.y; - localVelocitySiteData->velocity.z = (float) velocity.z; - } - - // Interpolates a velocity field to get the velocity at the position of a particle. - util::Vector3D VelocityField::InterpolateVelocityForPoint(const util::Vector3D point, - const util::Vector3D localVelocityField[2][2][2]) const - { - float dummy; - - // Get the fractional parts of each of x, y, z - float dx = modff(point.x, &dummy); - float dy = modff(point.y, &dummy); - float dz = modff(point.z, &dummy); - - util::Vector3D yInterpolatedVelocity[2]; - - for (int unitX = 0; unitX <= 1; unitX++) - { - util::Vector3D zInterpolatedVelocityForThisX[2]; - - for (int unitY = 0; unitY <= 1; unitY++) - { - zInterpolatedVelocityForThisX[unitY] = localVelocityField[unitX][unitY][0] * (1.F - dz) - + localVelocityField[unitX][unitY][1] * dz; - } - - yInterpolatedVelocity[unitX] = zInterpolatedVelocityForThisX[0] * (1.F - dy) - + zInterpolatedVelocityForThisX[1] * dy; - } - - return yInterpolatedVelocity[0] * (1.F - dx) + yInterpolatedVelocity[1] * dx; - } - - void VelocityField::InvalidateAllCalculatedVelocities() - { - ++counter; - } - - } - } -} diff --git a/Code/vis/streaklineDrawer/VelocityField.h b/Code/vis/streaklineDrawer/VelocityField.h deleted file mode 100644 index 1181f54e5..000000000 --- a/Code/vis/streaklineDrawer/VelocityField.h +++ /dev/null @@ -1,81 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_VELOCITYFIELD_H -#define HEMELB_VIS_STREAKLINEDRAWER_VELOCITYFIELD_H - -#include -#include - -#include "constants.h" -#include "net/mpi.h" - -#include "geometry/LatticeData.h" -#include "lb/MacroscopicPropertyCache.h" -#include "net/IOCommunicator.h" - -#include "vis/streaklineDrawer/NeighbouringProcessor.h" -#include "vis/streaklineDrawer/VelocitySiteData.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - class VelocityField - { - public: - VelocityField(proc_t localRank, - std::map& iNeighbouringProcessors, - const lb::MacroscopicPropertyCache& propertyCache); - - void BuildVelocityField(const geometry::LatticeData& latDat); - - bool BlockContainsData(size_t iBlockNumber) const; - - VelocitySiteData* GetVelocitySiteData(const geometry::LatticeData& latDat, - const util::Vector3D& location); - - void GetVelocityFieldAroundPoint(const util::Vector3D location, - const geometry::LatticeData& latDat, - util::Vector3D localVelocityField[2][2][2]); - - util::Vector3D - InterpolateVelocityForPoint(const util::Vector3D position, - const util::Vector3D localVelocityField[2][2][2]) const; - - void InvalidateAllCalculatedVelocities(); - - void UpdateLocalField(const util::Vector3D& position, const geometry::LatticeData& latDat); - - bool NeededFromNeighbour(const util::Vector3D location, - const geometry::LatticeData& latDat, - proc_t* sourceProcessor); - - private: - void UpdateLocalField(VelocitySiteData* localVelocitySiteData, const geometry::LatticeData& latDat); - - // Counter to make sure the velocity field blocks are correct for the current iteration. - site_t counter; - - VelocitySiteData& GetSiteData(site_t iBlockNumber, site_t iSiteNumber); - - void InitializeVelocityFieldBlock(const geometry::LatticeData& latDat, - const util::Vector3D location, - const proc_t proc_id); - - const proc_t localRank; - // Vector containing VelocityFields - std::vector > velocityField; - std::map& neighbouringProcessors; - const lb::MacroscopicPropertyCache& propertyCache; - }; - } - } -} - -#endif // HEMELB_VIS_STREAKLINEDRAWER_VELOCITYFIELD_H diff --git a/Code/vis/streaklineDrawer/VelocitySiteData.h b/Code/vis/streaklineDrawer/VelocitySiteData.h deleted file mode 100644 index 1eb6e8e3c..000000000 --- a/Code/vis/streaklineDrawer/VelocitySiteData.h +++ /dev/null @@ -1,53 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#ifndef HEMELB_VIS_STREAKLINEDRAWER_VELOCITYSITEDATA_H -#define HEMELB_VIS_STREAKLINEDRAWER_VELOCITYSITEDATA_H - -#include "util/Vector3D.h" - -namespace hemelb -{ - namespace vis - { - namespace streaklinedrawer - { - // Class to hold information about the velocity field at some point. - class VelocitySiteData - { - public: - VelocitySiteData() : - counter(-1), proc_id(-1), site_id(-1), velocity(NO_VALUE) - { - } - - /** - * Counter describes how many iterations have passed since the objects creation. - * It allows for quickly checking whether the local velocity field has been - * calculated this iteration. - **/ - site_t counter; - - /** - * The rank this volume of the geometry lives on. - */ - proc_t proc_id; - - /** - * The site id of the volume represented. - */ - site_t site_id; - - /** - * The velocity calculated for that site. - */ - util::Vector3D velocity; - }; - } - } -} - -#endif //HEMELB_VIS_STREAKLINEDRAWER_VELOCITYSITEDATA_H diff --git a/Tools/hemeTools/cache.py b/Tools/hemeTools/cache.py index 8a999a448..baf6f8aa1 100644 --- a/Tools/hemeTools/cache.py +++ b/Tools/hemeTools/cache.py @@ -35,11 +35,11 @@ def cacher(*args, **kwargs): if os.path.exists(cachefile): - ans = cPickle.load(file(cachefile)) + ans = cPickle.load(open(cachefile)) else: ans = f(*args, **kwargs) cPickle.dump(ans, - file(cachefile, 'wb'), + open(cachefile, 'wb'), protocol=2) pass return ans @@ -63,11 +63,11 @@ def cacher(infile): if os.path.exists(cachefile) and \ os.path.getmtime(cachefile) > os.path.getmtime(infile): # Is the cachefile newer than the input? - ans = cPickle.load(file(cachefile)) + ans = cPickle.load(open(cachefile)) else: ans = f(infile) cPickle.dump(ans, - file(cachefile, 'wb'), + open(cachefile, 'wb'), protocol=2) pass return ans diff --git a/Tools/hemeTools/image.py b/Tools/hemeTools/image.py deleted file mode 100644 index b6c0eb97a..000000000 --- a/Tools/hemeTools/image.py +++ /dev/null @@ -1,158 +0,0 @@ - -# This file is part of HemeLB and is Copyright (C) -# the HemeLB team and/or their institutions, as detailed in the -# file AUTHORS. This software is provided under the terms of the -# license in the file LICENSE. - -import pdb -import xdrlib -import numpy as N - -class Image(object): - - def __init__(self, filename): - self.filename = filename - - f = file(filename) - reader = xdrlib.Unpacker(f.read()) - self.mode = reader.unpack_uint() - self.pressure_threshold = (reader.unpack_float(), reader.unpack_float()) - self.velocity_max = reader.unpack_float() - self.stress_max = reader.unpack_float() - self.screen = (reader.unpack_uint(), reader.unpack_uint()) - self.nPixels = reader.unpack_uint() - - tempPixels = N.zeros((self.nPixels,4), dtype=N.uint32) - - for i in xrange(self.nPixels): - for j in xrange(4): - tempPixels[i,j] = reader.unpack_uint() - continue - continue - sortedIndices = N.argsort(tempPixels[:,0]) - self.pixels = tempPixels[sortedIndices].view(dtype=[('index', N.uint16, 2), - ('r', N.uint8, 4), - ('g', N.uint8, 4), - ('b', N.uint8, 4)]).view(N.recarray) - - # The above statement puts the pixel indices in the wrong order (i.e. j,i rather than i,j). - # This is corrected here. - self.pixels.index[:,0] = self.pixels.index[:, 0, [1,0]] - - return - - def old__init__(self, filename): - self.filename = filename - - f = file(filename) - reader = xdrlib.Unpacker(f.read()) - self.mode = reader.unpack_uint() - self.pressure_threshold = (reader.unpack_float(), reader.unpack_float()) - self.velocity_max = reader.unpack_float() - self.stress_max = reader.unpack_float() - self.screen = (reader.unpack_uint(), reader.unpack_uint()) - self.nPixels = reader.unpack_uint() - - tempPixels = N.zeros(self.nPixels, dtype=type(self).RowType) - - # Masks and shifts to get the bits for each vis mode - masks = [(2**32-1) ^ (2**24-1), - (2**24-1) ^ (2**16-1), - (2**16-1) ^ (2**8-1), - (2**8-1)] - shifts = [24, 16, 8, 0] - - conversions = zip(masks, shifts) - for i in xrange(self.nPixels): - ind = reader.unpack_uint() - tempPixels[i]['index'][0] = (ind & ((2**32-1) ^ (2**16-1))) >> 16 - tempPixels[i]['index'][1] = ind & (2**16-1) - r = reader.unpack_uint() - g = reader.unpack_uint() - b = reader.unpack_uint() - - for j, (mask, shift) in enumerate(conversions): - tempPixels[i]['r'][j] = (r & mask) >> shift - tempPixels[i]['g'][j] = (g & mask) >> shift - tempPixels[i]['b'][j] = (b & mask) >> shift - continue - - # assert r == ( - # (int(tempPixels[i]['r'][0]) << 24) + - # (int(tempPixels[i]['r'][1]) << 16) + - # (int(tempPixels[i]['r'][2]) << 8) + - # (int(tempPixels[i]['r'][3]) << 0) - # ) - # assert g == ( - # (int(tempPixels[i]['g'][0]) << 24) + - # (int(tempPixels[i]['g'][1]) << 16) + - # (int(tempPixels[i]['g'][2]) << 8) + - # (int(tempPixels[i]['g'][3]) << 0) - # ) - # assert b == ( - # (int(tempPixels[i]['b'][0]) << 24) + - # (int(tempPixels[i]['b'][1]) << 16) + - # (int(tempPixels[i]['b'][2]) << 8) + - # (int(tempPixels[i]['b'][3]) << 0) - # ) - - continue - sortedIndices = N.argsort(tempPixels['index'][:, 0]<<16 + tempPixels['index'][:, 1]) - self.pixels = tempPixels[sortedIndices] - - return - - def __eq__(self, other): - """Return True if the Images are identical. - """ - attrs = ('mode', 'pressure_threshold', 'velocity_max', 'stress_max', 'screen') - for at in attrs: - if not getattr(self, at) == getattr(other, at): - return False - continue - - if not N.alltrue(self.pixels.view(dtype=N.uint32) == other.pixels.view(dtype=N.uint32)): - return False - - return True - - def almost_eq(self, other, tol=1): - """Same as __eq__, except allow data array to differ by up to - tol (default 1). - """ - - attrs = ('mode', 'pressure_threshold', 'velocity_max', 'stress_max', 'screen') - for at in attrs: - if not getattr(self, at) == getattr(other, at): - print 'differed on ' + at + ': ' + str(getattr(self, at)) + ' and ' + str(getattr(other, at)) - return False - continue - - if not N.alltrue(self.pixels.index == other.pixels.index): - for i in range(self.pixels.index.shape[0]): - if not N.alltrue(self.pixels.index[i] == other.pixels.index[i]): - print 'Images differed on indices, first at index: ' + str(i) - print 'left array had ' + str(self.pixels.index[i]) - print 'right array had ' + str(other.pixels.index[i]) - return False - - for attr in ('r', 'g', 'b'): - d = self.pixels[attr] - other.pixels[attr] + 128 - if N.min(d) < (128-tol) or N.max(d) > (128+tol): - minArg = N.argmin(d, 0)[0][0] - maxArg = N.argmax(d, 0)[0][0] - print 'Differed on ' + attr + ' channel with delta range:' - print str(N.min(d) - 128) + ' between ' + str(self.pixels[minArg]) + ' and ' + str(other.pixels[minArg]) - print 'and' - print str(N.max(d) - 128) + ' between ' + str(self.pixels[maxArg]) + ' and ' + str(other.pixels[maxArg]) - return False - continue - - return True - - pass - -if __name__ == "__main__": - import sys - images = [Image(arg) for arg in sys.argv[1:]] - diff --git a/Tools/hemeTools/parsers/extraction/__init__.py b/Tools/hemeTools/parsers/extraction/__init__.py index 9a544b10f..d5bf9f2a0 100644 --- a/Tools/hemeTools/parsers/extraction/__init__.py +++ b/Tools/hemeTools/parsers/extraction/__init__.py @@ -32,7 +32,7 @@ class FieldSpec(object): def __init__(self, memspec): # name, XDR dtype, in-memory dtype, length, offset - self._filespec = [('grid', '>i4', np.uint32, (3,), 0)] + self._filespec = [(u'grid', '>i4', np.uint32, (3,), 0)] self._memspec = memspec return @@ -51,13 +51,13 @@ def Append(self, name, length, pyType, datatype): def GetMem(self): """Get the numpy datatype for the in-memory array. """ - return np.dtype([(name, memType, length) + return np.dtype([(str(name), memType, length) for name, xdrType, memType, length, offset in (self._memspec + self._filespec)]) def GetXdr(self): """Get the numpy datatype for the XDR file. """ - return np.dtype([(name, xdrType, length) + return np.dtype([(str(name), xdrType, length) for name, xdrType, memType, length, offset in self._filespec]) def GetRecordLength(self): @@ -71,6 +71,10 @@ def __iter__(self): return iter(self._filespec) pass +def unpack_string(decoder): + bstr = decoder.unpack_bytes() + return bstr.decode('utf-8') + class ExtractedPropertyV3Parser(object): def __init__(self, fieldCount, siteCount): self._fieldCount = fieldCount @@ -85,11 +89,11 @@ def parse(self, memoryMappedData): return result def ParseFieldHeader(self, decoder): - self._fieldSpec = FieldSpec([('id', None, np.uint64, 1, None), - ('position', None, np.float32, (3,), None)]) + self._fieldSpec = FieldSpec([(u'id', None, np.uint64, 1, None), + (u'position', None, np.float32, (3,), None)]) - for iField in xrange(self._fieldCount): - name = decoder.unpack_string() + for iField in range(self._fieldCount): + name = unpack_string(decoder) length = decoder.unpack_uint() self._fieldSpec.Append(name, length, '>f8', np.float64) continue @@ -113,12 +117,12 @@ def parse(self, memoryMappedData): return result def ParseFieldHeader(self, decoder): - self._fieldSpec = FieldSpec([('id', None, np.uint64, 1, None), - ('position', None, np.float32, (3,), None)]) + self._fieldSpec = FieldSpec([(u'id', None, np.uint64, 1, None), + (u'position', None, np.float32, (3,), None)]) self._dataOffset = [0] - for iField in xrange(self._fieldCount): - name = decoder.unpack_string() + for iField in range(self._fieldCount): + name = unpack_string(decoder) length = decoder.unpack_uint() self._dataOffset.append(decoder.unpack_double()) self._fieldSpec.Append(name, length, '>f4', np.float32) @@ -144,7 +148,7 @@ def __init__(self, filename): """ self.filename = filename - self._file = file(filename, 'rb') + self._file = open(filename, 'rb') self._ReadMainHeader() self._ReadFieldHeader() @@ -172,7 +176,7 @@ def _ReadMainHeader(self): assert version in self.HandledVersions, "Incorrect extraction format version number" self.voxelSizeMetres = decoder.unpack_double() - self.originMetres = np.array([decoder.unpack_double() for i in xrange(3)]) + self.originMetres = np.array([decoder.unpack_double() for i in range(3)]) self.siteCount = decoder.unpack_uhyper() self.fieldCount = decoder.unpack_uint() @@ -215,10 +219,10 @@ def _DetermineTimes(self): bodysize = filesize - self._totalHeaderLength assert bodysize % self._recordLength == 0, \ "Extraction file appears to have partial record(s), residual %s / %s , bodysize %s"%(bodysize % self._recordLength,self._recordLength,bodysize) - nTimes = bodysize / self._recordLength + nTimes = bodysize // self._recordLength times = np.zeros(nTimes, dtype=int) - for iT in xrange(nTimes): + for iT in range(nTimes): pos = self._totalHeaderLength + iT * self._recordLength self._file.seek(pos) timeBuf = self._file.read(TimeStepDataLength) diff --git a/Tools/hemeTools/parsers/geometry/simple.py b/Tools/hemeTools/parsers/geometry/simple.py index c3eefb840..2ed906aaf 100644 --- a/Tools/hemeTools/parsers/geometry/simple.py +++ b/Tools/hemeTools/parsers/geometry/simple.py @@ -65,7 +65,7 @@ def __init__(self, filename): self.Domain.Origin = np.array(oStr[1:-1].split(','), dtype=float) assert self.Domain.Origin.shape == (3,) - self.File = file(self.GmyFileName) + self.File = open(self.GmyFileName) return def Load(self): diff --git a/Tools/hemeTools/parsers/snapshot/Cfx.py b/Tools/hemeTools/parsers/snapshot/Cfx.py index 7552f1cff..d55a0ebef 100644 --- a/Tools/hemeTools/parsers/snapshot/Cfx.py +++ b/Tools/hemeTools/parsers/snapshot/Cfx.py @@ -1,4 +1,4 @@ - +\ # This file is part of HemeLB and is Copyright (C) # the HemeLB team and/or their institutions, as detailed in the # file AUTHORS. This software is provided under the terms of the @@ -103,7 +103,7 @@ def parseHeader(filename, AllData=False): '''Parses header to get data type for record array''' fieldRegEx = re.compile('(.+?)\s*\[\s*(.+?)\s*\]') - f = file(filename) + f = open(filename) for i in range(findStart(filename, AllData=AllData)+2): header = f.readline() continue @@ -163,7 +163,7 @@ def findStart(filename, AllData=False): if AllData == True: return -1 - for i, line in enumerate(file(filename)): + for i, line in enumerate(open(filename)): if line.find('[Data]')>=0: return i continue diff --git a/Tools/hemeTools/parsers/snapshot/__init__.py b/Tools/hemeTools/parsers/snapshot/__init__.py index 05a3a08d4..19ad28059 100644 --- a/Tools/hemeTools/parsers/snapshot/__init__.py +++ b/Tools/hemeTools/parsers/snapshot/__init__.py @@ -26,7 +26,7 @@ def HemeLbSnapshot(filename): numbers and more metadata. """ - start = file(filename).read(8) + start = open(filename).read(8) reader = xdrlib.Unpacker(start) firstInt = reader.unpack_uint() @@ -171,7 +171,7 @@ def _readHeader(cls, filename): """ - f = file(filename) + f = open(filename) stable = int(f.readline()) voxel_size = float(f.readline()) bb_min = np.array([int(x) for x in f.readline().split()]) @@ -198,7 +198,7 @@ class XdrVoxelFormatOneSnapshot(object): @classmethod def _load(cls, filename, header): # Skip past the header, slurp data, create XDR object - f = file(filename) + f = open(filename) f.seek(cls._headerLengthBytes) reader = xdrlib.Unpacker(f.read()) @@ -236,7 +236,7 @@ def _readHeader(cls, filename): 5- total number of fluid voxels """ - reader = xdrlib.Unpacker(file(filename).read(cls._headerLengthBytes)) + reader = xdrlib.Unpacker(open(filename).read(cls._headerLengthBytes)) header = {} header['stable'] = reader.unpack_int() header['voxel_size'] = reader.unpack_double() @@ -264,7 +264,7 @@ class XdrSnapshotVersionTwo(BaseSnapshot, XdrVoxelFormatOneSnapshot): def _readHeader(cls, filename): """Read the header lines, according to description in Code/io/formats/snapshot.h """ - reader = xdrlib.Unpacker(file(filename).read(cls._headerLengthBytes)) + reader = xdrlib.Unpacker(open(filename).read(cls._headerLengthBytes)) header = {} assert reader.unpack_uint() == HemeLbMagicNumber assert reader.unpack_uint() == SnapshotMagicNumber @@ -299,7 +299,7 @@ def VersionedXdrSnapshot(filename): """Examine the file and dispatch to the appropriate constructor. """ # Need the two magic numbers and the version number, i.e. 12 bytes - reader = xdrlib.Unpacker(file(filename).read(12)) + reader = xdrlib.Unpacker(open(filename).read(12)) assert reader.unpack_uint() == HemeLbMagicNumber assert reader.unpack_uint() == SnapshotMagicNumber diff --git a/Tools/hemeTools/utils/utils.py b/Tools/hemeTools/utils/utils.py index 4d963b5ad..e53f9d379 100644 --- a/Tools/hemeTools/utils/utils.py +++ b/Tools/hemeTools/utils/utils.py @@ -5,7 +5,7 @@ # license in the file LICENSE. """Utility functions for HemeTools """ - +from six.moves import xrange import numpy as np def MatchCorresponding(first, second): diff --git a/Tools/visclients/.project b/Tools/visclients/.project deleted file mode 100644 index 1ce174429..000000000 --- a/Tools/visclients/.project +++ /dev/null @@ -1,17 +0,0 @@ - - - CLINICAL-GUI - - - - - - org.eclipse.jdt.core.javabuilder - - - - - - org.eclipse.jdt.core.javanature - - diff --git a/Tools/visim/.cproject b/Tools/visim/.cproject deleted file mode 100644 index 9150f6308..000000000 --- a/Tools/visim/.cproject +++ /dev/null @@ -1,304 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/Tools/visim/.project b/Tools/visim/.project deleted file mode 100644 index 8cc45ebb6..000000000 --- a/Tools/visim/.project +++ /dev/null @@ -1,79 +0,0 @@ - - - visim - - - - - - org.eclipse.cdt.managedbuilder.core.genmakebuilder - clean,full,incremental, - - - ?name? - - - - org.eclipse.cdt.make.core.append_environment - true - - - org.eclipse.cdt.make.core.autoBuildTarget - all - - - org.eclipse.cdt.make.core.buildArguments - - - - org.eclipse.cdt.make.core.buildCommand - make - - - org.eclipse.cdt.make.core.cleanBuildTarget - clean - - - org.eclipse.cdt.make.core.contents - org.eclipse.cdt.make.core.activeConfigSettings - - - org.eclipse.cdt.make.core.enableAutoBuild - false - - - org.eclipse.cdt.make.core.enableCleanBuild - true - - - org.eclipse.cdt.make.core.enableFullBuild - true - - - org.eclipse.cdt.make.core.fullBuildTarget - - - - org.eclipse.cdt.make.core.stopOnError - true - - - org.eclipse.cdt.make.core.useDefaultBuildCmd - true - - - - - org.eclipse.cdt.managedbuilder.core.ScannerConfigBuilder - full,incremental, - - - - - - org.eclipse.cdt.core.cnature - org.eclipse.cdt.core.ccnature - org.eclipse.cdt.managedbuilder.core.managedBuildNature - org.eclipse.cdt.managedbuilder.core.ScannerConfigNature - - diff --git a/Tools/visim/Makefile b/Tools/visim/Makefile deleted file mode 100644 index 3641575d5..000000000 --- a/Tools/visim/Makefile +++ /dev/null @@ -1,25 +0,0 @@ - -# This file is part of HemeLB and is Copyright (C) -# the HemeLB team and/or their institutions, as detailed in the -# file AUTHORS. This software is provided under the terms of the -# license in the file LICENSE. -target = visim - -CC = g++ -ifeq ($(HEMELB_MACHINE),OSX) -CFLAGS = -framework OpenGL -framework GLUT -framework Foundation -O4 -DHEMELB_CFG_ON_OSX -else -CFLAGS = -O4 -L/usr/include/GT/ -lglut -lGLU -lGL -endif - -objects = visualize_images.o - -src = visualize_images.cc -$(target) : $(src) - $(CC) $(CFLAGS) $< -o $@ - - -.PHONY : clean - -clean : - -rm $(target) $(objects) diff --git a/Tools/visim/visualize_images.cc b/Tools/visim/visualize_images.cc deleted file mode 100644 index 4f82395e9..000000000 --- a/Tools/visim/visualize_images.cc +++ /dev/null @@ -1,471 +0,0 @@ - -// This file is part of HemeLB and is Copyright (C) -// the HemeLB team and/or their institutions, as detailed in the -// file AUTHORS. This software is provided under the terms of the -// license in the file LICENSE. - -#include -#include -#include -#ifdef HEMELB_CFG_ON_OSX -#include -#else -#include -#endif// HEMELB_CFG_OSX -#include -#include -#include -#include - -// g++ -framework OpenGL -framework GLUT -framework Foundation visualize_images.c -o visualize_images - -#define RAINBOW 0 -#define HALF_RAINBOW 1 -#define GREY 2 - -char *input_path; -char *output_path; - -unsigned char *image_data, *image_buffer; - -int pixels_x, pixels_y; -int images; -int image_count = 0; -int cycle_id = 1; - -int min(int a, int b) { - if (a < b) { - return a; - } else { - return b; - } -} - -int max(int a, int b) { - if (a > b) { - return a; - } else { - return b; - } -} - -void visRainbowPalette(int id, float col[3]) { - if (id == 0) { - col[0] = 0.F; - col[1] = 0.F; - col[2] = 1.F; - } else if (id == 1) { - col[0] = 0.F; - col[1] = 1.F; - col[2] = 1.F; - } else if (id == 2) { - col[0] = 0.F; - col[1] = 1.F; - col[2] = 0.F; - } else if (id == 3) { - col[0] = 1.F; - col[1] = 1.F; - col[2] = 0.F; - } else if (id == 4) { - col[0] = 1.F; - col[1] = 0.F; - col[2] = 0.F; - } -} - -void OpenWindow(int pixels_x, int pixels_y) { - glutInitDisplayMode(GLUT_RGB | GLUT_DOUBLE); - glutInitWindowPosition(0, 0); - glutInitWindowSize(pixels_x - 1, pixels_y - 1); - - glutCreateWindow(" "); - - glDisable (GL_DEPTH_TEST); - glDisable (GL_BLEND); - glShadeModel (GL_SMOOTH); - glDisable (GL_DITHER); - - glClear (GL_COLOR_BUFFER_BIT); -} - -void Projection(int pixels_x, int pixels_y) { - glLoadIdentity(); - gluOrtho2D(0.F, (float) (pixels_x - 1), - 0.F, (float)(pixels_y - 1)); - -glClearColor (1.0F, 1.0F, 1.0F, 0.0F); -} - -void DisplayFrame(int pixels_x, int pixels_y) { - for (int j = 0; j < pixels_y; j++) { - for (int i = 0; i < pixels_x; i++) { - if (i > 3 && i <= pixels_x - 3 && j > 3 && j <= pixels_y - 3) - continue; - - image_data[(j * pixels_x + i) * 3 + 0] = 0; - image_data[(j * pixels_x + i) * 3 + 1] = 0; - image_data[(j * pixels_x + i) * 3 + 2] = 0; - } - } -} - -void visVisualiseString(float r, float g, float b, float x, float y, - char *string, void *font) { - glColor3f(r, g, b); - glRasterPos2f(x, y); - - for (int i = 0; i < (int) strlen(string); i++) { - glutBitmapCharacter(font, string[i]); - } - glEnd(); -} - -void visVisualiseBarPars(char *unit, float threshold_min, float threshold_max, - float x_min, float y_min, float x_max) { - float dx = 0.25F * (x_max - x_min); - - char string[256]; - - for (int i = 0; i <= 4; i++) { - sprintf(string, "%.2f", - threshold_min + 0.25 * i * (threshold_max - threshold_min)); - - visVisualiseString(0.F, 0.F, 0.F, x_min + i * dx - 20.0F, y_min - 16.0F, - string, GLUT_BITMAP_HELVETICA_18); - } - visVisualiseString(0.F, 0.F, 0.F, x_max + 45, y_min - 16.0F, unit, - GLUT_BITMAP_HELVETICA_18); -} - -void visVisualiseColourBar(int type, float x_min, float y_min, float x_max, - float y_max) { - float dx = 0.25F * (x_max - x_min); - float col[3], grey; - - int edge = 0; - - glBegin (GL_QUAD_STRIP); - - for (int i = 0; i <= 4; i++) { - if (type == RAINBOW) - { - visRainbowPalette(edge++, col); - glColor3fv(col); - } else if (type == HALF_RAINBOW) - { - visRainbowPalette(edge++, col); - - for (int l = 0; l < 3; l++) - col[l] = 0.5F * (1.0F + col[l]); - glColor3fv(col); - } else { - grey = edge++ / (float) 8; - glColor3f(grey, grey, grey); - } - glVertex2f(x_min + i * dx, y_min); - glVertex2f(x_min + i * dx, y_max); - } - glEnd(); - - glBegin (GL_LINES); - - for (int i = 0; i <= 4; i++) { - if (type == GREY) - { - glColor3f(1.0F, 1.0F, 1.0F); - } else { - glColor3f(0.0F, 0.0F, 0.0F); - } - glVertex2f(x_min + i * dx, y_min); - glVertex2f(x_min + i * dx, y_max); - } - glEnd(); -} - -void SaveWindowImage(int pixels_x, int pixels_y, char *file_name) { - FILE *ppm_image_file_ptr = fopen(file_name, "wb"); - - int i, j; - - unsigned char *data_p = NULL; - - glReadBuffer (GL_FRONT); - - data_p = image_data; - - for (j = 0; j < pixels_y; j++) { - glReadPixels(0, j, pixels_x, 1, GL_RGB, GL_UNSIGNED_BYTE, image_buffer); - - for (i = 0; i < pixels_x; i++) { - *data_p = image_buffer[i * 3]; - data_p++; - *data_p = image_buffer[i * 3 + 1]; - data_p++; - *data_p = image_buffer[i * 3 + 2]; - data_p++; - } - } - fprintf(ppm_image_file_ptr, "P6\n%i %i\n255\n", pixels_x, pixels_y); - - for (j = pixels_y - 1; j >= 0; j--) { - fwrite(image_data + j * pixels_x * 3, 1, pixels_x * 3, - ppm_image_file_ptr); - } - fclose(ppm_image_file_ptr); -} - -void Display(void) { - FILE *input_image_file; - FILE *temp_file_ptr; - XDR xdr_image_file; - - float pressure_threshold_min, pressure_threshold_max; - float velocity_threshold_max, stress_threshold_max; - - int mode; - int bits_per_char = sizeof(char) * 8; - int colour_mask = (1 << (sizeof(char) * 8)) - 1; - int pixel_mask = (1 << (sizeof(char) * 16)) - 1; - int col_pixels; - int pixel_i, pixel_j; - int palette_type; - int dummy; - int i, n; - - unsigned int pixel_id; - unsigned int col_data[3]; - - unsigned char pixel_r[4], pixel_g[4], pixel_b[4]; - - char partial_image_name[256], input_image_name[256], output_image_name[256]; - - char *pressure_unit = "(mmHg)"; - char *velocity_unit = "(m/s)"; - char *stress_unit = "(Pa)"; - - glClear (GL_COLOR_BUFFER_BIT); - - glPointSize(1.F); - glBegin (GL_POINTS); - - temp_file_ptr = fopen("temp_file.txt", "r"); - - for (i = -1; i < image_count; i++) { - fscanf(temp_file_ptr, "%s\n", partial_image_name); - } - fclose(temp_file_ptr); - - sprintf(input_image_name, "%s/%s", input_path, partial_image_name); - - std::string lImageNameNoExtension = std::string(partial_image_name).substr( - 0, std::string(partial_image_name).rfind('.')); - - sprintf(output_image_name, "%s/%s.ppm", output_path, - lImageNameNoExtension.c_str()); - - input_image_file = fopen(input_image_name, "r"); - xdrstdio_create(&xdr_image_file, input_image_file, XDR_DECODE); - - xdr_int(&xdr_image_file, &mode); - xdr_float(&xdr_image_file, &pressure_threshold_min); - xdr_float(&xdr_image_file, &pressure_threshold_max); - xdr_float(&xdr_image_file, &velocity_threshold_max); - xdr_float(&xdr_image_file, &stress_threshold_max); - - xdr_int(&xdr_image_file, &dummy); - xdr_int(&xdr_image_file, &dummy); - xdr_int(&xdr_image_file, &col_pixels); - - for (n = 0; n < col_pixels; n++) { - xdr_u_int(&xdr_image_file, &pixel_id); - - xdr_u_int(&xdr_image_file, &col_data[0]); - xdr_u_int(&xdr_image_file, &col_data[1]); - xdr_u_int(&xdr_image_file, &col_data[2]); - - pixel_i = (pixel_id >> (2 * bits_per_char)) & pixel_mask; - pixel_j = (pixel_id) & pixel_mask; - - pixel_r[0] = (col_data[0] >> (3 * bits_per_char)) & colour_mask; - pixel_g[0] = (col_data[0] >> (2 * bits_per_char)) & colour_mask; - pixel_b[0] = (col_data[0] >> (1 * bits_per_char)) & colour_mask; - pixel_r[1] = (col_data[0] >> (0 * bits_per_char)) & colour_mask; - pixel_g[1] = (col_data[1] >> (3 * bits_per_char)) & colour_mask; - pixel_b[1] = (col_data[1] >> (2 * bits_per_char)) & colour_mask; - pixel_r[2] = (col_data[1] >> (1 * bits_per_char)) & colour_mask; - pixel_g[2] = (col_data[1] >> (0 * bits_per_char)) & colour_mask; - pixel_b[2] = (col_data[2] >> (3 * bits_per_char)) & colour_mask; - pixel_r[3] = (col_data[2] >> (2 * bits_per_char)) & colour_mask; - pixel_g[3] = (col_data[2] >> (1 * bits_per_char)) & colour_mask; - pixel_b[3] = (col_data[2] >> (0 * bits_per_char)) & colour_mask; - - glColor3f(pixel_r[0] * (1.F / 255.F), pixel_g[0] * (1.F / 255.F), - pixel_b[0] * (1.F / 255.F)); - glVertex2f(pixel_i, pixel_j + (pixels_y >> 1)); - - glColor3f(pixel_r[1] * (1.F / 255.F), pixel_g[1] * (1.F / 255.F), - pixel_b[1] * (1.F / 255.F)); - glVertex2f(pixel_i + (pixels_x >> 1), pixel_j + (pixels_y >> 1)); - - glColor3f(pixel_r[2] * (1.F / 255.F), pixel_g[2] * (1.F / 255.F), - pixel_b[2] * (1.F / 255.F)); - glVertex2f(pixel_i, pixel_j); - - glColor3f(pixel_r[3] * (1.F / 255.F), pixel_g[3] * (1.F / 255.F), - pixel_b[3] * (1.F / 255.F)); - - glVertex2f(pixel_i + (pixels_x >> 1), pixel_j); - } - glEnd(); - - xdr_destroy(&xdr_image_file); - fclose(input_image_file); - - visVisualiseColourBar(RAINBOW, pixels_x * 0.1F, pixels_y * 0.5F + 20.F, - pixels_x * 0.4F, pixels_y * 0.5F + 40.F); - visVisualiseBarPars(velocity_unit, 0.0F, velocity_threshold_max, - pixels_x * 0.1F, pixels_y * 0.5F + 20.F, pixels_x * 0.4F); - - visVisualiseColourBar(RAINBOW, pixels_x * 0.6F, pixels_y * 0.5F + 20.F, - pixels_x * 0.9F, pixels_y * 0.5F + 40.F); - visVisualiseBarPars(stress_unit, 0.0F, stress_threshold_max, - pixels_x * 0.6F, pixels_y * 0.5F + 20.F, pixels_x * 0.9F); - - if (mode == 0) { - palette_type = RAINBOW; - } else if (mode == 1) { - palette_type = HALF_RAINBOW; - } else { - palette_type = GREY; - } - visVisualiseColourBar(palette_type, pixels_x * 0.1F, 20.F, pixels_x * 0.4F, - 40.F); - visVisualiseBarPars(pressure_unit, pressure_threshold_min, - pressure_threshold_max, pixels_x * 0.1F, 20.F, pixels_x * 0.4F); - - visVisualiseColourBar(palette_type, pixels_x * 0.6F, 20.F, pixels_x * 0.9F, - 40.F); - visVisualiseBarPars(stress_unit, 0.0F, stress_threshold_max, - pixels_x * 0.6F, 20.F, pixels_x * 0.9F); - - glutSwapBuffers(); - - if (cycle_id <= 2) { - SaveWindowImage(pixels_x, pixels_y, output_image_name); - } - if (++image_count == images - 1) { - ++cycle_id; - image_count = 0; - } -} - -void KeybordFunction(unsigned char key, int x, int y) { - if (key == 'q') { - free(image_buffer); - free(image_data); - - exit(0); - } -} - -void Reshape(GLsizei w, GLsizei h) { - //Projection (pixels_x, pixels_y); -} - -void usage(char *progname) { - printf( - "Usage: %s <'x' if you want to view images> \n", - progname); -} - -int main(int argc, char *argv[]) { - int required_args = 3; - - if (argc < required_args) { - usage(argv[0]); - exit(1); - } - - input_path = argv[1]; - output_path = argv[2]; - - FILE *image_file; - FILE *temp_file_ptr; - XDR xdr_image_file; - - int mode; - - float pressure_threshold_min, pressure_threshold_max; - float velocity_threshold_max, stress_threshold_max; - - char partial_image_name[256], image_name[256]; - char *first_command_part, *last_command_part; - char my_command[256]; - - first_command_part = "ls -l"; - last_command_part = "| wc > temp_file.txt"; - - sprintf(my_command, "%s %s %s", first_command_part, input_path, - last_command_part); - system(my_command); - - temp_file_ptr = fopen("temp_file.txt", "r"); - fscanf(temp_file_ptr, "%i ", &images); - fclose(temp_file_ptr); - - temp_file_ptr = fopen("temp_file.txt", "r"); - - first_command_part = "ls -1"; - last_command_part = "> temp_file.txt"; - - sprintf(my_command, "%s %s %s", first_command_part, input_path, - last_command_part); - system(my_command); - - rewind(temp_file_ptr); - - fscanf(temp_file_ptr, "%s\n", partial_image_name); - sprintf(image_name, "%s/%s", input_path, partial_image_name); - - fclose(temp_file_ptr); - - image_file = fopen(image_name, "r"); - xdrstdio_create(&xdr_image_file, image_file, XDR_DECODE); - - xdr_int(&xdr_image_file, &mode); - xdr_float(&xdr_image_file, &pressure_threshold_min); - xdr_float(&xdr_image_file, &pressure_threshold_max); - xdr_float(&xdr_image_file, &velocity_threshold_max); - xdr_float(&xdr_image_file, &stress_threshold_max); - - xdr_int(&xdr_image_file, &pixels_x); - xdr_int(&xdr_image_file, &pixels_y); - - pixels_x *= 2; - pixels_y *= 2; - - xdr_destroy(&xdr_image_file); - fclose(image_file); - - if(argc == 4 && argv[3][0] == 'x' && argv[3][1] == '\0') - { - glutInit(&argc, argv); - OpenWindow(pixels_x, pixels_y); - - Projection(pixels_x, pixels_y); - - image_data = (unsigned char *) malloc( - sizeof(unsigned char) * pixels_x * pixels_y * 3); - - image_buffer = (unsigned char *) malloc( - sizeof(unsigned char) * pixels_x * 3); - - //glutReshapeFunc (Reshape); - glutIdleFunc(Display); - glutDisplayFunc(Display); - glutKeyboardFunc(KeybordFunction); - glutMainLoop(); - } - - return (0); -} diff --git a/dependencies/CMakeLists.txt b/dependencies/CMakeLists.txt index cc878282c..164f3a536 100644 --- a/dependencies/CMakeLists.txt +++ b/dependencies/CMakeLists.txt @@ -179,7 +179,7 @@ else() INSTALL_DIR ${HEMELB_DEPENDENCIES_INSTALL_PATH} URL ${BOOST_TARBALL} BUILD_COMMAND "" - INSTALL_COMMAND cp -r /boost /include + INSTALL_COMMAND mkdir -p /include && cp -r /boost /include CONFIGURE_COMMAND "" BUILD_IN_SOURCE 1 ) diff --git a/dependencies/Modules/FindMPI.cmake b/dependencies/Modules/FindMPI.cmake deleted file mode 100644 index d7436af3b..000000000 --- a/dependencies/Modules/FindMPI.cmake +++ /dev/null @@ -1,622 +0,0 @@ - -# This file is part of HemeLB and is Copyright (C) -# the HemeLB team and/or their institutions, as detailed in the -# file AUTHORS. This software is provided under the terms of the -# license in the file LICENSE. -# - Find a Message Passing Interface (MPI) implementation -# The Message Passing Interface (MPI) is a library used to write -# high-performance distributed-memory parallel applications, and -# is typically deployed on a cluster. MPI is a standard interface -# (defined by the MPI forum) for which many implementations are -# available. All of them have somewhat different include paths, -# libraries to link against, etc., and this module tries to smooth -# out those differences. -# -# === Variables === -# -# This module will set the following variables per language in your project, -# where is one of C, CXX, or Fortran: -# MPI__FOUND TRUE if FindMPI found MPI flags for -# MPI__COMPILER MPI Compiler wrapper for -# MPI__COMPILE_FLAGS Compilation flags for MPI programs -# MPI__INCLUDE_PATH Include path(s) for MPI header -# MPI__LINK_FLAGS Linking flags for MPI programs -# MPI__LIBRARIES All libraries to link MPI programs against -# Additionally, FindMPI sets the following variables for running MPI -# programs from the command line: -# MPIEXEC Executable for running MPI programs -# MPIEXEC_NUMPROC_FLAG Flag to pass to MPIEXEC before giving -# it the number of processors to run on -# MPIEXEC_PREFLAGS Flags to pass to MPIEXEC directly -# before the executable to run. -# MPIEXEC_POSTFLAGS Flags to pass to MPIEXEC after other flags -# === Usage === -# -# To use this module, simply call FindMPI from a CMakeLists.txt file, or -# run find_package(MPI), then run CMake. If you are happy with the auto- -# detected configuration for your language, then you're done. If not, you -# have two options: -# 1. Set MPI__COMPILER to the MPI wrapper (mpicc, etc.) of your -# choice and reconfigure. FindMPI will attempt to determine all the -# necessary variables using THAT compiler's compile and link flags. -# 2. If this fails, or if your MPI implementation does not come with -# a compiler wrapper, then set both MPI__LIBRARIES and -# MPI__INCLUDE_PATH. You may also set any other variables -# listed above, but these two are required. This will circumvent -# autodetection entirely. -# When configuration is successful, MPI__COMPILER will be set to the -# compiler wrapper for , if it was found. MPI__FOUND and other -# variables above will be set if any MPI implementation was found for , -# regardless of whether a compiler was found. -# -# When using MPIEXEC to execute MPI applications, you should typically use -# all of the MPIEXEC flags as follows: -# ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} PROCS -# ${MPIEXEC_PREFLAGS} EXECUTABLE ${MPIEXEC_POSTFLAGS} ARGS -# where PROCS is the number of processors on which to execute the program, -# EXECUTABLE is the MPI program, and ARGS are the arguments to pass to the -# MPI program. -# -# === Backward Compatibility === -# -# For backward compatibility with older versions of FindMPI, these -# variables are set, but deprecated: -# MPI_FOUND MPI_COMPILER MPI_LIBRARY -# MPI_COMPILE_FLAGS MPI_INCLUDE_PATH MPI_EXTRA_LIBRARY -# MPI_LINK_FLAGS MPI_LIBRARIES -# In new projects, please use the MPI__XXX equivalents. - -#============================================================================= -# Copyright 2001-2011 Kitware, Inc. -# Copyright 2010-2011 Todd Gamblin tgamblin@llnl.gov -# Copyright 2001-2009 Dave Partyka -# -# Distributed under the OSI-approved BSD License (the "License"); -# see accompanying file Copyright.txt for details. -# -# This software is distributed WITHOUT ANY WARRANTY; without even the -# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the License for more information. -#============================================================================= -# (To distribute this file outside of CMake, substitute the full -# License text for the above reference.) - -# include this to handle the QUIETLY and REQUIRED arguments -include(FindPackageHandleStandardArgs) # Modified by JH -include(GetPrerequisites) - -# -# This part detects MPI compilers, attempting to wade through the mess of compiler names in -# a sensible way. -# -# The compilers are detected in this order: -# -# 1. Try to find the most generic availble MPI compiler, as this is usually set up by -# cluster admins. e.g., if plain old mpicc is available, we'll use it and assume it's -# the right compiler. -# -# 2. If a generic mpicc is NOT found, then we attempt to find one that matches -# CMAKE__COMPILER_ID. e.g. if you are using XL compilers, we'll try to find mpixlc -# and company, but not mpiicc. This hopefully prevents toolchain mismatches. -# -# If you want to force a particular MPI compiler other than what we autodetect (e.g. if you -# want to compile regular stuff with GNU and parallel stuff with Intel), you can always set -# your favorite MPI__COMPILER explicitly and this stuff will be ignored. -# - -# Start out with the generic MPI compiler names, as these are most commonly used. -set(_MPI_C_COMPILER_NAMES mpicc mpcc mpicc_r mpcc_r) -set(_MPI_CXX_COMPILER_NAMES mpicxx mpiCC mpcxx mpCC mpic++ mpc++ - mpicxx_r mpiCC_r mpcxx_r mpCC_r mpic++_r mpc++_r) -set(_MPI_Fortran_COMPILER_NAMES mpif95 mpif95_r mpf95 mpf95_r - mpif90 mpif90_r mpf90 mpf90_r - mpif77 mpif77_r mpf77 mpf77_r) - -# GNU compiler names -set(_MPI_GNU_C_COMPILER_NAMES mpigcc mpgcc mpigcc_r mpgcc_r) -set(_MPI_GNU_CXX_COMPILER_NAMES mpig++ mpg++ mpig++_r mpg++_r) -set(_MPI_GNU_Fortran_COMPILER_NAMES mpigfortran mpgfortran mpigfortran_r mpgfortran_r - mpig77 mpig77_r mpg77 mpg77_r) - -# Intel MPI compiler names -set(_MPI_Intel_C_COMPILER_NAMES mpiicc) -set(_MPI_Intel_CXX_COMPILER_NAMES mpiicpc mpiicxx mpiic++ mpiiCC) -set(_MPI_Intel_Fortran_COMPILER_NAMES mpiifort mpiif95 mpiif90 mpiif77) - -# PGI compiler names -set(_MPI_PGI_C_COMPILER_NAMES mpipgcc mppgcc) -set(_MPI_PGI_CXX_COMPILER_NAMES mpipgCC mppgCC) -set(_MPI_PGI_Fortran_COMPILER_NAMES mpipgf95 mpipgf90 mppgf95 mppgf90 mpipgf77 mppgf77) - -# XLC MPI Compiler names -set(_MPI_XL_C_COMPILER_NAMES mpxlc mpxlc_r mpixlc mpixlc_r) -set(_MPI_XL_CXX_COMPILER_NAMES mpixlcxx mpixlC mpixlc++ mpxlcxx mpxlc++ mpixlc++ mpxlCC - mpixlcxx_r mpixlC_r mpixlc++_r mpxlcxx_r mpxlc++_r mpixlc++_r mpxlCC_r) -set(_MPI_XL_Fortran_COMPILER_NAMES mpixlf95 mpixlf95_r mpxlf95 mpxlf95_r - mpixlf90 mpixlf90_r mpxlf90 mpxlf90_r - mpixlf77 mpixlf77_r mpxlf77 mpxlf77_r - mpixlf mpixlf_r mpxlf mpxlf_r) - -# append vendor-specific compilers to the list if we either don't know the compiler id, -# or if we know it matches the regular compiler. -foreach (lang C CXX Fortran) - foreach (id GNU Intel PGI XL) - if (NOT CMAKE_${lang}_COMPILER_ID OR "${CMAKE_${lang}_COMPILER_ID}" STREQUAL "${id}") - list(APPEND _MPI_${lang}_COMPILER_NAMES ${_MPI_${id}_${lang}_COMPILER_NAMES}) - endif() - unset(_MPI_${id}_${lang}_COMPILER_NAMES) # clean up the namespace here - endforeach() -endforeach() - - -# Names to try for MPI exec -set(_MPI_EXEC_NAMES mpiexec mpirun lamexec srun) - -# Grab the path to MPI from the registry if we're on windows. -set(_MPI_PREFIX_PATH) -if(WIN32) - list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH\\SMPD;binary]/..") - list(APPEND _MPI_PREFIX_PATH "[HKEY_LOCAL_MACHINE\\SOFTWARE\\MPICH2;Path]") - list(APPEND _MPI_PREFIX_PATH "$ENV{ProgramW6432}/MPICH2/") -endif() - -# Build a list of prefixes to search for MPI. -foreach(SystemPrefixDir ${CMAKE_SYSTEM_PREFIX_PATH}) - foreach(MpiPackageDir ${_MPI_PREFIX_PATH}) - if(EXISTS ${SystemPrefixDir}/${MpiPackageDir}) - list(APPEND _MPI_PREFIX_PATH "${SystemPrefixDir}/${MpiPackageDir}") - endif() - endforeach() -endforeach() - - -# -# interrogate_mpi_compiler(lang try_libs) -# -# Attempts to extract compiler and linker args from an MPI compiler. The arguments set -# by this function are: -# -# MPI__INCLUDE_PATH MPI__LINK_FLAGS MPI__FOUND -# MPI__COMPILE_FLAGS MPI__LIBRARIES -# -# MPI__COMPILER must be set beforehand to the absolute path to an MPI compiler for -# . Additionally, MPI__INCLUDE_PATH and MPI__LIBRARIES may be set -# to skip autodetection. -# -# If try_libs is TRUE, this will also attempt to find plain MPI libraries in the usual -# way. In general, this is not as effective as interrogating the compilers, as it -# ignores language-specific flags and libraries. However, some MPI implementations -# (Windows implementations) do not have compiler wrappers, so this approach must be used. -# -function (interrogate_mpi_compiler lang try_libs) - # MPI_${lang}_NO_INTERROGATE will be set to a compiler name when the *regular* compiler was - # discovered to be the MPI compiler. This happens on machines like the Cray XE6 that use - # modules to set cc, CC, and ftn to the MPI compilers. If the user force-sets another MPI - # compiler, MPI_${lang}_COMPILER won't be equal to MPI_${lang}_NO_INTERROGATE, and we'll - # inspect that compiler anew. This allows users to set new compilers w/o rm'ing cache. - string(COMPARE NOTEQUAL "${MPI_${lang}_NO_INTERROGATE}" "${MPI_${lang}_COMPILER}" interrogate) - - # If MPI is set already in the cache, don't bother with interrogating the compiler. - if (interrogate AND ((NOT MPI_${lang}_INCLUDE_PATH) OR (NOT MPI_${lang}_LIBRARIES))) - if (MPI_${lang}_COMPILER) - # Check whether the -showme:compile option works. This indicates that we have either OpenMPI - # or a newer version of LAM-MPI, and implies that -showme:link will also work. - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -showme:compile - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_COMPILE_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - - if (MPI_COMPILER_RETURN EQUAL 0) - # If we appear to have -showme:compile, then we should - # also have -showme:link. Try it. - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -showme:link - OUTPUT_VARIABLE MPI_LINK_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_LINK_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - - if (MPI_COMPILER_RETURN EQUAL 0) - # We probably have -showme:incdirs and -showme:libdirs as well, - # so grab that while we're at it. - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -showme:incdirs - OUTPUT_VARIABLE MPI_INCDIRS OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_INCDIRS ERROR_STRIP_TRAILING_WHITESPACE) - - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -showme:libdirs - OUTPUT_VARIABLE MPI_LIBDIRS OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_LIBDIRS ERROR_STRIP_TRAILING_WHITESPACE) - - else() - # reset things here if something went wrong. - set(MPI_COMPILE_CMDLINE) - set(MPI_LINK_CMDLINE) - endif() - endif () - - # Older versions of LAM-MPI have "-showme". Try to find that. - if (NOT MPI_COMPILER_RETURN EQUAL 0) - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -showme - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_COMPILE_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - endif() - - # MVAPICH uses -compile-info and -link-info. Try them. - if (NOT MPI_COMPILER_RETURN EQUAL 0) - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -compile-info - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_COMPILE_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - - # If we have compile-info, also have link-info. - if (MPI_COMPILER_RETURN EQUAL 0) - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -link-info - OUTPUT_VARIABLE MPI_LINK_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_LINK_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - endif() - - # make sure we got compile and link. Reset vars if something's wrong. - if (NOT MPI_COMPILER_RETURN EQUAL 0) - set(MPI_COMPILE_CMDLINE) - set(MPI_LINK_CMDLINE) - endif() - endif() - - # MPICH just uses "-show". Try it. - if (NOT MPI_COMPILER_RETURN EQUAL 0) - execute_process( - COMMAND ${MPI_${lang}_COMPILER} -show - OUTPUT_VARIABLE MPI_COMPILE_CMDLINE OUTPUT_STRIP_TRAILING_WHITESPACE - ERROR_VARIABLE MPI_COMPILE_CMDLINE ERROR_STRIP_TRAILING_WHITESPACE - RESULT_VARIABLE MPI_COMPILER_RETURN) - endif() - - if (MPI_COMPILER_RETURN EQUAL 0) - # We have our command lines, but we might need to copy MPI_COMPILE_CMDLINE - # into MPI_LINK_CMDLINE, if we didn't find the link line. - if (NOT MPI_LINK_CMDLINE) - set(MPI_LINK_CMDLINE ${MPI_COMPILE_CMDLINE}) - endif() - else() - message(STATUS "Unable to determine MPI from MPI driver ${MPI_${lang}_COMPILER}") - set(MPI_COMPILE_CMDLINE) - set(MPI_LINK_CMDLINE) - endif() - - # Here, we're done with the interrogation part, and we'll try to extract args we care - # about from what we learned from the compiler wrapper scripts. - - # If interrogation came back with something, extract our variable from the MPI command line - if (MPI_COMPILE_CMDLINE OR MPI_LINK_CMDLINE) - # Extract compile flags from the compile command line. - string(REGEX MATCHALL "(^| )-[Df]([^\" ]+|\"[^\"]+\")" MPI_ALL_COMPILE_FLAGS "${MPI_COMPILE_CMDLINE}") - set(MPI_COMPILE_FLAGS_WORK) - - foreach(FLAG ${MPI_ALL_COMPILE_FLAGS}) - if (MPI_COMPILE_FLAGS_WORK) - set(MPI_COMPILE_FLAGS_WORK "${MPI_COMPILE_FLAGS_WORK} ${FLAG}") - else() - set(MPI_COMPILE_FLAGS_WORK ${FLAG}) - endif() - endforeach() - - # Extract include paths from compile command line - string(REGEX MATCHALL "(^| )-I([^\" ]+|\"[^\"]+\")" MPI_ALL_INCLUDE_PATHS "${MPI_COMPILE_CMDLINE}") - foreach(IPATH ${MPI_ALL_INCLUDE_PATHS}) - string(REGEX REPLACE "^ ?-I" "" IPATH ${IPATH}) - string(REGEX REPLACE "//" "/" IPATH ${IPATH}) - list(APPEND MPI_INCLUDE_PATH_WORK ${IPATH}) - endforeach() - - # try using showme:incdirs if extracting didn't work. - if (NOT MPI_INCLUDE_PATH_WORK) - set(MPI_INCLUDE_PATH_WORK ${MPI_INCDIRS}) - separate_arguments(MPI_INCLUDE_PATH_WORK) - endif() - - # If all else fails, just search for mpi.h in the normal include paths. - if (NOT MPI_INCLUDE_PATH_WORK) - set(MPI_HEADER_PATH "MPI_HEADER_PATH-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - find_path(MPI_HEADER_PATH mpi.h - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES include) - set(MPI_INCLUDE_PATH_WORK ${MPI_HEADER_PATH}) - endif() - - # Extract linker paths from the link command line - string(REGEX MATCHALL "(^| |-Wl,)-L([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_PATHS "${MPI_LINK_CMDLINE}") - set(MPI_LINK_PATH) - foreach(LPATH ${MPI_ALL_LINK_PATHS}) - string(REGEX REPLACE "^(| |-Wl,)-L" "" LPATH ${LPATH}) - string(REGEX REPLACE "//" "/" LPATH ${LPATH}) - list(APPEND MPI_LINK_PATH ${LPATH}) - endforeach() - - # try using showme:libdirs if extracting didn't work. - if (NOT MPI_LINK_PATH) - set(MPI_LINK_PATH ${MPI_LIBDIRS}) - separate_arguments(MPI_LINK_PATH) - endif() - - # Extract linker flags from the link command line - string(REGEX MATCHALL "(^| )-Wl,([^\" ]+|\"[^\"]+\")" MPI_ALL_LINK_FLAGS "${MPI_LINK_CMDLINE}") - set(MPI_LINK_FLAGS_WORK) - foreach(FLAG ${MPI_ALL_LINK_FLAGS}) - if (MPI_LINK_FLAGS_WORK) - set(MPI_LINK_FLAGS_WORK "${MPI_LINK_FLAGS_WORK} ${FLAG}") - else() - set(MPI_LINK_FLAGS_WORK ${FLAG}) - endif() - endforeach() - - # Extract the set of libraries to link against from the link command - # line - string(REGEX MATCHALL "(^| )-l([^\" ]+|\"[^\"]+\")" MPI_LIBNAMES "${MPI_LINK_CMDLINE}") - - # Determine full path names for all of the libraries that one needs - # to link against in an MPI program - foreach(LIB ${MPI_LIBNAMES}) - string(REGEX REPLACE "^ ?-l" "" LIB ${LIB}) - # MPI_LIB is cached by find_library, but we don't want that. Clear it first. - set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - find_library(MPI_LIB NAMES ${LIB} HINTS ${MPI_LINK_PATH}) - - if (MPI_LIB) - list(APPEND MPI_LIBRARIES_WORK ${MPI_LIB}) - elseif (NOT MPI_FIND_QUIETLY) - message(WARNING "Unable to find MPI library ${LIB}") - endif() - endforeach() - - # Sanity check MPI_LIBRARIES to make sure there are enough libraries - list(LENGTH MPI_LIBRARIES_WORK MPI_NUMLIBS) - list(LENGTH MPI_LIBNAMES MPI_NUMLIBS_EXPECTED) - if (NOT MPI_NUMLIBS EQUAL MPI_NUMLIBS_EXPECTED) - set(MPI_LIBRARIES_WORK "MPI_${lang}_LIBRARIES-NOTFOUND") - endif() - endif() - - elseif(try_libs) - # If we didn't have an MPI compiler script to interrogate, attempt to find everything - # with plain old find functions. This is nasty because MPI implementations have LOTS of - # different library names, so this section isn't going to be very generic. We need to - # make sure it works for MS MPI, though, since there are no compiler wrappers for that. - find_path(MPI_HEADER_PATH mpi.h - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES include Inc) - set(MPI_INCLUDE_PATH_WORK ${MPI_HEADER_PATH}) - - # Decide between 32-bit and 64-bit libraries for Microsoft's MPI - if("${CMAKE_SIZEOF_VOID_P}" EQUAL 8) - set(MS_MPI_ARCH_DIR amd64) - else() - set(MS_MPI_ARCH_DIR i386) - endif() - - set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - find_library(MPI_LIB - NAMES mpi mpich mpich2 msmpi - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES lib lib/${MS_MPI_ARCH_DIR} Lib Lib/${MS_MPI_ARCH_DIR}) - set(MPI_LIBRARIES_WORK ${MPI_LIB}) - - # Right now, we only know about the extra libs for C++. - # We could add Fortran here (as there is usually libfmpich, etc.), but - # this really only has to work with MS MPI on Windows. - # Assume that other MPI's are covered by the compiler wrappers. - if (${lang} STREQUAL CXX) - set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - find_library(MPI_LIB - NAMES mpi++ mpicxx cxx mpi_cxx - HINTS ${_MPI_BASE_DIR} ${_MPI_PREFIX_PATH} - PATH_SUFFIXES lib) - if (MPI_LIBRARIES_WORK AND MPI_LIB) - set(MPI_LIBRARIES_WORK "${MPI_LIBRARIES_WORK} ${MPI_LIB}") - endif() - endif() - - if (NOT MPI_LIBRARIES_WORK) - set(MPI_LIBRARIES_WORK "MPI_${lang}_LIBRARIES-NOTFOUND") - endif() - endif() - - # If we found MPI, set up all of the appropriate cache entries - set(MPI_${lang}_COMPILE_FLAGS ${MPI_COMPILE_FLAGS_WORK} CACHE STRING "MPI ${lang} compilation flags" FORCE) - set(MPI_${lang}_INCLUDE_PATH ${MPI_INCLUDE_PATH_WORK} CACHE STRING "MPI ${lang} include path" FORCE) - set(MPI_${lang}_LINK_FLAGS ${MPI_LINK_FLAGS_WORK} CACHE STRING "MPI ${lang} linking flags" FORCE) - set(MPI_${lang}_LIBRARIES ${MPI_LIBRARIES_WORK} CACHE STRING "MPI ${lang} libraries to link against" FORCE) - mark_as_advanced(MPI_${lang}_COMPILE_FLAGS MPI_${lang}_INCLUDE_PATH MPI_${lang}_LINK_FLAGS MPI_${lang}_LIBRARIES) - - # clear out our temporary lib/header detectionv variable here. - set(MPI_LIB "MPI_LIB-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI lib detection" FORCE) - set(MPI_HEADER_PATH "MPI_HEADER_PATH-NOTFOUND" CACHE INTERNAL "Scratch variable for MPI header detection" FORCE) - endif() - - # finally set a found variable for each MPI language - if (MPI_${lang}_INCLUDE_PATH AND MPI_${lang}_LIBRARIES) - set(MPI_${lang}_FOUND TRUE PARENT_SCOPE) - else() - set(MPI_${lang}_FOUND FALSE PARENT_SCOPE) - endif() -endfunction() - - -# This function attempts to compile with the regular compiler, to see if MPI programs -# work with it. This is a last ditch attempt after we've tried interrogating mpicc and -# friends, and after we've tried to find generic libraries. Works on machines like -# Cray XE6, where the modules environment changes what MPI version cc, CC, and ftn use. -function(try_regular_compiler lang success) - set(scratch_directory ${CMAKE_CURRENT_BINARY_DIR}${CMAKE_FILES_DIRECTORY}) - if (${lang} STREQUAL Fortran) - set(test_file ${scratch_directory}/cmake_mpi_test.f90) - file(WRITE ${test_file} - "program hello\n" - "include 'mpif.h'\n" - "integer ierror\n" - "call MPI_INIT(ierror)\n" - "call MPI_FINALIZE(ierror)\n" - "end\n") - else() - if (${lang} STREQUAL CXX) - set(test_file ${scratch_directory}/cmake_mpi_test.cpp) - else() - set(test_file ${scratch_directory}/cmake_mpi_test.c) - endif() - file(WRITE ${test_file} - "#include \n" - "int main(int argc, char **argv) {\n" - " MPI_Init(&argc, &argv);\n" - " MPI_Finalize();\n" - "}\n") - endif() - try_compile(compiler_has_mpi ${scratch_directory} ${test_file}) - if (compiler_has_mpi) - set(MPI_${lang}_NO_INTERROGATE ${CMAKE_${lang}_COMPILER} CACHE STRING "Whether to interrogate MPI ${lang} compiler" FORCE) - set(MPI_${lang}_COMPILER ${CMAKE_${lang}_COMPILER} CACHE STRING "MPI ${lang} compiler" FORCE) - set(MPI_${lang}_COMPILE_FLAGS "" CACHE STRING "MPI ${lang} compilation flags" FORCE) - set(MPI_${lang}_INCLUDE_PATH "" CACHE STRING "MPI ${lang} include path" FORCE) - set(MPI_${lang}_LINK_FLAGS "" CACHE STRING "MPI ${lang} linking flags" FORCE) - set(MPI_${lang}_LIBRARIES "" CACHE STRING "MPI ${lang} libraries to link against" FORCE) - endif() - set(${success} ${compiler_has_mpi} PARENT_SCOPE) - unset(compiler_has_mpi CACHE) -endfunction() - -# End definitions, commence real work here. - -# Most mpi distros have some form of mpiexec which gives us something we can reliably look for. -find_program(MPIEXEC - NAMES ${_MPI_EXEC_NAMES} - PATHS ${_MPI_PREFIX_PATH} - PATH_SUFFIXES bin - DOC "Executable for running MPI programs.") - -# call get_filename_component twice to remove mpiexec and the directory it exists in (typically bin). -# This gives us a fairly reliable base directory to search for /bin /lib and /include from. -get_filename_component(_MPI_BASE_DIR "${MPIEXEC}" PATH) -get_filename_component(_MPI_BASE_DIR "${_MPI_BASE_DIR}" PATH) - -set(MPIEXEC_NUMPROC_FLAG "-np" CACHE STRING "Flag used by MPI to specify the number of processes for MPIEXEC; the next option will be the number of processes.") -set(MPIEXEC_PREFLAGS "" CACHE STRING "These flags will be directly before the executable that is being run by MPIEXEC.") -set(MPIEXEC_POSTFLAGS "" CACHE STRING "These flags will come after all flags given to MPIEXEC.") -set(MPIEXEC_MAX_NUMPROCS "2" CACHE STRING "Maximum number of processors available to run MPI applications.") -mark_as_advanced(MPIEXEC MPIEXEC_NUMPROC_FLAG MPIEXEC_PREFLAGS MPIEXEC_POSTFLAGS MPIEXEC_MAX_NUMPROCS) - - -#============================================================================= -# Backward compatibility input hacks. Propagate the FindMPI hints to C and -# CXX if the respective new versions are not defined. Translate the old -# MPI_LIBRARY and MPI_EXTRA_LIBRARY to respective MPI_${lang}_LIBRARIES. -# -# Once we find the new variables, we translate them back into their old -# equivalents below. -foreach (lang C CXX) - # Old input variables. - set(_MPI_OLD_INPUT_VARS COMPILER COMPILE_FLAGS INCLUDE_PATH LINK_FLAGS) - - # Set new vars based on their old equivalents, if the new versions are not already set. - foreach (var ${_MPI_OLD_INPUT_VARS}) - if (NOT MPI_${lang}_${var} AND MPI_${var}) - set(MPI_${lang}_${var} "${MPI_${var}}") - endif() - endforeach() - - # Special handling for MPI_LIBRARY and MPI_EXTRA_LIBRARY, which we nixed in the - # new FindMPI. These need to be merged into MPI__LIBRARIES - if (NOT MPI_${lang}_LIBRARIES AND (MPI_LIBRARY OR MPI_EXTRA_LIBRARY)) - set(MPI_${lang}_LIBRARIES ${MPI_LIBRARY} ${MPI_EXTRA_LIBRARY}) - endif() -endforeach() -#============================================================================= - - -# This loop finds the compilers and sends them off for interrogation. -foreach (lang C CXX Fortran) - if (CMAKE_${lang}_COMPILER_WORKS) - # If the user supplies a compiler *name* instead of an absolute path, assume that we need to find THAT compiler. - if (MPI_${lang}_COMPILER) - is_file_executable(MPI_${lang}_COMPILER MPI_COMPILER_IS_EXECUTABLE) - if (NOT MPI_COMPILER_IS_EXECUTABLE) - # Get rid of our default list of names and just search for the name the user wants. - set(_MPI_${lang}_COMPILER_NAMES ${MPI_${lang}_COMPILER}) - set(MPI_${lang}_COMPILER "MPI_${lang}_COMPILER-NOTFOUND" CACHE FILEPATH "Cleared" FORCE) - # If the user specifies a compiler, we don't want to try to search libraries either. - set(try_libs FALSE) - endif() - else() - set(try_libs TRUE) - endif() - - find_program(MPI_${lang}_COMPILER - NAMES ${_MPI_${lang}_COMPILER_NAMES} - PATHS "${MPI_HOME}/bin" "$ENV{MPI_HOME}/bin" ${_MPI_PREFIX_PATH}) - interrogate_mpi_compiler(${lang} ${try_libs}) - mark_as_advanced(MPI_${lang}_COMPILER) - - # last ditch try -- if nothing works so far, just try running the regular compiler and - # see if we can create an MPI executable. - set(regular_compiler_worked 0) - if (NOT MPI_${lang}_LIBRARIES OR NOT MPI_${lang}_INCLUDE_PATH) - try_regular_compiler(${lang} regular_compiler_worked) - endif() - - if (regular_compiler_worked) - find_package_handle_standard_args(MPI_${lang} DEFAULT_MSG MPI_${lang}_COMPILER) - else() - find_package_handle_standard_args(MPI_${lang} DEFAULT_MSG MPI_${lang}_LIBRARIES MPI_${lang}_INCLUDE_PATH) - endif() - endif() -endforeach() - - -#============================================================================= -# More backward compatibility stuff -# -# Bare MPI sans ${lang} vars are set to CXX then C, depending on what was found. -# This mimics the behavior of the old language-oblivious FindMPI. -set(_MPI_OLD_VARS FOUND COMPILER INCLUDE_PATH COMPILE_FLAGS LINK_FLAGS LIBRARIES) -if (MPI_CXX_FOUND) - foreach (var ${_MPI_OLD_VARS}) - set(MPI_${var} ${MPI_CXX_${var}}) - endforeach() -elseif (MPI_C_FOUND) - foreach (var ${_MPI_OLD_VARS}) - set(MPI_${var} ${MPI_C_${var}}) - endforeach() -else() - # Note that we might still have found Fortran, but you'll need to use MPI_Fortran_FOUND - set(MPI_FOUND FALSE) -endif() - -# Chop MPI_LIBRARIES into the old-style MPI_LIBRARY and MPI_EXTRA_LIBRARY, and set them in cache. -if (MPI_LIBRARIES) - list(GET MPI_LIBRARIES 0 MPI_LIBRARY_WORK) - set(MPI_LIBRARY ${MPI_LIBRARY_WORK} CACHE FILEPATH "MPI library to link against" FORCE) -else() - set(MPI_LIBRARY "MPI_LIBRARY-NOTFOUND" CACHE FILEPATH "MPI library to link against" FORCE) -endif() - -list(LENGTH MPI_LIBRARIES MPI_NUMLIBS) -if (MPI_NUMLIBS GREATER 1) - set(MPI_EXTRA_LIBRARY_WORK ${MPI_LIBRARIES}) - list(REMOVE_AT MPI_EXTRA_LIBRARY_WORK 0) - set(MPI_EXTRA_LIBRARY ${MPI_EXTRA_LIBRARY_WORK} CACHE STRING "Extra MPI libraries to link against" FORCE) -else() - set(MPI_EXTRA_LIBRARY "MPI_EXTRA_LIBRARY-NOTFOUND" CACHE STRING "Extra MPI libraries to link against" FORCE) -endif() -#============================================================================= - -# unset these vars to cleanup namespace -unset(_MPI_OLD_VARS) -unset(_MPI_PREFIX_PATH) -unset(_MPI_BASE_DIR) -foreach (lang C CXX Fortran) - unset(_MPI_${lang}_COMPILER_NAMES) -endforeach() diff --git a/deploy/compile_options.yml b/deploy/compile_options.yml index c02407c16..fdd36e3c9 100644 --- a/deploy/compile_options.yml +++ b/deploy/compile_options.yml @@ -7,10 +7,6 @@ default: CMAKE_BUILD_TYPE: "Release" CMAKE_CXX_FLAGS_RELEASE: "-O4" -no_steering: - HEMELB_STEERING_LIB: "none" -basic_steering: - HEMELB_STEERING_LIB: "basic" debug: CMAKE_BUILD_TYPE: "Debug" HEMELB_LOG_LEVEL: "debug" @@ -19,8 +15,6 @@ gdb: HEMELB_USE_DEBUGGER: "ON" multiscale: HEMELB_BUILD_MULTISCALE: "ON" -no_streaklines: - HEMELB_USE_STREAKLINES: "OFF" wait_on_connect: HEMELB_WAIT_ON_CONNECT: "ON" d3q15: diff --git a/deploy/fab.py b/deploy/fab.py index c35382d49..e0d2dfa93 100644 --- a/deploy/fab.py +++ b/deploy/fab.py @@ -160,8 +160,7 @@ def configure_cmake(configurations, extras): options.update(extras) options.update(env.cmake_options) options.update({'CMAKE_INSTALL_PREFIX':env.install_path, - "HEMELB_DEPENDENCIES_INSTALL_PATH":env.install_path, - "HEMELB_SUBPROJECT_MAKE_JOBS":env.make_jobs}) + "HEMELB_DEPENDENCIES_INSTALL_PATH":env.install_path}) env.total_cmake_options = options env.cmake_flags = ' '.join(["-D%s=%s" % option for option in env.total_cmake_options.iteritems()]) @@ -495,17 +494,13 @@ def hemelb(config, **args): config : config directory to use to define geometry, e.g. config=cylinder Keyword arguments: cores : number of compute cores to request - images : number of images to take - steering : steering session i.d. wall_time : wall-time job limit memory : memory per node """ with_config(config) execute(put_configs, config) job(dict(script='hemelb', - cores=4, images=10, steering=1111, wall_time='0:15:0', memory='2G'), args) - if args.get('steer', False): - execute(steer, env.name, retry=True, framerate=args.get('framerate'), orbit=args.get('orbit')) + cores=4, wall_time='0:15:0', memory='2G'), args) @task def multiscale_hemelb(config,**args): @@ -515,18 +510,13 @@ def multiscale_hemelb(config,**args): config : config directory to use to define geometry, e.g. config=cylinder Keyword arguments: cores : number of compute cores to request - images : number of images to take - steering : steering session i.d. wall_time : wall-time job limit memory : memory per node """ with_config(config) execute(put_configs,config) job(dict(script='multiscale_hemelb', - cores=4,images=10, steering=1111, wall_time='0:15:0',memory='2G'),args) - if args.get('steer',False): - execute(steer,env.name,retry=True,framerate=args.get('framerate'),orbit=args.get('orbit')) - + cores=4, wall_time='0:15:0',memory='2G'),args) @task def resubmit(name): @@ -543,7 +533,7 @@ def multijob(*names, **args): env.jobstorun = "\n".join(jobscriptpaths) # And then, submit it job(dict(script='multijob', job_name_template='multijob', - cores=4, images=10, steering=1111, wall_time='0:15:0', memory='2G'), args) + cores=4, wall_time='0:15:0', memory='2G'), args) @task def hemelbs(config, **args): @@ -554,8 +544,6 @@ def hemelbs(config, **args): config : config directory to use to define geometry, e.g. config=cylinder Keyword arguments: cores : number of compute cores to request - images : number of images to take - steering : steering session i.d. wall_time : wall-time job limit memory : memory per node """ @@ -575,7 +563,7 @@ def hemelb_benchmark(config, min_cores, max_cores, **args): with_config(config) execute(put_configs, config) job(dict(script='hemelb', - cores=cores_used, images=10, steering=1111, wall_time='0:15:0', memory='2G'), args) + cores=cores_used, wall_time='0:15:0', memory='2G'), args) cores_used *= 2 @task(alias='regress') @@ -585,7 +573,7 @@ def regression_test(**args): execute(copy_regression_tests) execute(build_python_tools) job(dict(job_name_template='regression_${build_number}_${machine_name}', cores=3, - wall_time='0:20:0', memory='2G', images=0, steering=1111, script='regression'), args) + wall_time='0:20:0', memory='2G', script='regression'), args) def calc_nodes(): # If we're not reserving whole nodes, then if we request less than one node's worth of cores, need to keep N<=n @@ -808,36 +796,6 @@ def vampir(original_job, *args): def vampir_tunnel(node, port): local("ssh hector -L 30070:nid%s:%s -N" % node, port) -@task -def steer(job, orbit=False, view=False, retry=False, framerate=None): - with_job(job) - if view: - env.steering_client = 'steering.py' - else: - env.steering_client = 'timing_client.py' - if orbit: - env.steering_options = "--orbit" - else: - env.steering_options = "" - if retry: - env.steering_options += " --retry" - if framerate: - env.steering_options += " --MaxFramerate=%s" % framerate - command_template = "python $repository_path/Tools/steering/python/hemelb_steering/${steering_client} ${steering_options} ${running_node} >> $job_results/steering_results.txt" - if retry: - while True: - try: - get_running_location() - run(template(command_template)) - break - except: - print "Couldn't connect. Will retry" - execute(stat) - time.sleep(10) - else: - get_running_location() - run(template(command_template)) - @task def ensemble(config,cores,wall_time): diff --git a/deploy/machines.yml b/deploy/machines.yml index be747417b..27087f21a 100644 --- a/deploy/machines.yml +++ b/deploy/machines.yml @@ -62,7 +62,9 @@ archer: #the ARCHER supercomputer at EPCC make_jobs: 4 verbose: 1 job_dispatch: "qsub" - run_command: "aprun -n $cores -N $corespernode" + # -n : total number of MPI ranks + # -N : MPI ranks per node (must be <= n) default is min(n, 24) + run_command: "aprun -n $cores" batch_header: pbs-archer remote: "login.archer.ac.uk" # On ARCHER, *all files* which are needed at runtime, must be on the /work filesystem, so we must make the install location be on the /work filesystem @@ -70,21 +72,61 @@ archer: #the ARCHER supercomputer at EPCC home_path_template: "/home/$project/$project/$username" runtime_path_template: "/work/$project/$project/$username" fabric_dir: "FabricHemeLb" - modules: ["load cmake/3.2.3", "swap PrgEnv-cray PrgEnv-gnu", "load boost", "load cray-hdf5-parallel", "load cray-tpsl"] - build_prefix_commands: ["export LDFLAGS=-dynamic"] # Required for HDF5 to compile + modules: ["load cmake/3.2.3", "swap PrgEnv-cray PrgEnv-gnu", "load boost", "load cray-hdf5-parallel", "load cray-tpsl", "load anaconda/2.2.0-python2"] + # Tell autoconf for dependencies where the compilers are + build_prefix_commands: ["export LDFLAGS=-dynamic","export CXX=CC","export CC=cc", "export LD=CC", "export XTPE_LINK_TYPE=dynamic" ] temp_path_template: "$work_path/tmp" regression_test_path_template: "$work_path/regression" queue: "standard" python_build: "lib/python2.7" corespernode: 24 + run_prefix_commands: + # Changes here must also be made below in archer_dmapp + - export MPICH_MAX_THREAD_SAFETY=multiple + - export MPICH_NEMESIS_ASYNC_PROGRESS=1 + - export MPICH_GNI_USE_UNASSIGNED_CPUS=enabled cmake_options: + # Changes here must also be made below in archer_dmapp + CMAKE_CXX_FLAGS: '"-dynamic -Wno-unused-local-typedefs"' + CMAKE_C_FLAGS: "-dynamic" + HEMELB_USE_ALL_WARNINGS_GNU: "OFF" + CMAKE_CXX_COMPILER: "CC" + CMAKE_C_COMPILER: "cc" + CMAKE_CXX_FLAGS_RELEASE: "" + MPI_CXX_COMPILER: "CC" + MPI_C_COMPILER: "cc" HEMELB_OPTIMISATION: "-O3" + HEMELB_DEPENDENCIES_SET_RPATH: "OFF" + HEMELB_USE_SSE3: "ON" + CTEMPLATE_USE_STATIC: "OFF" + HEMELB_COMPUTE_ARCHITECTURE: "INTELSANDYBRIDGE" + METIS_INCLUDE_DIR: "$CRAY_TPSL_PREFIX_DIR/include" + PARMETIS_INCLUDE_DIR: "$CRAY_TPSL_PREFIX_DIR/include" + +archer_dmapp: + import: "archer" + cmake_options: + CMAKE_EXE_LINKER_FLAGS: '"-dynamic $CRAY_DMAPP_POST_LINK_OPTS -ldmapp"' + CMAKE_CXX_FLAGS: '"-dynamic -Wno-unused-local-typedefs"' + CMAKE_C_FLAGS: "-dynamic" + HEMELB_USE_ALL_WARNINGS_GNU: "OFF" + CMAKE_CXX_COMPILER: "CC" + CMAKE_C_COMPILER: "cc" + CMAKE_CXX_FLAGS_RELEASE: "" + MPI_CXX_COMPILER: "CC" + MPI_C_COMPILER: "cc" + HEMELB_OPTIMISATION: "-O3" + HEMELB_DEPENDENCIES_SET_RPATH: "OFF" HEMELB_USE_SSE3: "ON" + CTEMPLATE_USE_STATIC: "OFF" HEMELB_COMPUTE_ARCHITECTURE: "INTELSANDYBRIDGE" METIS_INCLUDE_DIR: "$CRAY_TPSL_PREFIX_DIR/include" - METIS_LIBRARY: "$CRAY_TPSL_PREFIX_DIR/lib" PARMETIS_INCLUDE_DIR: "$CRAY_TPSL_PREFIX_DIR/include" - PARMETIS_LIBRARY: "$CRAY_TPSL_PREFIX_DIR/lib" + run_prefix_commands: + - export MPICH_MAX_THREAD_SAFETY=multiple + - export MPICH_NEMESIS_ASYNC_PROGRESS=1 + - export MPICH_GNI_USE_UNASSIGNED_CPUS=enabled + - export MPICH_USE_DMAPP_COLL=1 legion: job_dispatch: "qsub" diff --git a/deploy/templates/hemelb b/deploy/templates/hemelb index 5f3a32d8b..857a09d06 100644 --- a/deploy/templates/hemelb +++ b/deploy/templates/hemelb @@ -9,4 +9,4 @@ cd $job_results $run_prefix rm -rf results cp $job_config_path/* . -$run_command $install_path/bin/$executable -in config.xml -i $images +$run_command $install_path/bin/$executable -in config.xml diff --git a/deploy/templates/multiscale_hemelb b/deploy/templates/multiscale_hemelb index 584b0eeaf..918887a5d 100644 --- a/deploy/templates/multiscale_hemelb +++ b/deploy/templates/multiscale_hemelb @@ -7,4 +7,4 @@ cd $job_results $run_prefix rm -rf results cp $job_config_path/* . -$run_command $install_path/bin/multiscale_hemelb -in config.xml -i $images -s $snapshots -ss $steering +$run_command $install_path/bin/multiscale_hemelb -in config.xml -s $snapshots diff --git a/deploy/templates/regression b/deploy/templates/regression index b989fed6b..c1cb2b426 100644 --- a/deploy/templates/regression +++ b/deploy/templates/regression @@ -6,6 +6,5 @@ cd $job_results $run_prefix rm -rf results -$run_command $install_path/bin/hemelb -in $regression_test_path/config.xml -out $job_results/results -i $images -ss $steering -$run_command_one_proc $regression_test_path/ImageComparison $regression_test_path/CleanImages results/Images +$run_command $install_path/bin/hemelb -in $regression_test_path/config.xml -out $job_results/results $run_command_one_proc $regression_test_path/NumericalComparison $regression_test_path/CleanExtracted results/Extracted \ No newline at end of file diff --git a/deploy/test/fixtures/templates/hemelb b/deploy/test/fixtures/templates/hemelb index 2de0d1421..aef351b9b 100644 --- a/deploy/test/fixtures/templates/hemelb +++ b/deploy/test/fixtures/templates/hemelb @@ -2,4 +2,4 @@ cd $job_results $run_prefix rm -rf results cp $job_config_path/* . -mpirun -np $cores $install_path/bin/hemelb -in config.xml -i $images -ss $steering +mpirun -np $cores $install_path/bin/hemelb -in config.xml