From afba801654016e29f327c7cac985ed607adcfd7e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 29 May 2024 06:16:26 -0700 Subject: [PATCH 01/87] Make particle IDs valid in BoundaryScraping (#4955) * Make particle IDs valid in BoundaryScraping * Add CI test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix automated test * Remove test in scraping with filtering This is because the scraping diagnostic does not output all particles in that case (particle filtering is on). * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Examples/Tests/scraping/analysis_rz.py | 8 ++++++++ Source/Particles/ParticleBoundaryBuffer.cpp | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/Examples/Tests/scraping/analysis_rz.py b/Examples/Tests/scraping/analysis_rz.py index 193b2575992..11d0194707f 100755 --- a/Examples/Tests/scraping/analysis_rz.py +++ b/Examples/Tests/scraping/analysis_rz.py @@ -60,6 +60,14 @@ def n_scraped_particles( iteration ): n_total = n_remaining[0] assert np.all( n_scraped+n_remaining == n_total) +# Check that the particle IDs match between the initial iteration +# (all particles in the simulation domain) and the finall iteration (particles are either scraped or still in simulation box) +id_initial, = ts_full.get_particle(['id'], iteration=0) +id_final_scrape, = ts_scraping.get_particle(['id'], iteration=ts_scraping.iterations[0]) +id_final_box, = ts_full.get_particle(['id'], iteration=ts_full.iterations[-1]) +id_final = np.concatenate( (id_final_scrape, id_final_box)) +assert np.all( np.sort(id_initial) == np.sort(id_final) ) # Sort because particles may not be in the same order + # Checksum test test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 345a0bfa592..784c27816da 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -160,6 +160,9 @@ struct FindEmbeddedBoundaryIntersection { #else amrex::ignore_unused(x_temp, y_temp, z_temp,normal); #endif + + // flip id to positive in destination + amrex::ParticleIDWrapper{dst.m_idcpu[dst_i]}.make_valid(); } }; #endif @@ -199,6 +202,9 @@ struct CopyAndTimestamp { dst.m_runtime_rdata[m_normal_index][dst_i]= n[0]; dst.m_runtime_rdata[m_normal_index+1][dst_i]= n[1]; dst.m_runtime_rdata[m_normal_index+2][dst_i]= n[2]; + + // flip id to positive in destination + amrex::ParticleIDWrapper{dst.m_idcpu[dst_i]}.make_valid(); } }; From 0e41d93892dd5925b02bb22f57dda8d226246484 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 29 May 2024 18:17:41 +0200 Subject: [PATCH 02/87] Profiler: use amrex logic for device synchronization instead of custom solution (#4889) * Use amrex logic for device synchronization in tiny profiler instead of custom solution * improve comment * remove useless comment * fix missing include * fix bug * fix bug --- Source/Diagnostics/ParticleIO.cpp | 1 + .../HybridPICModel/HybridPICModel.H | 2 + .../HybridPICModel/HybridPICModel.cpp | 1 + .../SpectralSolver/SpectralSolver.cpp | 1 + Source/Filter/Filter.cpp | 1 + Source/Initialization/WarpXAMReXInit.cpp | 19 +++++++ Source/Initialization/WarpXInitData.cpp | 1 + Source/Particles/LaserParticleContainer.cpp | 1 + Source/Particles/WarpXParticleContainer.cpp | 4 +- Source/Utils/WarpXProfilerWrapper.H | 18 +++---- Source/WarpX.cpp | 8 --- Source/ablastr/fields/PoissonSolver.H | 2 +- Source/ablastr/math/fft/WrapCuFFT.cpp | 6 +-- Source/ablastr/particles/DepositCharge.H | 16 +++--- Source/ablastr/profiler/ProfilerWrapper.H | 54 +++---------------- 15 files changed, 52 insertions(+), 83 deletions(-) diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index f1f4d426a4a..bfb1867e741 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -18,6 +18,7 @@ #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" #include diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 5c0c2bcc96a..15208727559 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -22,6 +22,8 @@ #include #include +#include + /** * \brief This class contains the parameters needed to evaluate hybrid field * solutions (kinetic ions with fluid electrons). diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 8979036fbea..6a72bb3569c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -10,6 +10,7 @@ #include "HybridPICModel.H" #include "FieldSolver/Fields.H" +#include "WarpX.H" using namespace amrex; using namespace warpx::fields; diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index 80fb2c39545..460afee999f 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -14,6 +14,7 @@ #include "SpectralKSpace.H" #include "SpectralSolver.H" #include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXProfilerWrapper.H" #include diff --git a/Source/Filter/Filter.cpp b/Source/Filter/Filter.cpp index 5952ffde59c..6243ce4ebbf 100644 --- a/Source/Filter/Filter.cpp +++ b/Source/Filter/Filter.cpp @@ -9,6 +9,7 @@ #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" #include #include diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index 4036a49235c..900dbb53e06 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -7,9 +7,12 @@ #include "Initialization/WarpXAMReXInit.H" +#include "Utils/TextMsg.H" + #include #include #include +#include #include @@ -46,6 +49,22 @@ namespace { pp_amr.add("blocking_factor", 1); } + //See https://github.com/AMReX-Codes/amrex/pull/3763 +#ifdef AMREX_USE_GPU + bool warpx_do_device_synchronize = true; +#else + bool warpx_do_device_synchronize = false; +#endif + pp_warpx.query("do_device_synchronize", warpx_do_device_synchronize); + bool do_device_synchronize = warpx_do_device_synchronize; + amrex::ParmParse pp_tiny_profiler("tiny_profiler"); + if (pp_tiny_profiler.queryAdd("device_synchronize_around_region", do_device_synchronize) ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + do_device_synchronize == warpx_do_device_synchronize, + "tiny_profiler.device_synchronize_around_region overrides warpx.do_device_synchronize."); + } + // Here we override the default tiling option for particles, which is always // "false" in AMReX, to "false" if compiling for GPU execution and "true" // if compiling for CPU. diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index ff2ca01deec..0d5ab70c657 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -16,6 +16,7 @@ #endif #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "FieldSolver/Fields.H" #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Filter/BilinearFilter.H" diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index bf846c674f0..7b735053f0b 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -19,6 +19,7 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" #include diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 8692dff3302..4800d9e209f 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1161,9 +1161,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, ng_rho, depos_lev, ref_ratio, offset, np_to_deposit, - icomp, nc, - WarpX::do_device_synchronize - ); + icomp, nc); } } diff --git a/Source/Utils/WarpXProfilerWrapper.H b/Source/Utils/WarpXProfilerWrapper.H index 856fc9d0e02..8b13b0df2ab 100644 --- a/Source/Utils/WarpXProfilerWrapper.H +++ b/Source/Utils/WarpXProfilerWrapper.H @@ -8,17 +8,13 @@ #ifndef WARPX_PROFILERWRAPPER_H_ #define WARPX_PROFILERWRAPPER_H_ -#include "WarpX.H" -#include "ablastr/profiler/ProfilerWrapper.H" +#include - -// `BL_PROFILE_PASTE(SYNC_SCOPE_, __COUNTER__)` and `SYNC_V_##vname` used to make unique names for -// synchronizeOnDestruct objects, like `SYNC_SCOPE_0` and `SYNC_V_pmain` -#define WARPX_PROFILE(fname) ABLASTR_PROFILE(fname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR(fname, vname) ABLASTR_PROFILE_VAR(fname, vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR_NS(fname, vname) ABLASTR_PROFILE_VAR_NS(fname, vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR_START(vname) ABLASTR_PROFILE_VAR_START(vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR_STOP(vname) ABLASTR_PROFILE_VAR_STOP(vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_REGION(rname) ABLASTR_PROFILE_REGION(rname, WarpX::do_device_synchronize) +#define WARPX_PROFILE(fname) BL_PROFILE(fname) +#define WARPX_PROFILE_VAR(fname, vname) BL_PROFILE_VAR(fname, vname) +#define WARPX_PROFILE_VAR_NS(fname, vname) BL_PROFILE_VAR_NS(fname, vname) +#define WARPX_PROFILE_VAR_START(vname) BL_PROFILE_VAR_START(vname) +#define WARPX_PROFILE_VAR_STOP(vname) BL_PROFILE_VAR_STOP(vname) +#define WARPX_PROFILE_REGION(rname) BL_PROFILE_REGION(rname) #endif // WARPX_PROFILERWRAPPER_H_ diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 8c3a6f15b1c..b4c088383ef 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -211,12 +211,6 @@ int WarpX::n_current_deposition_buffer = -1; short WarpX::grid_type; amrex::IntVect m_rho_nodal_flag; -#ifdef AMREX_USE_GPU -bool WarpX::do_device_synchronize = true; -#else -bool WarpX::do_device_synchronize = false; -#endif - WarpX* WarpX::m_instance = nullptr; void WarpX::MakeWarpX () @@ -683,8 +677,6 @@ WarpX::ReadParameters () ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction); - pp_warpx.query("do_device_synchronize", do_device_synchronize); - // queryWithParser returns 1 if argument zmax_plasma_to_compute_max_step is // specified by the user, 0 otherwise. do_compute_max_step_from_zmax = utils::parser::queryWithParser( diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index da0078f8b5a..106c5475dd7 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -116,7 +116,7 @@ computePhi (amrex::Vector const & rho, { using namespace amrex::literals; - ABLASTR_PROFILE("computePhi", false); + ABLASTR_PROFILE("computePhi"); if (!rel_ref_ratio.has_value()) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(rho.size() == 1u, diff --git a/Source/ablastr/math/fft/WrapCuFFT.cpp b/Source/ablastr/math/fft/WrapCuFFT.cpp index b6b1706f807..9ceb91457c9 100644 --- a/Source/ablastr/math/fft/WrapCuFFT.cpp +++ b/Source/ablastr/math/fft/WrapCuFFT.cpp @@ -31,7 +31,7 @@ namespace ablastr::math::anyfft Complex * const complex_array, const direction dir, const int dim) { FFTplan fft_plan; - ABLASTR_PROFILE("ablastr::math::anyfft::CreatePlan", false); + ABLASTR_PROFILE("ablastr::math::anyfft::CreatePlan"); // Initialize fft_plan.m_plan with the vendor fft plan. cufftResult result; @@ -71,12 +71,12 @@ namespace ablastr::math::anyfft void DestroyPlan(FFTplan& fft_plan) { - ABLASTR_PROFILE("ablastr::math::anyfft::DestroyPlan", false); + ABLASTR_PROFILE("ablastr::math::anyfft::DestroyPlan"); cufftDestroy( fft_plan.m_plan ); } void Execute(FFTplan& fft_plan){ - ABLASTR_PROFILE("ablastr::math::anyfft::Execute", false); + ABLASTR_PROFILE("ablastr::math::anyfft::Execute"); // make sure that this is done on the same GPU stream as the above copy cudaStream_t stream = amrex::Gpu::Device::cudaStream(); cufftSetStream ( fft_plan.m_plan, stream); diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index b2dedccd03e..ff3741a7a43 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -44,7 +44,6 @@ namespace ablastr::particles * \param np_to_deposit number of particles to deposit (default: pti.numParticles()) * \param icomp component in MultiFab to start depositing to * \param nc number of components to deposit - * \param do_device_synchronize call amrex::Gpu::synchronize() for tiny profiler regions (default: true) */ template< typename T_PC > static void @@ -63,8 +62,7 @@ deposit_charge (typename T_PC::ParIterType& pti, std::optional rel_ref_ratio = std::nullopt, long const offset = 0, std::optional np_to_deposit = std::nullopt, - int const icomp = 0, int const nc = 1, - bool const do_device_synchronize = true) + int const icomp = 0, int const nc = 1) { // deposition guards amrex::IntVect ng_rho = rho->nGrowVect(); @@ -131,8 +129,8 @@ deposit_charge (typename T_PC::ParIterType& pti, amrex::numParticlesOutOfRange(pti, range) == 0, "Particles shape does not fit within tile (CPU) or guard cells (GPU) used for charge deposition"); - ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::ChargeDeposition", blp_ppc_chd, do_device_synchronize); - ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::Accumulate", blp_accumulate, do_device_synchronize); + ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::ChargeDeposition", blp_ppc_chd); + ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::Accumulate", blp_accumulate); // Get tile box where charge is deposited. // The tile box is different when depositing in the buffers (depos_lev(GetPosition, wp.dataPtr()+offset, ion_lev, @@ -192,13 +190,13 @@ deposit_charge (typename T_PC::ParIterType& pti, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes); } - ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd, do_device_synchronize); + ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd); #ifndef AMREX_USE_GPU // CPU, tiling: atomicAdd local_rho into rho - ABLASTR_PROFILE_VAR_START(blp_accumulate, do_device_synchronize); + ABLASTR_PROFILE_VAR_START(blp_accumulate); (*rho)[pti].lockAdd(local_rho, tb, tb, 0, icomp*nc, nc); - ABLASTR_PROFILE_VAR_STOP(blp_accumulate, do_device_synchronize); + ABLASTR_PROFILE_VAR_STOP(blp_accumulate); #endif } diff --git a/Source/ablastr/profiler/ProfilerWrapper.H b/Source/ablastr/profiler/ProfilerWrapper.H index 6b476d78114..268c1abd971 100644 --- a/Source/ablastr/profiler/ProfilerWrapper.H +++ b/Source/ablastr/profiler/ProfilerWrapper.H @@ -9,54 +9,12 @@ #define ABLASTR_PROFILERWRAPPER_H_ #include -#include - -namespace ablastr::profiler -{ - /** Conditionally synchronizes active GPU operations - * - * @param do_device_synchronize perform amrex::Gpu::synchronize() if true - */ - AMREX_FORCE_INLINE - void - device_synchronize(bool const do_device_synchronize = false) { - if (do_device_synchronize) { - amrex::Gpu::synchronize(); - } - } - - /** An object that conditionally calls device_synchronize() on destruction - * - * Note that objects are destructed in the reverse order of declaration - */ - struct SynchronizeOnDestruct { - SynchronizeOnDestruct(bool const do_device_synchronize = false) - : m_do_device_synchronize(do_device_synchronize) {} - - AMREX_FORCE_INLINE - ~SynchronizeOnDestruct() { - device_synchronize(m_do_device_synchronize); - } - - // default move and copy operations - SynchronizeOnDestruct(const SynchronizeOnDestruct&) = default; - SynchronizeOnDestruct& operator=(const SynchronizeOnDestruct&) = default; - SynchronizeOnDestruct(SynchronizeOnDestruct&&) = default; - SynchronizeOnDestruct& operator=(SynchronizeOnDestruct&& field_data) = default; - - bool m_do_device_synchronize = false; - }; - -} // namespace ablastr::profiler - -// `BL_PROFILE_PASTE(SYNC_SCOPE_, __COUNTER__)` and `SYNC_V_##vname` used to make unique names for -// synchronizeOnDestruct objects, like `SYNC_SCOPE_0` and `SYNC_V_pmain` -#define ABLASTR_PROFILE(fname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE(fname); const ablastr::profiler::SynchronizeOnDestruct BL_PROFILE_PASTE(SYNC_SCOPE_, __COUNTER__){sync} -#define ABLASTR_PROFILE_VAR(fname, vname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_VAR(fname, vname); const ablastr::profiler::SynchronizeOnDestruct SYNC_V_##vname{sync} -#define ABLASTR_PROFILE_VAR_NS(fname, vname, sync) BL_PROFILE_VAR_NS(fname, vname); const ablastr::profiler::SynchronizeOnDestruct SYNC_V_##vname{sync} -#define ABLASTR_PROFILE_VAR_START(vname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_VAR_START(vname) -#define ABLASTR_PROFILE_VAR_STOP(vname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_VAR_STOP(vname) -#define ABLASTR_PROFILE_REGION(rname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_REGION(rname); const ablastr::profiler::SynchronizeOnDestruct BL_PROFILE_PASTE(SYNC_R_, __COUNTER__){sync} +#define ABLASTR_PROFILE(fname) BL_PROFILE(fname) +#define ABLASTR_PROFILE_VAR(fname, vname) BL_PROFILE_VAR(fname, vname) +#define ABLASTR_PROFILE_VAR_NS(fname, vname) BL_PROFILE_VAR_NS(fname, vname) +#define ABLASTR_PROFILE_VAR_START(vname) BL_PROFILE_VAR_START(vname) +#define ABLASTR_PROFILE_VAR_STOP(vname) BL_PROFILE_VAR_STOP(vname) +#define ABLASTR_PROFILE_REGION(rname) BL_PROFILE_REGION(rname) #endif // ABLASTR_PROFILERWRAPPER_H_ From 25a4c5dc2969e4053b46adccd09d923628906974 Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 29 May 2024 13:46:03 -0700 Subject: [PATCH 03/87] Add particle boundary none (#4920) * Add None particle boundary condition * Use None particle boundary in CI test langmuir/inputs_rz * Update BC for picmi input files --- Docs/source/usage/parameters.rst | 4 ++++ .../laser_acceleration/PICMI_inputs_rz.py | 2 +- .../spacecraft_charging/PICMI_inputs_rz.py | 2 +- Examples/Tests/langmuir/PICMI_inputs_rz.py | 2 +- Examples/Tests/langmuir/inputs_rz | 2 ++ Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py | 2 +- Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py | 2 +- .../particle_boundary_interaction/PICMI_inputs_rz.py | 2 +- Source/Utils/WarpXAlgorithmSelection.H | 3 ++- Source/Utils/WarpXAlgorithmSelection.cpp | 1 + Source/WarpX.cpp | 8 ++++++++ 11 files changed, 23 insertions(+), 7 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 40c16a06df5..4f011025213 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -437,6 +437,10 @@ Domain Boundary Conditions The standard deviation for these distributions should be provided for each species using ``boundary..u_th``. The same standard deviation is used to sample all components. + * ``None``: No boundary conditions are applied to the particles. + When using RZ, this option must be used for the lower radial boundary, the first value of ``boundary.particle_lo``. + This should not be used in any other cases. + * ``boundary.reflect_all_velocities`` (`bool`) optional (default `false`) For a reflecting boundary condition, this flags whether the sign of only the normal velocity is changed or all velocities. diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py index 9990cca01cf..7f09db8d6b3 100755 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py @@ -31,7 +31,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'absorbing'], + lower_boundary_conditions_particles = ['none', 'absorbing'], upper_boundary_conditions_particles = ['absorbing', 'absorbing'], moving_window_velocity = [0., c], warpx_max_grid_size = max_grid_size, diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index d541c2393ca..5b57b59fbe3 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -220,7 +220,7 @@ def compute_actual_charge_on_spacecraft(): upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'reflecting'], + lower_boundary_conditions_particles = ['none', 'reflecting'], upper_boundary_conditions_particles = ['absorbing', 'reflecting'] ) diff --git a/Examples/Tests/langmuir/PICMI_inputs_rz.py b/Examples/Tests/langmuir/PICMI_inputs_rz.py index 8328aba5185..8da03b00469 100755 --- a/Examples/Tests/langmuir/PICMI_inputs_rz.py +++ b/Examples/Tests/langmuir/PICMI_inputs_rz.py @@ -90,7 +90,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'periodic'], upper_boundary_conditions = ['none', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], + lower_boundary_conditions_particles = ['none', 'periodic'], upper_boundary_conditions_particles = ['absorbing', 'periodic'], moving_window_velocity = [0.,0.], warpx_max_grid_size=64) diff --git a/Examples/Tests/langmuir/inputs_rz b/Examples/Tests/langmuir/inputs_rz index 534296759a2..3b70a91f207 100644 --- a/Examples/Tests/langmuir/inputs_rz +++ b/Examples/Tests/langmuir/inputs_rz @@ -28,6 +28,8 @@ geometry.prob_lo = 0.e-6 -20.e-6 # physical domain geometry.prob_hi = 20.e-6 20.e-6 boundary.field_lo = none periodic boundary.field_hi = none periodic +boundary.particle_lo = none periodic +boundary.particle_hi = absorbing periodic warpx.serialize_initial_conditions = 1 diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py index 1268f7a02b0..e46f561f538 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py @@ -69,7 +69,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['neumann', 'neumann'], - lower_boundary_conditions_particles = ['reflecting', 'absorbing'], + lower_boundary_conditions_particles = ['none', 'absorbing'], upper_boundary_conditions_particles = ['absorbing', 'absorbing'], warpx_potential_lo_z = V_domain_boundary, warpx_blocking_factor=8, diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py index 89c5230a6ad..5bd1e3518f9 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py @@ -156,7 +156,7 @@ def setup_run(self): upper_bound=[self.Lr, self.Lz/2.0], lower_boundary_conditions = ['none', 'periodic'], upper_boundary_conditions = ['dirichlet', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], + lower_boundary_conditions_particles = ['none', 'periodic'], upper_boundary_conditions_particles = ['reflecting', 'periodic'] ) simulation.time_step_size = self.dt diff --git a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py index 4017a05d413..df4a4579e2f 100644 --- a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py +++ b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py @@ -39,7 +39,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'reflecting'], + lower_boundary_conditions_particles = ['none', 'reflecting'], upper_boundary_conditions_particles = ['absorbing', 'reflecting'] ) diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index c34516aedc5..e067ed03a80 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -175,7 +175,8 @@ enum struct ParticleBoundaryType { Open = 1, //!< particles cross domain boundary leave with damped j Reflecting = 2, //!< particles are reflected Periodic = 3, //!< particles are introduced from the periodic boundary - Thermal = 4 + Thermal = 4, + None = 5 //!< For r=0 boundary with RZ simulations }; /** MPI reductions diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index b1f93c2f6c8..2449ea2f8dd 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -142,6 +142,7 @@ const std::map ParticleBCType_algo_to_enum = {"reflecting", ParticleBoundaryType::Reflecting}, {"periodic", ParticleBoundaryType::Periodic}, {"thermal", ParticleBoundaryType::Thermal}, + {"none", ParticleBoundaryType::None}, {"default", ParticleBoundaryType::Absorbing} }; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index b4c088383ef..de632211bad 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1169,6 +1169,14 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( WarpX::field_boundary_lo[0] == FieldBoundaryType::None, "Error : Field boundary at r=0 must be ``none``. \n"); + + const ParmParse pp_boundary("boundary"); + if (pp_boundary.contains("particle_lo")) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + WarpX::particle_boundary_lo[0] == ParticleBoundaryType::None, + "Error : Particle boundary at r=0 must be ``none``. \n"); + } + } if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { From d7b2ed28bc40e1c6d19cc60c2900f28006bb846f Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 30 May 2024 00:05:37 +0200 Subject: [PATCH 04/87] make RemakeMultiFab a lambda function inside WarpX::RemakeLevel (#4947) --- Source/Parallelization/WarpXRegrid.cpp | 105 ++++++++++++------------- 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 22979912a84..2846e79c361 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -164,22 +164,19 @@ WarpX::LoadBalance () #endif } - -template void -RemakeMultiFab (std::unique_ptr& mf, const DistributionMapping& dm, - const bool redistribute, const int lev) -{ - if (mf == nullptr) { return; } - const IntVect& ng = mf->nGrowVect(); - std::unique_ptr pmf; - WarpX::AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); - if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } - mf = std::move(pmf); -} - void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { + + const auto RemakeMultiFab = [&](auto& mf, const bool redistribute){ + if (mf == nullptr) { return; } + const IntVect& ng = mf->nGrowVect(); + auto pmf = std::remove_reference_t{}; + AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); + if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } + mf = std::move(pmf); + }; + if (ba == boxArray(lev)) { if (ParallelDescriptor::NProcs() == 1) { return; } @@ -187,60 +184,60 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi // Fine patch for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_fp[lev][idim], dm, true ,lev); - RemakeMultiFab(Efield_fp[lev][idim], dm, true ,lev); + RemakeMultiFab(Bfield_fp[lev][idim], true); + RemakeMultiFab(Efield_fp[lev][idim], true); if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Bfield_fp_external[lev][idim], dm, true ,lev); + RemakeMultiFab(Bfield_fp_external[lev][idim], true); } if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Efield_fp_external[lev][idim], dm, true ,lev); + RemakeMultiFab(Efield_fp_external[lev][idim], true); } - RemakeMultiFab(current_fp[lev][idim], dm, false ,lev); - RemakeMultiFab(current_store[lev][idim], dm, false ,lev); + RemakeMultiFab(current_fp[lev][idim], false); + RemakeMultiFab(current_store[lev][idim], false); if (current_deposition_algo == CurrentDepositionAlgo::Vay) { - RemakeMultiFab(current_fp_vay[lev][idim], dm, false ,lev); + RemakeMultiFab(current_fp_vay[lev][idim], false); } if (do_current_centering) { - RemakeMultiFab(current_fp_nodal[lev][idim], dm, false ,lev); + RemakeMultiFab(current_fp_nodal[lev][idim], false); } if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_fp[lev][idim], dm, true ,lev); - RemakeMultiFab(Bfield_avg_fp[lev][idim], dm, true ,lev); + RemakeMultiFab(Efield_avg_fp[lev][idim], true); + RemakeMultiFab(Bfield_avg_fp[lev][idim], true); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], dm, true, lev); - RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], dm, false, lev); - RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim], dm, true, lev); + RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], true); + RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], false); + RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim],true); } #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - RemakeMultiFab(m_edge_lengths[lev][idim], dm, false ,lev); - RemakeMultiFab(m_face_areas[lev][idim], dm, false ,lev); + RemakeMultiFab(m_edge_lengths[lev][idim], false); + RemakeMultiFab(m_face_areas[lev][idim], false); if(WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT){ - RemakeMultiFab(Venl[lev][idim], dm, false ,lev); - RemakeMultiFab(m_flag_info_face[lev][idim], dm, false ,lev); - RemakeMultiFab(m_flag_ext_face[lev][idim], dm, false ,lev); - RemakeMultiFab(m_area_mod[lev][idim], dm, false ,lev); - RemakeMultiFab(ECTRhofield[lev][idim], dm, false ,lev); + RemakeMultiFab(Venl[lev][idim], false); + RemakeMultiFab(m_flag_info_face[lev][idim], false); + RemakeMultiFab(m_flag_ext_face[lev][idim], false); + RemakeMultiFab(m_area_mod[lev][idim], false); + RemakeMultiFab(ECTRhofield[lev][idim], false); m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); } } #endif } - RemakeMultiFab(F_fp[lev], dm, true ,lev); - RemakeMultiFab(rho_fp[lev], dm, false ,lev); + RemakeMultiFab(F_fp[lev], true); + RemakeMultiFab(rho_fp[lev], false); // phi_fp should be redistributed since we use the solution from // the last step as the initial guess for the next solve - RemakeMultiFab(phi_fp[lev], dm, true ,lev); + RemakeMultiFab(phi_fp[lev], true); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], dm, true, lev); - RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], dm, false, lev); + RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], true); + RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], false); } #ifdef AMREX_USE_EB - RemakeMultiFab(m_distance_to_eb[lev], dm, false ,lev); + RemakeMultiFab(m_distance_to_eb[lev], false); int max_guard = guard_cells.ng_FieldSolver.max(); m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, @@ -296,8 +293,8 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi } else { for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_aux[lev][idim], dm, false ,lev); - RemakeMultiFab(Efield_aux[lev][idim], dm, false ,lev); + RemakeMultiFab(Bfield_aux[lev][idim], false); + RemakeMultiFab(Efield_aux[lev][idim], false); } } @@ -305,16 +302,16 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi if (lev > 0) { for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_cp[lev][idim], dm, true ,lev); - RemakeMultiFab(Efield_cp[lev][idim], dm, true ,lev); - RemakeMultiFab(current_cp[lev][idim], dm, false ,lev); + RemakeMultiFab(Bfield_cp[lev][idim], true); + RemakeMultiFab(Efield_cp[lev][idim], true); + RemakeMultiFab(current_cp[lev][idim], false); if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_cp[lev][idim], dm, true ,lev); - RemakeMultiFab(Bfield_avg_cp[lev][idim], dm, true ,lev); + RemakeMultiFab(Efield_avg_cp[lev][idim], true); + RemakeMultiFab(Bfield_avg_cp[lev][idim], true); } } - RemakeMultiFab(F_cp[lev], dm, true ,lev); - RemakeMultiFab(rho_cp[lev], dm, false ,lev); + RemakeMultiFab(F_cp[lev], true); + RemakeMultiFab(rho_cp[lev], false); #ifdef WARPX_USE_PSATD if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -354,14 +351,14 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi if (lev > 0 && (n_field_gather_buffer > 0 || n_current_deposition_buffer > 0)) { for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_cax[lev][idim], dm, false ,lev); - RemakeMultiFab(Efield_cax[lev][idim], dm, false ,lev); - RemakeMultiFab(current_buf[lev][idim], dm, false ,lev); + RemakeMultiFab(Bfield_cax[lev][idim], false); + RemakeMultiFab(Efield_cax[lev][idim], false); + RemakeMultiFab(current_buf[lev][idim], false); } - RemakeMultiFab(charge_buf[lev], dm, false ,lev); + RemakeMultiFab(charge_buf[lev], false); // we can avoid redistributing these since we immediately re-build the values via BuildBufferMasks() - RemakeMultiFab(current_buffer_masks[lev], dm, false ,lev); - RemakeMultiFab(gather_buffer_masks[lev], dm, false ,lev); + RemakeMultiFab(current_buffer_masks[lev], false); + RemakeMultiFab(gather_buffer_masks[lev], false); if (current_buffer_masks[lev] || gather_buffer_masks[lev]) { BuildBufferMasks(); From ca5978ad55f530a13afcc081b1d52c5bfaaba0b8 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 29 May 2024 16:44:14 -0700 Subject: [PATCH 05/87] Fix: `MPIInitHelpers.cpp` Macro (#4957) This failed if `AMREX_USE_OMP` is undefined. --- Source/ablastr/parallelization/MPIInitHelpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/ablastr/parallelization/MPIInitHelpers.cpp b/Source/ablastr/parallelization/MPIInitHelpers.cpp index 513035c483c..5ba15a9e64d 100644 --- a/Source/ablastr/parallelization/MPIInitHelpers.cpp +++ b/Source/ablastr/parallelization/MPIInitHelpers.cpp @@ -35,7 +35,7 @@ namespace ablastr::parallelization #ifdef AMREX_USE_MPI # ifdef AMREX_MPI_THREAD_MULTIPLE // i.e. for async_io return MPI_THREAD_MULTIPLE; -# elif AMREX_USE_OMP +# elif defined(AMREX_USE_OMP) return MPI_THREAD_FUNNELED; # else return MPI_THREAD_SINGLE; // equiv. to MPI_Init From 54ed5ccfe955c1b1e2995e64c181f2a6b3946033 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 29 May 2024 18:59:07 -0700 Subject: [PATCH 06/87] Picmi: allow different `random_fraction` for different species (#4951) * picmi: allow different `random_fraction` or `uniform_stride` for different species * fix codeQL issue * use dictionary to specify particle diagnostic stride or random fraction * use 1 as stride and random fraction default for unspecified species * use species rather than species.name as key for random fraction dict * update docstring --- Python/pywarpx/picmi.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index e7575f83b26..34c21c2a34a 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2497,11 +2497,17 @@ class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnostic warpx_file_min_digits: integer, optional Minimum number of digits for the time step number in the file name - warpx_random_fraction: float, optional - Random fraction of particles to include in the diagnostic - - warpx_uniform_stride: integer, optional - Stride to down select to the particles to include in the diagnostic + warpx_random_fraction: float or dict, optional + Random fraction of particles to include in the diagnostic. If a float + is given the same fraction will be used for all species, if a dictionary + is given the keys should be species with the value specifying the random + fraction for that species. + + warpx_uniform_stride: integer or dict, optional + Stride to down select to the particles to include in the diagnostic. + If an integer is given the same stride will be used for all species, if + a dictionary is given the keys should be species with the value + specifying the stride for that species. warpx_plot_filter_function: string, optional Analytic expression to down select the particles to in the diagnostic @@ -2599,6 +2605,22 @@ def diagnostic_initialize_inputs(self): else: species_names = [self.species.name] + # check if random fraction is specified and whether a value is given per species + random_fraction = {} + random_fraction_default = self.random_fraction + if isinstance(self.random_fraction, dict): + random_fraction_default = 1.0 + for key, val in self.random_fraction.items(): + random_fraction[key.name] = val + + # check if uniform stride is specified and whether a value is given per species + uniform_stride = {} + uniform_stride_default = self.uniform_stride + if isinstance(self.uniform_stride, dict): + uniform_stride_default = 1 + for key, val in self.uniform_stride.items(): + uniform_stride[key.name] = val + if self.mangle_dict is None: # Only do this once so that the same variables are used in this distribution # is used multiple times @@ -2607,8 +2629,8 @@ def diagnostic_initialize_inputs(self): for name in species_names: diag = pywarpx.Bucket.Bucket(self.name + '.' + name, variables = variables, - random_fraction = self.random_fraction, - uniform_stride = self.uniform_stride) + random_fraction = random_fraction.get(name, random_fraction_default), + uniform_stride = uniform_stride.get(name, uniform_stride_default)) expression = pywarpx.my_constants.mangle_expression(self.plot_filter_function, self.mangle_dict) diag.__setattr__('plot_filter_function(t,x,y,z,ux,uy,uz)', expression) self.diagnostic._species_dict[name] = diag From a40d0063f275c2bb801cdf24ee9afebf132b8da0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 29 May 2024 20:29:42 -0700 Subject: [PATCH 07/87] AMReX/pyAMReX/PICSAR: Weekly Update (#4956) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index de1c8237021..c58a35eace8 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - + cd ../amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 855dcff58fd..12b0a0fdfcb 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 +branch = 28b010126a1b39297d8a496ba81f171d8563953b [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index ac321b8b694..58d97e4547c 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 +branch = 28b010126a1b39297d8a496ba81f171d8563953b [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 349a4fd3f63..f80ec0d5af7 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "7ca419ebb90da60fefc01d8c1816846fff8638a5" +set(WarpX_amrex_branch "28b010126a1b39297d8a496ba81f171d8563953b" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 79feddf3184..cdef8f277f6 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "1b795c2c9741c8a63a362ceef8bd5e70cb242e92" +set(WarpX_pyamrex_branch "d4d409bd21bc4c48487883ac2331efdb1a6b3d61" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 099a59702a2..e9dbb0f2533 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - +cd amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From b9549372d8f85071bcbffff906c38477e5909c85 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 30 May 2024 00:41:30 -0700 Subject: [PATCH 08/87] CMake: `WarpX_PSATD` -> `WarpX_FFT` (#4912) * CMake: `WarpX_PSATD` -> `WarpX_FFT` Rename `WarpX_PSATD` to `WarpX_FFT` in the build configuration options. We now use FFTs for more than the PSATD solver since the introduction of the IGF solver for electrostatic simulations. * Change Define: `WARPX_USE_PSATD` * GNUMake: `USE_PSATD` -> `USE_FFT` * Temporary backwards compatibility We keep the flag `WarpX_PSATD` as an alias around for a few releases before we drop it. --- .github/workflows/clang_tidy.yml | 2 +- .github/workflows/cuda.yml | 6 +- .github/workflows/hip.yml | 4 +- .github/workflows/ubuntu.yml | 8 +- CMakeLists.txt | 28 ++- Docs/Doxyfile | 2 +- Docs/source/developers/gnumake.rst | 2 +- Docs/source/developers/gnumake/rzgeometry.rst | 6 +- Docs/source/developers/gnumake/spectral.rst | 4 +- Docs/source/install/cmake.rst | 8 +- Docs/source/install/hpc/adastra.rst | 4 +- Docs/source/install/hpc/crusher.rst | 4 +- Docs/source/install/hpc/frontier.rst | 4 +- Docs/source/install/hpc/greatlakes.rst | 4 +- Docs/source/install/hpc/hpc3.rst | 4 +- Docs/source/install/hpc/juwels.rst | 2 +- Docs/source/install/hpc/karolina.rst | 4 +- Docs/source/install/hpc/lassen.rst | 4 +- Docs/source/install/hpc/lawrencium.rst | 6 +- Docs/source/install/hpc/leonardo.rst | 4 +- Docs/source/install/hpc/lumi.rst | 4 +- Docs/source/install/hpc/perlmutter.rst | 8 +- Docs/source/install/hpc/polaris.rst | 4 +- Docs/source/install/hpc/quartz.rst | 4 +- Docs/source/install/hpc/spock.rst | 2 +- Docs/source/install/hpc/summit.rst | 4 +- Docs/source/usage/parameters.rst | 4 +- GNUmakefile | 3 +- Regression/WarpX-GPU-tests.ini | 10 +- Regression/WarpX-tests.ini | 220 +++++++++--------- Regression/prepare_file_ci.py | 4 +- Source/BoundaryConditions/PML.H | 8 +- Source/BoundaryConditions/PML.cpp | 8 +- Source/BoundaryConditions/PML_RZ.H | 6 +- Source/BoundaryConditions/PML_RZ.cpp | 4 +- Source/BoundaryConditions/WarpXEvolvePML.cpp | 4 +- .../ComputeDiagFunctors/RhoFunctor.cpp | 4 +- .../FlushFormats/FlushFormatCheckpoint.cpp | 4 +- Source/Diagnostics/WarpXIO.cpp | 4 +- Source/Evolve/WarpXEvolve.cpp | 6 +- Source/Evolve/WarpXOneStepImplicitPicard.cpp | 2 +- Source/FieldSolver/CMakeLists.txt | 2 +- Source/FieldSolver/Make.package | 2 +- .../PsatdAlgorithmComoving.H | 4 +- .../PsatdAlgorithmComoving.cpp | 4 +- .../PsatdAlgorithmFirstOrder.H | 4 +- .../PsatdAlgorithmFirstOrder.cpp | 4 +- .../PsatdAlgorithmJConstantInTime.H | 4 +- .../PsatdAlgorithmJConstantInTime.cpp | 4 +- .../PsatdAlgorithmJLinearInTime.H | 4 +- .../PsatdAlgorithmJLinearInTime.cpp | 4 +- .../SpectralAlgorithms/PsatdAlgorithmPml.H | 4 +- .../SpectralAlgorithms/PsatdAlgorithmPml.cpp | 4 +- .../SpectralBaseAlgorithm.H | 4 +- .../SpectralSolver/SpectralFieldData.cpp | 4 +- .../SpectralSolver/SpectralSolver.H | 4 +- .../SpectralSolver/SpectralSolver.cpp | 4 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 8 +- Source/Initialization/WarpXInitData.cpp | 12 +- Source/Make.WarpX | 4 +- Source/Parallelization/WarpXComm.cpp | 8 +- Source/Parallelization/WarpXRegrid.cpp | 4 +- Source/Python/WarpX.cpp | 2 +- Source/Utils/WarpXMovingWindow.cpp | 6 +- Source/Utils/WarpX_Complex.H | 4 +- Source/WarpX.H | 14 +- Source/WarpX.cpp | 24 +- Source/ablastr/fields/CMakeLists.txt | 2 +- Source/ablastr/fields/Make.package | 2 +- Source/ablastr/fields/PoissonSolver.H | 4 +- Source/ablastr/math/fft/CMakeLists.txt | 2 +- Source/ablastr/math/fft/Make.package | 2 +- Tools/Linter/runClangTidy.sh | 2 +- WarpXConfig.cmake | 13 +- cmake/WarpXFunctions.cmake | 4 +- setup.py | 4 +- 76 files changed, 311 insertions(+), 297 deletions(-) diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index 5abb42352f4..28b46b52530 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -43,7 +43,7 @@ jobs: -DWarpX_DIMS="${{ matrix.dim }}" \ -DWarpX_MPI=ON \ -DWarpX_COMPUTE=OMP \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index c58a35eace8..07422c36e15 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -70,7 +70,7 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build_sp -j 4 @@ -116,7 +116,7 @@ jobs: git clone https://github.com/AMReX-Codes/amrex.git ../amrex cd ../amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd - - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 + make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s du -hs ~/.cache/ccache @@ -168,7 +168,7 @@ jobs: -DWarpX_PYTHON=OFF \ -DAMReX_CUDA_ARCH=8.0 \ -DWarpX_OPENPMD=ON \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build -j 4 diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 0e311f061ef..0f6710b1405 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -55,7 +55,7 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_PSATD=ON + -DWarpX_FFT=ON cmake --build build_sp -j 4 export WARPX_MPI=OFF @@ -115,7 +115,7 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ - -DWarpX_PSATD=ON + -DWarpX_FFT=ON cmake --build build_2d -j 4 export WARPX_MPI=OFF diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 2997e9cdd16..9f4953b7a98 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -77,7 +77,7 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="1;2" \ -DWarpX_EB=OFF \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_QED_TOOLS=ON @@ -127,7 +127,7 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;3" \ -DWarpX_EB=OFF \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_PRECISION=SINGLE \ -DWarpX_PARTICLE_PRECISION=SINGLE \ -DWarpX_QED_TABLE_GEN=ON @@ -211,7 +211,7 @@ jobs: cmake -S . -B build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_APP=OFF \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_PYTHON=ON \ -DWarpX_QED_TABLE_GEN=ON cmake --build build -j 4 --target pip_install @@ -260,7 +260,7 @@ jobs: -GNinja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;1;2;3" \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 31b410fc55e..85b4a42b040 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,12 @@ endif() # CMake policies ############################################################## # +# Setting a cmake_policy to OLD is deprecated by definition and will raise a +# verbose warning +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "" FORCE) +endif() + # AMReX 21.06+ supports CUDA_ARCHITECTURES with CMake 3.20+ # CMake 3.18+: CMAKE_CUDA_ARCHITECTURES # https://cmake.org/cmake/help/latest/policy/CMP0104.html @@ -72,7 +78,7 @@ option(WarpX_EB "Embedded boundary support" OFF) option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) -option(WarpX_PSATD "spectral solver support" OFF) +option(WarpX_FFT "FFT-based solvers" OFF) option(WarpX_PYTHON "Python bindings" OFF) option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF) option(WarpX_QED "QED support (requires PICSAR)" ON) @@ -149,6 +155,14 @@ if(WarpX_APP OR WarpX_PYTHON) set(WarpX_LIB ON CACHE STRING "Build WarpX as a library" FORCE) endif() +# deprecated options: transition phase + +if(DEFINED WarpX_PSATD) + message(WARNING "CMake option WarpX_PSATD is deprecated. Use WarpX_FFT instead.\n" + "Overwriting WarpX_FFT with '${WarpX_PSATD}' because WarpX_PSATD was set.") + set(WarpX_FFT ${WarpX_PSATD} CACHE STRING "FFT-based solvers" FORCE) +endif() + # note: we could skip this if we solely build WarpX_APP, but if we build a # shared WarpX library or a third party, like ImpactX, uses ablastr in a # shared library (e.g., for Python bindings), then we need relocatable code. @@ -156,9 +170,9 @@ option(ABLASTR_POSITION_INDEPENDENT_CODE "Build ABLASTR with position independent code" ON) mark_as_advanced(ABLASTR_POSITION_INDEPENDENT_CODE) -option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_PSATD}) -if(WarpX_PSATD) - set(ABLASTR_FFT ON CACHE STRING "compile AnyFFT wrappers" FORCE) +option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT}) +if(WarpX_FFT) + set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE) endif() # this defined the variable BUILD_TESTING which is ON by default @@ -192,7 +206,7 @@ include(${WarpX_SOURCE_DIR}/cmake/dependencies/openPMD.cmake) # PSATD include(${WarpX_SOURCE_DIR}/cmake/dependencies/FFT.cmake) -if(WarpX_PSATD) +if(WarpX_FFT) # BLASPP and LAPACKPP if(RZ IN_LIST WarpX_DIMS) find_package(blaspp CONFIG REQUIRED) @@ -517,8 +531,8 @@ foreach(D IN LISTS WarpX_DIMS) endif() endif() - if(WarpX_PSATD) - target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_PSATD) + if(WarpX_FFT) + target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_FFT) endif() if(ABLASTR_FFT) # We need to enable FFT support in ABLASTR for PSATD solver diff --git a/Docs/Doxyfile b/Docs/Doxyfile index 550ed9a7a01..5fbb7651b18 100644 --- a/Docs/Doxyfile +++ b/Docs/Doxyfile @@ -2301,7 +2301,7 @@ PREDEFINED = AMREX_Linux=1 \ WARPX_DIM_XZ=1 \ WARPX_USE_GPU=1 \ WARPX_USE_OPENPMD=1 \ - WARPX_USE_PSATD=1 \ + WARPX_USE_FFT=1 \ WARPX_QED=1 \ WARPX_QED_TABLE_GEN=1 diff --git a/Docs/source/developers/gnumake.rst b/Docs/source/developers/gnumake.rst index ca51dacfbd5..6324ceb4ac7 100644 --- a/Docs/source/developers/gnumake.rst +++ b/Docs/source/developers/gnumake.rst @@ -56,7 +56,7 @@ options are: * ``DIM=3`` or ``2``: Geometry of the simulation (note that running an executable compiled for 3D with a 2D input file will crash). * ``DEBUG=FALSE`` or ``TRUE``: Compiling in ``DEBUG`` mode can help tremendously during code development. - * ``USE_PSATD=FALSE`` or ``TRUE``: Compile the Pseudo-Spectral Analytical Time Domain Maxwell solver. Requires an FFT library. + * ``USE_FFT=FALSE`` or ``TRUE``: Compile the Pseudo-Spectral Analytical Time Domain Maxwell solver. Requires an FFT library. * ``USE_RZ=FALSE`` or ``TRUE``: Compile for 2D axisymmetric geometry. * ``COMP=gcc`` or ``intel``: Compiler. * ``USE_MPI=TRUE`` or ``FALSE``: Whether to compile with MPI support. diff --git a/Docs/source/developers/gnumake/rzgeometry.rst b/Docs/source/developers/gnumake/rzgeometry.rst index ac90fb754f2..a5c0993a805 100644 --- a/Docs/source/developers/gnumake/rzgeometry.rst +++ b/Docs/source/developers/gnumake/rzgeometry.rst @@ -17,7 +17,7 @@ RZ geometry with spectral solver Additional steps are needed to build the spectral solver. Some of the steps are the same as is done for the Cartesian spectral solver, setting up the FFTW -package and setting ``USE_PSATD=TRUE``. +package and setting ``USE_FFT=TRUE``. - Install (or load) an MPI-enabled version of FFTW. For instance, for Debian, this can be done with @@ -54,7 +54,7 @@ package and setting ``USE_PSATD=TRUE``. export BLAS_LIB=-lblas - - Set ``USE_PSATD=TRUE`` when compiling: + - Set ``USE_FFT=TRUE`` when compiling: :: - make -j 4 USE_RZ=TRUE USE_PSATD=TRUE + make -j 4 USE_RZ=TRUE USE_FFT=TRUE diff --git a/Docs/source/developers/gnumake/spectral.rst b/Docs/source/developers/gnumake/spectral.rst index a5a0e6448f5..bb0baf01c09 100644 --- a/Docs/source/developers/gnumake/spectral.rst +++ b/Docs/source/developers/gnumake/spectral.rst @@ -16,10 +16,10 @@ In order to run the code with a spectral solver, you need to: export FFTW_HOME=/usr/ - - Set ``USE_PSATD=TRUE`` when compiling: + - Set ``USE_FFT=TRUE`` when compiling: :: - make -j 4 USE_PSATD=TRUE + make -j 4 USE_FFT=TRUE See :doc:`rzgeometry` for using the spectral solver with USE_RZ. Additional steps are needed. PSATD is compatible with single precision, but please note that, on CPU, FFTW needs to be compiled with option ``--enable-float``. diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 006cd843b31..b9fd0b2be45 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -96,7 +96,7 @@ CMake Option Default & Values Descr ``WarpX_OPENPMD`` **ON**/OFF openPMD I/O (HDF5, ADIOS) ``WarpX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WarpX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set -``WarpX_PSATD`` ON/**OFF** Spectral solver +``WarpX_FFT`` ON/**OFF** FFT-based solvers ``WarpX_PYTHON`` ON/**OFF** Python bindings ``WarpX_QED`` **ON**/OFF QED support (requires PICSAR) ``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost) @@ -270,7 +270,7 @@ Environment Variable Default & Values Descr ``WARPX_OPENPMD`` **ON**/OFF openPMD I/O (HDF5, ADIOS) ``WARPX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WARPX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set -``WARPX_PSATD`` ON/**OFF** Spectral solver +``WARPX_FFT`` ON/**OFF** FFT-based solvers ``WARPX_QED`` **ON**/OFF PICSAR QED (requires PICSAR) ``WARPX_QED_TABLE_GEN`` ON/**OFF** QED table generation (requires PICSAR and Boost) ``BUILD_PARALLEL`` ``2`` Number of threads to use for parallel builds @@ -310,11 +310,11 @@ Some Developers like to code directly against a local copy of AMReX, changing bo WARPX_AMREX_SRC=$PWD/../amrex python3 -m pip install --force-reinstall --no-deps -v . Additional environment control as common for CMake (:ref:`see above `) can be set as well, e.g. ``CC``, `CXX``, and ``CMAKE_PREFIX_PATH`` hints. -So another sophisticated example might be: use Clang as the compiler, build with local source copies of PICSAR and AMReX, support the PSATD solver, MPI and openPMD, hint a parallel HDF5 installation in ``$HOME/sw/hdf5-parallel-1.10.4``, and only build 2D and 3D geometry: +So another sophisticated example might be: use Clang as the compiler, build with local source copies of PICSAR and AMReX, support the FFT-based solvers, MPI and openPMD, hint a parallel HDF5 installation in ``$HOME/sw/hdf5-parallel-1.10.4``, and only build 2D and 3D geometry: .. code-block:: bash - CC=$(which clang) CXX=$(which clang++) WARPX_AMREX_SRC=$PWD/../amrex WARPX_PICSAR_SRC=$PWD/../picsar WARPX_PSATD=ON WARPX_MPI=ON WARPX_DIMS="2;3" CMAKE_PREFIX_PATH=$HOME/sw/hdf5-parallel-1.10.4:$CMAKE_PREFIX_PATH python3 -m pip install --force-reinstall --no-deps -v . + CC=$(which clang) CXX=$(which clang++) WARPX_AMREX_SRC=$PWD/../amrex WARPX_PICSAR_SRC=$PWD/../picsar WARPX_FFT=ON WARPX_MPI=ON WARPX_DIMS="2;3" CMAKE_PREFIX_PATH=$HOME/sw/hdf5-parallel-1.10.4:$CMAKE_PREFIX_PATH python3 -m pip install --force-reinstall --no-deps -v . Here we wrote this all in one line, but one can also set all environment variables in a development environment and keep the pip call nice and short as in the beginning. Note that you need to use absolute paths for external source trees, because pip builds in a temporary directory, e.g. ``export WARPX_AMREX_SRC=$HOME/src/amrex``. diff --git a/Docs/source/install/hpc/adastra.rst b/Docs/source/install/hpc/adastra.rst index 0b984d5e2be..e47cb5f535e 100644 --- a/Docs/source/install/hpc/adastra.rst +++ b/Docs/source/install/hpc/adastra.rst @@ -100,7 +100,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $SHAREDHOMEDIR/src/warpx rm -rf build_adastra - cmake -S . -B build_adastra -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_adastra -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_adastra -j 16 The WarpX application executables are now in ``$SHAREDHOMEDIR/src/warpx/build_adastra/bin/``. @@ -110,7 +110,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_adastra_py - cmake -S . -B build_adastra_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_adastra_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_adastra_py -j 16 --target pip_install Now, you can :ref:`submit Adstra compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/crusher.rst b/Docs/source/install/hpc/crusher.rst index 1aa471ce68b..7a657e8a51c 100644 --- a/Docs/source/install/hpc/crusher.rst +++ b/Docs/source/install/hpc/crusher.rst @@ -99,7 +99,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_crusher - cmake -S . -B build_crusher -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_crusher -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_crusher -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_crusher/bin/``. @@ -109,7 +109,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_crusher_py - cmake -S . -B build_crusher_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_crusher_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_crusher_py -j 16 --target pip_install Now, you can :ref:`submit Crusher compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/frontier.rst b/Docs/source/install/hpc/frontier.rst index e9f8b687d6b..c33c2fc7401 100644 --- a/Docs/source/install/hpc/frontier.rst +++ b/Docs/source/install/hpc/frontier.rst @@ -99,7 +99,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_frontier - cmake -S . -B build_frontier -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_frontier -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_frontier -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_frontier/bin/``. @@ -109,7 +109,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_frontier_py - cmake -S . -B build_frontier_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_frontier_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_frontier_py -j 16 --target pip_install Now, you can :ref:`submit Frontier compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/greatlakes.rst b/Docs/source/install/hpc/greatlakes.rst index d238d498e5c..9691134f571 100644 --- a/Docs/source/install/hpc/greatlakes.rst +++ b/Docs/source/install/hpc/greatlakes.rst @@ -116,7 +116,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_v100 - cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_v100 -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build_v100/bin/``. @@ -127,7 +127,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_v100_py - cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_v100_py -j 8 --target pip_install diff --git a/Docs/source/install/hpc/hpc3.rst b/Docs/source/install/hpc/hpc3.rst index e2f900de273..5d398b1f4f5 100644 --- a/Docs/source/install/hpc/hpc3.rst +++ b/Docs/source/install/hpc/hpc3.rst @@ -95,7 +95,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build/bin/``. @@ -105,7 +105,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_py - cmake -S . -B build_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_py -j 8 --target pip_install Now, you can :ref:`submit HPC3 compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/juwels.rst b/Docs/source/install/hpc/juwels.rst index a8d820ccd46..5358e649825 100644 --- a/Docs/source/install/hpc/juwels.rst +++ b/Docs/source/install/hpc/juwels.rst @@ -57,7 +57,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_MPI_THREAD_MULTIPLE=OFF + cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_MPI_THREAD_MULTIPLE=OFF cmake --build build -j 16 The other :ref:`general compile-time options ` apply as usual. diff --git a/Docs/source/install/hpc/karolina.rst b/Docs/source/install/hpc/karolina.rst index fffb917df1c..127b81477da 100644 --- a/Docs/source/install/hpc/karolina.rst +++ b/Docs/source/install/hpc/karolina.rst @@ -80,7 +80,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $WORK/src/warpx rm -rf build_gpu - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu -j 48 The WarpX application executables are now in ``$WORK/src/warpx/build_gpu/bin/``. @@ -91,7 +91,7 @@ Additionally, the following commands will install WarpX as a Python module: cd $WORK/src/warpx rm -rf build_gpu_py - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu_py -j 48 --target pip_install Now, you can :ref:`submit Karolina compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index d8565f06983..0d885dff04d 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -200,7 +200,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd /usr/workspace/${USER}/lassen/src/warpx rm -rf build_lassen - cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lassen -j 8 The WarpX application executables are now in ``/usr/workspace/${USER}/lassen/src/warpx/build_lassen/bin/``. @@ -210,7 +210,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_lassen_py - cmake -S . -B build_lassen_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lassen_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lassen_py -j 8 --target pip_install Now, you can :ref:`submit lassen compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/lawrencium.rst b/Docs/source/install/hpc/lawrencium.rst index 23be9b75431..2217c5a31ce 100644 --- a/Docs/source/install/hpc/lawrencium.rst +++ b/Docs/source/install/hpc/lawrencium.rst @@ -102,7 +102,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_DIMS="1;2;RZ;3" -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON + cmake -S . -B build -DWarpX_DIMS="1;2;RZ;3" -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON cmake --build build -j 12 The general :ref:`cmake compile-time options ` apply as usual. @@ -122,14 +122,14 @@ For a *full PICMI install*, follow the :ref:`instructions for Python (PICMI) bin python3 -m pip install -r requirements.txt # compile parallel PICMI interfaces in 3D, 2D, 1D and RZ - WARPX_MPI=ON WARPX_COMPUTE=CUDA WARPX_PSATD=ON BUILD_PARALLEL=12 python3 -m pip install --force-reinstall --no-deps -v . + WARPX_MPI=ON WARPX_COMPUTE=CUDA WARPX_FFT=ON BUILD_PARALLEL=12 python3 -m pip install --force-reinstall --no-deps -v . Or, if you are *developing*, do a quick PICMI install of a *single geometry* (see: :ref:`WarpX_DIMS `) using: .. code-block:: bash # find dependencies & configure - cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS=RZ + cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS=RZ # build and then call "python3 -m pip install ..." cmake --build build --target pip_install -j 12 diff --git a/Docs/source/install/hpc/leonardo.rst b/Docs/source/install/hpc/leonardo.rst index 1f3c1e8fbfa..568a5612250 100644 --- a/Docs/source/install/hpc/leonardo.rst +++ b/Docs/source/install/hpc/leonardo.rst @@ -89,7 +89,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_gpu - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``. @@ -100,7 +100,7 @@ Additionally, the following commands will install WarpX as a Python module: cd $HOME/src/warpx rm -rf build_gpu_py - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu_py -j 16 --target pip_install Now, you can :ref:`submit Leonardo compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/lumi.rst b/Docs/source/install/hpc/lumi.rst index 2bae4d21599..2d155dab222 100644 --- a/Docs/source/install/hpc/lumi.rst +++ b/Docs/source/install/hpc/lumi.rst @@ -98,7 +98,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_lumi - cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_lumi/bin/``. @@ -108,7 +108,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_lumi_py - cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi_py -j 16 --target pip_install diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst index 14972566abc..9612b64476d 100644 --- a/Docs/source/install/hpc/perlmutter.rst +++ b/Docs/source/install/hpc/perlmutter.rst @@ -153,7 +153,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu - cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. @@ -164,7 +164,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu_py - cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu_py -j 16 --target pip_install .. tab-item:: CPU Nodes @@ -174,7 +174,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_cpu - cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_cpu/bin/``. @@ -184,7 +184,7 @@ Use the following :ref:`cmake commands ` to compile the applicat rm -rf build_pm_cpu_py - cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu_py -j 16 --target pip_install Now, you can :ref:`submit Perlmutter compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/polaris.rst b/Docs/source/install/hpc/polaris.rst index d20ecccee32..a2d95a144d6 100644 --- a/Docs/source/install/hpc/polaris.rst +++ b/Docs/source/install/hpc/polaris.rst @@ -101,7 +101,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu - cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. @@ -112,7 +112,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu_py - cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu_py -j 16 --target pip_install .. tab-item:: CPU Nodes diff --git a/Docs/source/install/hpc/quartz.rst b/Docs/source/install/hpc/quartz.rst index 03860687971..a49327e8613 100644 --- a/Docs/source/install/hpc/quartz.rst +++ b/Docs/source/install/hpc/quartz.rst @@ -93,7 +93,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_quartz - cmake -S . -B build_quartz -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_quartz -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_quartz -j 6 The WarpX application executables are now in ``$HOME/src/warpx/build_quartz/bin/``. @@ -103,7 +103,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_quartz_py - cmake -S . -B build_quartz_py -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_quartz_py -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_quartz_py -j 6 --target pip_install Now, you can :ref:`submit Quartz compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/spock.rst b/Docs/source/install/hpc/spock.rst index 64f307f9dae..144257289f1 100644 --- a/Docs/source/install/hpc/spock.rst +++ b/Docs/source/install/hpc/spock.rst @@ -51,7 +51,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DAMReX_AMD_ARCH=gfx908 -DMPI_CXX_COMPILER=$(which CC) -DMPI_C_COMPILER=$(which cc) -DMPI_COMPILER_FLAGS="--cray-print-opts=all" + cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DAMReX_AMD_ARCH=gfx908 -DMPI_CXX_COMPILER=$(which CC) -DMPI_C_COMPILER=$(which cc) -DMPI_COMPILER_FLAGS="--cray-print-opts=all" cmake --build build -j 10 The general :ref:`cmake compile-time options ` apply as usual. diff --git a/Docs/source/install/hpc/summit.rst b/Docs/source/install/hpc/summit.rst index 918da842165..bf226df69f2 100644 --- a/Docs/source/install/hpc/summit.rst +++ b/Docs/source/install/hpc/summit.rst @@ -124,7 +124,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_summit - cmake -S . -B build_summit -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_summit -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_summit -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build_summit/bin/``. @@ -134,7 +134,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_summit_py - cmake -S . -B build_summit_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_summit_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_summit_py -j 8 --target pip_install Now, you can :ref:`submit Summit compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 4f011025213..f3a74f969a0 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -173,7 +173,7 @@ Overall simulation parameters * ``fft``: Poisson's equation is solved using an Integrated Green Function method (which requires FFT calculations). See these references for more details :cite:t:`QiangPhysRevSTAB2006`, :cite:t:`QiangPhysRevSTAB2006err`. - It only works in 3D and it requires the compilation flag ``-DWarpX_PSATD=ON``. + It only works in 3D and it requires the compilation flag ``-DWarpX_FFT=ON``. If mesh refinement is enabled, this solver only works on the coarsest level. On the refined patches, the Poisson equation is solved with the multigrid solver. In electrostatic mode, this solver requires open field boundary conditions (``boundary.field_lo,hi = open``). @@ -2256,7 +2256,7 @@ Maxwell solver: PSATD method * ``psatd.v_comoving`` (3 floating-point values, in units of the speed of light; default ``0. 0. 0.``) Defines the comoving velocity in the comoving PSATD scheme. - A non-zero comoving velocity selects the comoving PSATD algorithm, which suppresses the numerical Cherenkov instability (NCI) in boosted-frame simulations, under certain assumptions. This option requires that WarpX is compiled with ``USE_PSATD = TRUE``. It also requires the use of direct current deposition (``algo.current_deposition = direct``) and has not been neither implemented nor tested with other current deposition schemes. + A non-zero comoving velocity selects the comoving PSATD algorithm, which suppresses the numerical Cherenkov instability (NCI) in boosted-frame simulations, under certain assumptions. This option requires that WarpX is compiled with ``USE_FFT = TRUE``. It also requires the use of direct current deposition (``algo.current_deposition = direct``) and has not been neither implemented nor tested with other current deposition schemes. * ``psatd.do_time_averaging`` (`0` or `1`; default: 0) Whether to use an averaged Galilean PSATD algorithm or standard Galilean PSATD. diff --git a/GNUmakefile b/GNUmakefile index 863d37dd56f..86bdab2709f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -37,8 +37,7 @@ USE_OPENPMD = FALSE WarpxBinDir = Bin -USE_PSATD = FALSE -USE_PSATD_PICSAR = FALSE +USE_FFT = FALSE USE_RZ = FALSE USE_EB = FALSE diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 12b0a0fdfcb..8542e5f35d9 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -107,7 +107,7 @@ analysisRoutine = Examples/Tests/pml/analysis_pml_ckc.py #inputFile = Examples/Tests/pml/inputs_2d #runtime_params = algo.maxwell_solver=psatd warpx.do_dynamic_scheduling=0 #dim = 2 -#addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +#addToCompileString = USE_FFT=TRUE USE_GPU=TRUE #restartTest = 0 #useMPI = 1 #numprocs = 2 @@ -377,7 +377,7 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d_multi_rt runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +addToCompileString = USE_FFT=TRUE USE_GPU=TRUE restartTest = 0 useMPI = 1 numprocs = 2 @@ -395,7 +395,7 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d_multi_rt runtime_params = algo.maxwell_solver=psatd warpx.do_dynamic_scheduling=0 warpx.grid_type=collocated algo.current_deposition=direct warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +addToCompileString = USE_FFT=TRUE USE_GPU=TRUE restartTest = 0 useMPI = 1 numprocs = 2 @@ -431,7 +431,7 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d_multi_rt runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +addToCompileString = USE_FFT=TRUE USE_GPU=TRUE restartTest = 0 useMPI = 1 numprocs = 2 @@ -449,7 +449,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png # inputFile = Examples/Tests/langmuir/inputs_2d_multi_rt # runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell # dim = 2 -# addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +# addToCompileString = USE_FFT=TRUE USE_GPU=TRUE # restartTest = 0 # useMPI = 1 # numprocs = 4 diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 58d97e4547c..6ee7ad3929d 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -74,8 +74,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_2d runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -92,8 +92,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_2d runtime_params = amr.max_grid_size_x=128 amr.max_grid_size_y=64 warpx.grid_type=hybrid psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -110,8 +110,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_3d runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -128,8 +128,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_3d runtime_params = warpx.grid_type=hybrid psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -247,8 +247,8 @@ buildDir = . inputFile = Examples/Tests/collision/inputs_rz runtime_params = dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=FALSE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=OFF +addToCompileString = USE_RZ=TRUE USE_FFT=FALSE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=OFF restartTest = 0 useMPI = 1 numprocs = 1 @@ -301,8 +301,8 @@ buildDir = . inputFile = Examples/Tests/comoving/inputs_2d_hybrid runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -798,8 +798,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d runtime_params = warpx.grid_type=collocated algo.current_deposition=direct psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -816,8 +816,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d runtime_params = psatd.periodic_single_box_fft=0 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE amr.max_grid_size=64 amr.blocking_factor=64 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -834,8 +834,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d runtime_params = psatd.periodic_single_box_fft=1 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -852,8 +852,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d_hybrid runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -870,8 +870,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -888,8 +888,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 warpx.numprocs=1 1 2 psatd.periodic_single_box_fft=0 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -906,8 +906,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 warpx.numprocs=1 1 1 psatd.periodic_single_box_fft=1 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -924,8 +924,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_rz runtime_params = electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -942,8 +942,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_rz runtime_params = psatd.periodic_single_box_fft=1 psatd.current_correction=1 electrons.random_theta=0 ions.random_theta=0 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -960,8 +960,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_rz runtime_params = psatd.periodic_single_box_fft=0 psatd.current_correction=1 electrons.random_theta=0 ions.random_theta=0 amr.max_grid_size=32 amr.blocking_factor=32 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1284,8 +1284,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver = psatd warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1322,8 +1322,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1341,8 +1341,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1360,8 +1360,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1379,8 +1379,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1398,8 +1398,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.7071067811865475 psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1417,8 +1417,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.7071067811865475 psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium warpx.grid_type=collocated dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1436,8 +1436,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1455,8 +1455,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1474,8 +1474,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1493,8 +1493,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 warpx.grid_type=collocated algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1531,8 +1531,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1550,8 +1550,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1569,8 +1569,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1588,8 +1588,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 psatd.update_with_rho = 1 algo.current_deposition = direct warpx.do_dive_cleaning = 1 warpx.do_divb_cleaning = 1 diag1.intervals = 0, 38:40:1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE F warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1607,8 +1607,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1626,8 +1626,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.5773502691896258 algo.current_deposition=direct psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1645,8 +1645,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.5773502691896258 algo.current_deposition=direct psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium warpx.grid_type=collocated dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1664,8 +1664,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct warpx.cfl = 0.5773502691896258 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1683,8 +1683,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON -DWarpX_PRECISION=SINGLE +addToCompileString = USE_FFT=TRUE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_PRECISION=SINGLE restartTest = 0 useMPI = 1 numprocs = 2 @@ -1702,8 +1702,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.current_deposition=vay diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1721,8 +1721,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=vay diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1760,8 +1760,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1780,8 +1780,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 amr.max_grid_size=128 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot=jr jz Er Ez Bt rho divE electrons.random_theta=0 ions.random_theta=0 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1800,8 +1800,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz runtime_params = amr.max_grid_size=32 algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 warpx.n_rz_azimuthal_modes=2 electrons.random_theta=0 electrons.num_particles_per_cell_each_dim=2 4 2 ions.random_theta=0 ions.num_particles_per_cell_each_dim=2 4 2 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=4 warpx.use_filter=1 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2307,8 +2307,8 @@ buildDir = . inputFile = Examples/Tests/maxwell_hybrid_qed/inputs_2d runtime_params = warpx.cfl=0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2341,8 +2341,8 @@ buildDir = . inputFile = Examples/Tests/multi_j/inputs_rz runtime_params = warpx.abort_on_warning_threshold=medium psatd.J_in_time=linear dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2896,8 +2896,8 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_3d runtime_params = warpx.do_similar_dm_pml=0 warpx.abort_on_warning_threshold=medium ablastr.fillboundary_always_sync=1 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2912,8 +2912,8 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_rz runtime_params = warpx.cfl=0.7 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2944,8 +2944,8 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_2d runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz rho divE warpx.cfl=0.7071067811865475 warpx.do_pml_dive_cleaning=1 warpx.do_pml_divb_cleaning=1 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium psatd.v_galilean=0. 0. 0.99 warpx.grid_type=collocated dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2960,8 +2960,8 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_2d runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE warpx.cfl = 0.7071067811865475 warpx.do_pml_dive_cleaning=0 warpx.do_pml_divb_cleaning=0 chk.file_prefix=pml_x_psatd_chk chk.file_min_digits=5 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 1 restartFileNum = 150 useMPI = 1 @@ -3865,8 +3865,8 @@ inputFile = Examples/Tests/python_wrappers/PICMI_inputs_2d.py runtime_params = customRunCmd = python3 PICMI_inputs_2d.py dim = 2 -addToCompileString = USE_PSATD=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON +addToCompileString = USE_FFT=TRUE USE_PYTHON_MAIN=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON target = pip_install restartTest = 0 useMPI = 1 @@ -4124,8 +4124,8 @@ buildDir = . inputFile = Examples/Tests/reduced_diags/inputs_loadbalancecosts runtime_params = algo.load_balance_costs_update=Timers dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -4264,8 +4264,8 @@ buildDir = . inputFile = Examples/Tests/restart/inputs runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_chk chk.file_min_digits=5 boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 1 restartFileNum = 5 useMPI = 1 @@ -4283,8 +4283,8 @@ buildDir = . inputFile = Examples/Tests/restart/inputs runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_time_avg_chk chk.file_min_digits=5 boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped psatd.do_time_averaging=1 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 1 restartFileNum = 5 useMPI = 1 @@ -4523,8 +4523,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.solution_type=first-order psatd.J_in_time=constant psatd.rho_in_time=constant warpx.do_dive_cleaning=1 warpx.do_divb_cleaning=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=1 diag1.fields_to_plot=Bx By Bz divE Ex Ey Ez F G jx jy jz rho warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -4541,8 +4541,8 @@ buildDir = . inputFile = Examples/Tests/vay_deposition/inputs_2d runtime_params = dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -4559,8 +4559,8 @@ buildDir = . inputFile = Examples/Tests/vay_deposition/inputs_3d runtime_params = dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -4794,8 +4794,8 @@ buildDir = . inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d runtime_params = warpx.abort_on_warning_threshold = high dim = 3 -addToCompileString = USE_OPENPMD=TRUE USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON -DWarpX_OPENPMD=ON +addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON restartTest = 0 useMPI = 1 numprocs = 2 diff --git a/Regression/prepare_file_ci.py b/Regression/prepare_file_ci.py index ce7a3398b95..f7b610dace9 100644 --- a/Regression/prepare_file_ci.py +++ b/Regression/prepare_file_ci.py @@ -59,8 +59,8 @@ # always build with PSATD support (runtime controlled if used) if ci_psatd: text = re.sub('addToCompileString =', - 'addToCompileString = USE_PSATD=TRUE ', text) - text = re.sub('USE_PSATD=FALSE', + 'addToCompileString = USE_FFT=TRUE ', text) + text = re.sub('USE_FFT=FALSE', '', text) # CCache diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 42a96f3628a..7ed1def8f72 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -13,7 +13,7 @@ #include "Utils/WarpXAlgorithmSelection.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralSolver.H" #endif @@ -179,7 +179,7 @@ public: return *sigba_cp; } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPSATD (int lev); #endif @@ -245,7 +245,7 @@ private: std::unique_ptr sigba_fp; std::unique_ptr sigba_cp; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT std::unique_ptr spectral_solver_fp; std::unique_ptr spectral_solver_cp; #endif @@ -289,7 +289,7 @@ private: static void CopyToPML (amrex::MultiFab& pml, amrex::MultiFab& reg, const amrex::Geometry& geom); }; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPMLPSATDSinglePatch( int lev, SpectralSolver& solver, std::array,3>& pml_E, diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 25b34818dd1..ed82cc07bb9 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -11,7 +11,7 @@ #include "BoundaryConditions/PML.H" #include "BoundaryConditions/PMLComponent.H" #include "FieldSolver/Fields.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralFieldData.H" #endif #include "Utils/TextMsg.H" @@ -743,7 +743,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, IntVect(ncell), IntVect(delta), single_domain_box, v_sigma_sb); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT amrex::ignore_unused(lev, dt, psatd_solution_type, J_in_time, rho_in_time); # if(AMREX_SPACEDIM!=3) amrex::ignore_unused(noy_fft); @@ -858,7 +858,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, cncells, cdelta, single_domain_box, v_sigma_sb); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT amrex::ignore_unused(dt); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, "PML: PSATD solver selected but not built."); @@ -1367,7 +1367,7 @@ PML::Restart (const std::string& dir) } } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PML::PushPSATD (const int lev) { diff --git a/Source/BoundaryConditions/PML_RZ.H b/Source/BoundaryConditions/PML_RZ.H index 74d3da2e19b..c908681d8e5 100644 --- a/Source/BoundaryConditions/PML_RZ.H +++ b/Source/BoundaryConditions/PML_RZ.H @@ -12,7 +12,7 @@ #include "Utils/WarpXAlgorithmSelection.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #endif @@ -40,7 +40,7 @@ public: std::array GetE_fp (); std::array GetB_fp (); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPSATD (int lev); #endif @@ -62,7 +62,7 @@ private: std::array,2> pml_E_fp; std::array,2> pml_B_fp; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPMLPSATDSinglePatchRZ ( int lev, SpectralSolverRZ& solver, std::array,2>& pml_E, diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index faa7609b3ab..78f3cf24987 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -9,7 +9,7 @@ #include "BoundaryConditions/PML_RZ.H" #include "FieldSolver/Fields.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" #endif #include "Utils/WarpXConst.H" @@ -187,7 +187,7 @@ PML_RZ::Restart (const std::string& dir) } } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PML_RZ::PushPSATD (const int lev) { diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index af721d70b6d..ec696689373 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -8,7 +8,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "PML_current.H" @@ -60,7 +60,7 @@ WarpX::DampPML (const int lev, PatchType patch_type) if (!do_pml) { return; } WARPX_PROFILE("WarpX::DampPML()"); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->ApplyDamping(Efield_fp[lev][1].get(), Efield_fp[lev][2].get(), Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get(), diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp index 340cc635fea..32e11903778 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp @@ -1,7 +1,7 @@ #include "RhoFunctor.H" #include "Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) #include "FieldSolver/SpectralSolver/SpectralFieldData.H" #include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -60,7 +60,7 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i // apply the filtering if requested. warpx.ApplyFilterandSumBoundaryRho(m_lev, m_lev, *rho, 0, rho->nComp()); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) // Apply k-space filtering when using the PSATD solver if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 27d0ad84b9c..1a3318ae0d8 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -1,7 +1,7 @@ #include "FlushFormatCheckpoint.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/ParticleDiag/ParticleDiag.H" @@ -152,7 +152,7 @@ FlushFormatCheckpoint::WriteToFile ( warpx.GetPML(lev)->CheckPoint( amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "pml")); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (warpx.GetPML_RZ(lev)) { warpx.GetPML_RZ(lev)->CheckPoint( amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "pml_rz")); diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 91409bc294d..a17abe04178 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -8,7 +8,7 @@ * License: BSD-3-Clause-LBNL */ #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "FieldIO.H" @@ -385,7 +385,7 @@ WarpX::InitFromCheckpoint () if (pml[lev]) { pml[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index de586895c2d..881967a0479 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -15,7 +15,7 @@ #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "Evolve/WarpXDtType.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" # else @@ -603,7 +603,7 @@ void WarpX::SyncCurrentAndRho () void WarpX::OneStep_multiJ (const amrex::Real cur_time) { -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, @@ -766,7 +766,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) amrex::ignore_unused(cur_time); WARPX_ABORT_WITH_MESSAGE( "multi-J algorithm not implemented for FDTD"); -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT } /* /brief Perform one PIC iteration, with subcycling diff --git a/Source/Evolve/WarpXOneStepImplicitPicard.cpp b/Source/Evolve/WarpXOneStepImplicitPicard.cpp index 22d42dfe89d..6df3cd0d4e6 100644 --- a/Source/Evolve/WarpXOneStepImplicitPicard.cpp +++ b/Source/Evolve/WarpXOneStepImplicitPicard.cpp @@ -11,7 +11,7 @@ #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "Evolve/WarpXDtType.H" #include "Evolve/WarpXPushType.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" # else diff --git a/Source/FieldSolver/CMakeLists.txt b/Source/FieldSolver/CMakeLists.txt index 58dd10f41cf..cf741aa6e01 100644 --- a/Source/FieldSolver/CMakeLists.txt +++ b/Source/FieldSolver/CMakeLists.txt @@ -11,6 +11,6 @@ endforeach() add_subdirectory(FiniteDifferenceSolver) add_subdirectory(MagnetostaticSolver) -if(WarpX_PSATD) +if(WarpX_FFT) add_subdirectory(SpectralSolver) endif() diff --git a/Source/FieldSolver/Make.package b/Source/FieldSolver/Make.package index c1535ae2f22..3b88f4627e5 100644 --- a/Source/FieldSolver/Make.package +++ b/Source/FieldSolver/Make.package @@ -2,7 +2,7 @@ CEXE_sources += WarpXPushFieldsEM.cpp CEXE_sources += WarpXPushFieldsHybridPIC.cpp CEXE_sources += ElectrostaticSolver.cpp CEXE_sources += WarpX_QED_Field_Pushers.cpp -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) include $(WARPX_HOME)/Source/FieldSolver/SpectralSolver/Make.package endif include $(WARPX_HOME)/Source/FieldSolver/FiniteDifferenceSolver/Make.package diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H index 87dfe6f2d9a..70468277749 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H @@ -14,7 +14,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* \brief Class that updates the field in spectral space and stores the coefficients * of the corresponding update equation, according to the comoving spectral scheme. @@ -86,5 +86,5 @@ class PsatdAlgorithmComoving : public SpectralBaseAlgorithm amrex::Real m_dt; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_COMOVING_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp index 640cf1b82af..6da6641c052 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp @@ -19,7 +19,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -505,4 +505,4 @@ PsatdAlgorithmComoving::VayDeposition (SpectralFieldData& /*field_data*/) "Vay deposition not implemented for comoving PSATD"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H index 8fde50d8172..23d4c2f82e3 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H @@ -20,7 +20,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* * \brief Class that updates the fields in spectral space according to the first-order PSATD equations. */ @@ -94,5 +94,5 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm int m_J_in_time; int m_rho_in_time; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_FIRST_ORDER_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp index 6d4d8613940..041b70b8edf 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp @@ -24,7 +24,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex::literals; @@ -371,4 +371,4 @@ PsatdAlgorithmFirstOrder::VayDeposition (SpectralFieldData& field_data) "Vay deposition not implemented for first-order PSATD equations"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H index 94f00d6d4eb..5d83b5519a6 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H @@ -20,7 +20,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -139,5 +139,5 @@ class PsatdAlgorithmJConstantInTime : public SpectralBaseAlgorithm bool m_divb_cleaning; bool m_is_galilean; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_J_CONSTANT_IN_TIME_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp index 167d7565bc0..59d56b99afc 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp @@ -25,7 +25,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -861,4 +861,4 @@ PsatdAlgorithmJConstantInTime::VayDeposition (SpectralFieldData& field_data) } } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H index aa3bcc08e77..970657978bb 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H @@ -20,7 +20,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* * \brief Class that updates the fields in spectral space according to the multi-J algorithm * and stores the coefficients of the corresponding update equations. J is assumed to be @@ -125,5 +125,5 @@ class PsatdAlgorithmJLinearInTime : public SpectralBaseAlgorithm bool m_dive_cleaning; bool m_divb_cleaning; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_J_LINEAR_IN_TIME_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp index 92e35c8c03b..a6ee55147c9 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp @@ -24,7 +24,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex::literals; @@ -444,4 +444,4 @@ PsatdAlgorithmJLinearInTime::VayDeposition (SpectralFieldData& field_data) "Vay deposition not implemented for multi-J PSATD algorithm"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H index 1c25ca91903..1c19827a0a7 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H @@ -19,7 +19,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* * \brief Class that updates the field in spectral space @@ -115,5 +115,5 @@ class PsatdAlgorithmPml : public SpectralBaseAlgorithm bool m_is_galilean; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_PML_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp index 792d5126ef7..ee776a71a20 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp @@ -26,7 +26,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -466,4 +466,4 @@ void PsatdAlgorithmPml::VayDeposition (SpectralFieldData& /*field_data*/) WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented for PML PSATD"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H index 4af24cfa559..e6787c9e52d 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H @@ -23,7 +23,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. @@ -101,5 +101,5 @@ class SpectralBaseAlgorithm KVectorComponent modified_kz_vec; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_SPECTRAL_BASE_ALGORITHM_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp index 35d37df6038..20c97f4b5d4 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp @@ -29,7 +29,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -475,4 +475,4 @@ SpectralFieldData::BackwardTransform (const int lev, } } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.H b/Source/FieldSolver/SpectralSolver/SpectralSolver.H index 9c74cdbea5a..3d751974a2b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.H @@ -21,7 +21,7 @@ #include #include -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT /** * \brief Top-level class for the electromagnetic spectral solver * @@ -205,5 +205,5 @@ class SpectralSolver // to an instance of a sub-class defining a specific algorithm std::unique_ptr algorithm; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_SPECTRAL_SOLVER_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index 460afee999f..9fc8f8056ca 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -19,7 +19,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT SpectralSolver::SpectralSolver( const int lev, @@ -145,4 +145,4 @@ SpectralSolver::pushSpectralFields(){ algorithm->pushSpectralFields( field_data ); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index fe7642fd172..6631c1b88ee 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -11,7 +11,7 @@ #include "BoundaryConditions/PML.H" #include "Evolve/WarpXDtType.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" -#if defined(WARPX_USE_PSATD) +#if defined(WARPX_USE_FFT) # include "FieldSolver/SpectralSolver/SpectralFieldData.H" # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" @@ -54,7 +54,7 @@ using namespace amrex; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT namespace { void ForwardTransformVect ( @@ -645,12 +645,12 @@ WarpX::PSATDScaleAverageFields (const amrex::Real scale_factor) } } } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT void WarpX::PushPSATD () { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ABORT_WITH_MESSAGE( "PushFieldsEM: PSATD solver selected but not built"); #else diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 0d5ab70c657..bd40c5e0cd4 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -11,7 +11,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/MultiDiagnostics.H" @@ -279,7 +279,7 @@ WarpX::PrintMainPICparameters () else if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC){ amrex::Print() << "Maxwell Solver: | Hybrid-PIC (Ohm's law) \n"; } - #ifdef WARPX_USE_PSATD + #ifdef WARPX_USE_FFT // Print PSATD solver's configuration if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD){ amrex::Print() << "Maxwell Solver: | PSATD \n"; @@ -326,12 +326,12 @@ WarpX::PrintMainPICparameters () if (fft_do_time_averaging == 1){ amrex::Print()<<" | - time-averaged is ON \n"; } - #endif // WARPX_USE_PSATD + #endif // WARPX_USE_FFT if (grid_type == GridType::Collocated){ amrex::Print() << " | - collocated grid \n"; } - #ifdef WARPX_USE_PSATD + #ifdef WARPX_USE_FFT if ( (grid_type == GridType::Staggered) && (field_gathering_algo == GatheringAlgo::EnergyConserving) ){ amrex::Print()<<" | - staggered grid " << "\n"; } @@ -384,7 +384,7 @@ WarpX::PrintMainPICparameters () amrex::Print() << "Guard cells | - ng_alloc_EB = " << guard_cells.ng_alloc_EB << "\n"; amrex::Print() << " (allocated for E/B) | \n"; - #endif // WARPX_USE_PSATD + #endif // WARPX_USE_FFT amrex::Print() << "-------------------------------------------------------------------------------" << "\n"; //Print main boosted frame algorithm's parameters if (WarpX::gamma_boost!=1){ @@ -590,7 +590,7 @@ WarpX::InitPML () if (finest_level > 0) { do_pml = 1; } if (do_pml) { -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) do_pml_Lo[0][0] = 0; // no PML at r=0, in cylindrical geometry pml_rz[0] = std::make_unique(0, boxArray(0), DistributionMap(0), &Geom(0), pml_ncell, do_pml_in_domain); #else diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 4662dc18abd..29f047c14e6 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -171,9 +171,9 @@ ifeq ($(USE_OPENPMD), TRUE) endif -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) USERSuffix := $(USERSuffix).PSATD - DEFINES += -DWARPX_USE_PSATD -DABLASTR_USE_FFT + DEFINES += -DWARPX_USE_FFT -DABLASTR_USE_FFT ifeq ($(USE_CUDA),TRUE) # Use cuFFT libraries += -lcufft diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index f3bff69546b..dcf5fbe5621 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -9,7 +9,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Filter/BilinearFilter.H" @@ -64,7 +64,7 @@ WarpX::UpdateAuxilaryData () void WarpX::UpdateAuxilaryDataStagToNodal () { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( false, "WarpX::UpdateAuxilaryDataStagToNodal: PSATD solver requires " @@ -584,7 +584,7 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In pml[lev]->FillBoundaryE(patch_type, nodal_sync); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->FillBoundaryE(patch_type, nodal_sync); @@ -641,7 +641,7 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In pml[lev]->FillBoundaryB(patch_type, nodal_sync); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->FillBoundaryB(patch_type, nodal_sync); diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 2846e79c361..52586a5c88c 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -249,7 +249,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi m_field_factory[lev] = std::make_unique(); #endif -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { if (spectral_solver_fp[lev] != nullptr) { // Get the cell-centered box @@ -313,7 +313,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(F_cp[lev], true); RemakeMultiFab(rho_cp[lev], false); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { if (spectral_solver_cp[lev] != nullptr) { BoxArray cba = ba; diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 4b0ad56a0ce..b3bd5414303 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -14,7 +14,7 @@ #include #include #include -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include # ifdef WARPX_DIM_RZ # include diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index eb7eceaa2fd..5fcaccf761a 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -9,7 +9,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Initialization/ExternalField.H" @@ -252,7 +252,7 @@ WarpX::MoveWindow (const int step, bool move_j) shiftMF(*pml_B[dim], geom[lev], num_shift, dir, lev, dont_update_cost); shiftMF(*pml_E[dim], geom[lev], num_shift, dir, lev, dont_update_cost); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev] && dim < 2) { const std::array& pml_rz_B = pml_rz[lev]->GetB_fp(); const std::array& pml_rz_E = pml_rz[lev]->GetE_fp(); @@ -598,7 +598,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, } } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (WarpX::GetInstance().getPMLRZ()) { // This does the exchange of data in the corner guard cells, the cells that are in the // guard region both radially and longitudinally. These are the PML cells in the overlapping diff --git a/Source/Utils/WarpX_Complex.H b/Source/Utils/WarpX_Complex.H index 7a1f1669286..54209b1d5d8 100644 --- a/Source/Utils/WarpX_Complex.H +++ b/Source/Utils/WarpX_Complex.H @@ -8,7 +8,7 @@ #ifndef WARPX_COMPLEX_H_ #define WARPX_COMPLEX_H_ -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include #endif @@ -21,7 +21,7 @@ // Defines a complex type on GPU & CPU using Complex = amrex::GpuComplex; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT static_assert(sizeof(Complex) == sizeof(ablastr::math::anyfft::Complex), "The complex type in WarpX and the FFT library do not match."); #endif diff --git a/Source/WarpX.H b/Source/WarpX.H index 6c6538325e6..df06a1723ca 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -27,7 +27,7 @@ #include "Fluids/MultiFluidContainer_fwd.H" #include "Fluids/WarpXFluidContainer_fwd.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ_fwd.H" # include "BoundaryConditions/PML_RZ_fwd.H" @@ -509,7 +509,7 @@ public: [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) const PML_RZ* getPMLRZ() {return pml_rz[0].get();} #endif @@ -747,7 +747,7 @@ public: static bool isAnyParticleBoundaryThermal(); PML* GetPML (int lev); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) PML_RZ* GetPML_RZ (int lev); #endif @@ -1167,7 +1167,7 @@ public: */ void PSATDSubtractCurrentPartialSumsAvg (); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ SpectralSolverRZ& @@ -1422,7 +1422,7 @@ private: const amrex::IntVect& ngRho, const amrex::IntVect& ngF, const amrex::IntVect& ngG, bool aux_is_nodal); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ void AllocLevelSpectralSolverRZ (amrex::Vector>& spectral_solver, int lev, @@ -1591,7 +1591,7 @@ private: amrex::Vector do_pml_Lo; amrex::Vector do_pml_Hi; amrex::Vector > pml; -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) amrex::Vector > pml_rz; #endif amrex::Real v_particle_pml; @@ -1766,7 +1766,7 @@ private: void PushPSATD (); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT /** * \brief Forward FFT of E,B on all mesh refinement levels diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index de632211bad..512ed7fe9d0 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -18,7 +18,7 @@ #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralKSpace.H" # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" @@ -393,7 +393,7 @@ WarpX::WarpX () charge_buf.resize(nlevs_max); pml.resize(nlevs_max); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) pml_rz.resize(nlevs_max); #endif @@ -467,7 +467,7 @@ WarpX::WarpX () } // Allocate field solver objects -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { spectral_solver_fp.resize(nlevs_max); spectral_solver_cp.resize(nlevs_max); @@ -760,10 +760,10 @@ WarpX::ReadParameters () poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, "The FFT Poisson solver only works in 3D."); #endif -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, - "To use the FFT Poisson solver, compile with WARPX_USE_PSATD=ON."); + "To use the FFT Poisson solver, compile with WARPX_USE_FFT=ON."); #endif WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -1155,7 +1155,7 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id != ElectromagneticSolverAlgo::CKC, "algo.maxwell_solver = ckc is not (yet) available for RZ geometry"); #endif -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD, "algo.maxwell_solver = psatd is not supported because WarpX was built without spectral solvers"); #endif @@ -2091,7 +2091,7 @@ WarpX::ClearLevel (int lev) G_cp [lev].reset(); rho_cp[lev].reset(); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { spectral_solver_fp[lev].reset(); spectral_solver_cp[lev].reset(); @@ -2477,7 +2477,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { // Allocate and initialize the spectral solver -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( false, "WarpX::AllocLevelMFs: PSATD solver requires WarpX build with spectral solver support."); #else @@ -2635,7 +2635,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { // Allocate and initialize the spectral solver -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( false, "WarpX::AllocLevelMFs: PSATD solver requires WarpX build with spectral solver support."); #else @@ -2729,7 +2729,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ /* \brief Allocate spectral Maxwell solver (RZ dimensions) at a level * @@ -2986,7 +2986,7 @@ void WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) { if ( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD ) { -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT spectral_solver_fp[lev]->ComputeSpectralDivE( lev, Efield_aux[lev], divE ); #else WARPX_ABORT_WITH_MESSAGE( @@ -2997,7 +2997,7 @@ WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) } } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) PML_RZ* WarpX::GetPML_RZ (int lev) { diff --git a/Source/ablastr/fields/CMakeLists.txt b/Source/ablastr/fields/CMakeLists.txt index f7dfb6763ec..158e623ebaf 100644 --- a/Source/ablastr/fields/CMakeLists.txt +++ b/Source/ablastr/fields/CMakeLists.txt @@ -1,6 +1,6 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) - if(WarpX_PSATD AND D EQUAL 3) + if(WarpX_FFT AND D EQUAL 3) target_sources(ablastr_${SD} PRIVATE IntegratedGreenFunctionSolver.cpp diff --git a/Source/ablastr/fields/Make.package b/Source/ablastr/fields/Make.package index 4fb7bd88e5f..01392991559 100644 --- a/Source/ablastr/fields/Make.package +++ b/Source/ablastr/fields/Make.package @@ -1,4 +1,4 @@ -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) ifeq ($(DIM),3) CEXE_sources += IntegratedGreenFunctionSolver.cpp endif diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 106c5475dd7..6b1cce415b4 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -15,7 +15,7 @@ #include #include -#if defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D) +#if defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D) #include #endif @@ -160,7 +160,7 @@ computePhi (amrex::Vector const & rho, {{ beta[0], beta[1], beta[2] }}; #endif -#if (defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D)) +#if (defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D)) // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected if(!is_solver_multigrid && lev==0){ amrex::Array const dx_igf diff --git a/Source/ablastr/math/fft/CMakeLists.txt b/Source/ablastr/math/fft/CMakeLists.txt index f7d689c98e9..2b9fc3d37f9 100644 --- a/Source/ablastr/math/fft/CMakeLists.txt +++ b/Source/ablastr/math/fft/CMakeLists.txt @@ -1,6 +1,6 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) - if(WarpX_PSATD STREQUAL ON) + if(WarpX_FFT STREQUAL ON) if(WarpX_COMPUTE STREQUAL CUDA) target_sources(ablastr_${SD} PRIVATE WrapCuFFT.cpp) elseif(WarpX_COMPUTE STREQUAL HIP) diff --git a/Source/ablastr/math/fft/Make.package b/Source/ablastr/math/fft/Make.package index b04062bd9d5..63786cdc006 100644 --- a/Source/ablastr/math/fft/Make.package +++ b/Source/ablastr/math/fft/Make.package @@ -1,4 +1,4 @@ -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) ifeq ($(USE_CUDA),TRUE) CEXE_sources += WrapCuFFT.cpp else ifeq ($(USE_HIP),TRUE) diff --git a/Tools/Linter/runClangTidy.sh b/Tools/Linter/runClangTidy.sh index 39a96cc97d4..046c72d7b27 100755 --- a/Tools/Linter/runClangTidy.sh +++ b/Tools/Linter/runClangTidy.sh @@ -104,7 +104,7 @@ cmake -S ${REPO_DIR} -B ${REPO_DIR}/build_clang_tidy \ -DWarpX_DIMS="1;2;3;RZ" \ -DWarpX_MPI=ON \ -DWarpX_COMPUTE=OMP \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ diff --git a/WarpXConfig.cmake b/WarpXConfig.cmake index 5a2c0916432..8536d08f67a 100644 --- a/WarpXConfig.cmake +++ b/WarpXConfig.cmake @@ -35,11 +35,12 @@ if(WarpX_OPENPMD) endif() set(WarpX_OPENPMD_FOUND ${WarpX_OPENPMD}) -WarpX_ASCENT "Ascent in situ diagnostics" OFF) -WarpX_PSATD "spectral solver support" OFF) -WarpX_SENSEI "SENSEI in situ diagnostics" OFF) -WarpX_QED "QED support (requires PICSAR)" ON) -WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" OFF) +# TODO +#WarpX_ASCENT "Ascent in situ diagnostics" OFF) +#WarpX_FFT "FFT-based solvers" OFF) +#WarpX_SENSEI "SENSEI in situ diagnostics" OFF) +#WarpX_QED "QED support (requires PICSAR)" ON) +#WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" OFF) @@ -50,7 +51,7 @@ endif() set(WarpX_ADIOS2_FOUND ${WarpX_HAVE_ADIOS2}) # define central WarpX::app WarpX::shared ... targets -ABLASTR here, too? or separate Config.cmake? +#ABLASTR here, too? or separate Config.cmake? include("${CMAKE_CURRENT_LIST_DIR}/WarpXTargets.cmake") # check if components are fulfilled and set WarpX__FOUND vars diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index 9b9feb64437..a68ebca6c60 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -305,7 +305,7 @@ function(set_warpx_binary_name D) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".OPMD") endif() - if(WarpX_PSATD) + if(WarpX_FFT) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".PSATD") endif() @@ -452,7 +452,7 @@ function(warpx_print_summary) endif() message(" PARTICLE PRECISION: ${WarpX_PARTICLE_PRECISION}") message(" PRECISION: ${WarpX_PRECISION}") - message(" PSATD: ${WarpX_PSATD}") + message(" FFT Solvers: ${WarpX_FFT}") message(" PYTHON: ${WarpX_PYTHON}") if(WarpX_PYTHON) message(" PYTHON IPO: ${WarpX_PYTHON_IPO}") diff --git a/setup.py b/setup.py index 164f4f1395f..3d61cc5be36 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ def build_extension(self, ext): '-DWarpX_OPENPMD:BOOL=' + WARPX_OPENPMD, '-DWarpX_PRECISION=' + WARPX_PRECISION, '-DWarpX_PARTICLE_PRECISION=' + WARPX_PARTICLE_PRECISION, - '-DWarpX_PSATD:BOOL=' + WARPX_PSATD, + '-DWarpX_FFT:BOOL=' + WARPX_FFT, '-DWarpX_PYTHON:BOOL=ON', '-DWarpX_PYTHON_IPO:BOOL=' + WARPX_PYTHON_IPO, '-DWarpX_QED:BOOL=' + WARPX_QED, @@ -205,7 +205,7 @@ def build_extension(self, ext): WARPX_OPENPMD = env.pop('WARPX_OPENPMD', 'ON') WARPX_PRECISION = env.pop('WARPX_PRECISION', 'DOUBLE') WARPX_PARTICLE_PRECISION = env.pop('WARPX_PARTICLE_PRECISION', WARPX_PRECISION) -WARPX_PSATD = env.pop('WARPX_PSATD', 'OFF') +WARPX_FFT = env.pop('WARPX_FFT', 'OFF') WARPX_QED = env.pop('WARPX_QED', 'ON') WARPX_QED_TABLE_GEN = env.pop('WARPX_QED_TABLE_GEN', 'OFF') WARPX_DIMS = env.pop('WARPX_DIMS', '1;2;RZ;3') From bfffedf5862fb180619567559b48d4e8aa7a0c6d Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Thu, 30 May 2024 20:09:46 -0700 Subject: [PATCH 09/87] FFT-based Poisson solver follow-up (#4945) * changed 2 * add assert to check fft compilation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Source/ablastr/fields/PoissonSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/fields/PoissonSolver.H Co-authored-by: Axel Huebl --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- Source/FieldSolver/ElectrostaticSolver.cpp | 6 +++--- Source/ablastr/fields/PoissonSolver.H | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index a74beae3c91..3860bd55325 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -382,8 +382,8 @@ WarpX::computePhi (const amrex::Vector >& rho, const std::optional > eb_farray_box_factory; #endif - bool const is_solver_multigrid = - WarpX::poisson_solver_id != PoissonSolverAlgo::IntegratedGreenFunction; + bool const is_solver_igf_on_lev0 = + WarpX::poisson_solver_id == PoissonSolverAlgo::IntegratedGreenFunction; ablastr::fields::computePhi( sorted_rho, @@ -397,7 +397,7 @@ WarpX::computePhi (const amrex::Vector >& rho, this->dmap, this->grids, this->m_poisson_boundary_handler, - is_solver_multigrid, + is_solver_igf_on_lev0, WarpX::do_single_precision_comms, this->ref_ratio, post_phi_calculation, diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 6b1cce415b4..e5340e825c9 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -82,7 +82,7 @@ namespace ablastr::fields { * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler - * \param[in] is_solver_multigrid boolean to select the Poisson solver: 1 for Multigrid, 0 for FFT + * \param[in] is_solver_igf_on_lev0 boolean to select the Poisson solver: 1 for FFT on level 0 & Multigrid on other levels, 0 for Multigrid on all levels * \param[in] do_single_precision_comms perform communications in single precision * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) * \param[in] post_phi_calculation perform a calculation per level directly after phi was calculated; required for embedded boundaries (default: none) @@ -106,7 +106,7 @@ computePhi (amrex::Vector const & rho, amrex::Vector const& dmap, amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, - [[maybe_unused]] bool is_solver_multigrid, + bool is_solver_igf_on_lev0, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, @@ -160,9 +160,19 @@ computePhi (amrex::Vector const & rho, {{ beta[0], beta[1], beta[2] }}; #endif +#if !defined(WARPX_USE_FFT) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "Must compile with -DWarpX_FFT=ON to use the FFT solver!"); +#endif + +#if !defined(WARPX_DIM_3D) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "The FFT Poisson solver is currently only implemented for 3D!"); +#endif + #if (defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D)) // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected - if(!is_solver_multigrid && lev==0){ + if(is_solver_igf_on_lev0 && lev==0){ amrex::Array const dx_igf {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), From f9daed91c0a72062aaf0a87420f83185998f025c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 31 May 2024 13:05:09 -0700 Subject: [PATCH 10/87] Document PICMI function `LoadInitialField` (#4962) --- Docs/source/usage/python.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 3aba5f165de..4d53b311764 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -77,6 +77,8 @@ Instances of the classes below need to be passed to the method `add_applied_fiel .. autoclass:: pywarpx.picmi.AnalyticAppliedField +.. autoclass:: pywarpx.picmi.LoadInitialField + .. autoclass:: pywarpx.picmi.PlasmaLens .. autoclass:: pywarpx.picmi.Mirror From 7f4b086eefb1d0b3b36a9d72faacf7d472cc740e Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Sun, 2 Jun 2024 07:02:26 -0700 Subject: [PATCH 11/87] New Implicit Solver interface with options to select Picard or Newton (JFNK) for the nonlinear solver. (#4736) * created Source/FieldSolver/WarXImplicitFieldsEM.cpp file to store functions used with the implicit solvers. Only has ComputeRHSE() and ComputeRHSB() functions thus far. * OneStep_ImplicitPicard routine now uses ComputeRHSE and ComputeRHSB functions in place of EvolveE and EvolveB functions. * created Source/Evolve/WarpXImplicitOps.cpp file to store common functions used by implicit solvers. Moved several functions from WarpXOneStepImplicitPicard.cpp to here. * added NonlinearSolver:: enum. Options are picard and newton. Set from input file using algo.nonlinear_solver = picard, for example. Default is picard. It is not used yet. * changed EvolveScheme::ImplicitPicard and SemiImplicitPicard to ThetaImplicit and SemiImplicit, respectively. This affects parsing from input file. algo.evolve_scheme = implicit_picard ==> theta_implicit. Have not updated Docs yet. * NonlinearSolver ==> NonlinearSolverType * intermediate commit. Added an ImplicitSolversEM class. WarpX owns an instance of this class. OneStepImplicitEM is now done here. This class owns all things pertaining to the implicit integrator. Also added a NonlinerSolvers class, but it is not used yet. * cleaning up ImplicitSolverEM::OneStep(). * more refactoring of ImplicitSovlerEM class. * WarpXFieldVec ==> WarpXSolverVec * removed depricated functions. WarpXSolverVec has zero ghost. * ImplicitSolverEM::OneStep now looks exactly like Picard solver. Next step is to move it there. * ImplicitSolverEM::OneStep() now uses the Picard solver object to solve the nonlinear equation. * changed where implicit solver parameters are printed. * refactoring of WarpXImplicitOps.cpp * added NewtonSolver.H file. Doesn't work yet. * adding more functionality to WarpXSovlerVec class. * added JacobianFunctionJFNK.H file in NonlinarSolvers/ folder. It contains all of the necessary functions required by the linear operator template parameter for AMReX_GMRES. * dotMask object used for dot product from Linear Function class now lives in the implicit solver class. This ensures that 1 and only 1 instance of this object will be defined. dotProduct and norm can now be called through the implicit sovler class. * moved temporary linear_function and GMRES testing lines out of Picard::Define() and into Newton::Define() * intermediate commit. JFNK almost ready. * small refactoring of PreRHSOp() and PostUpdateState() functions. * cleaning things up. * Newton solver runs. GMRES runs. Next step is to do Particle-suppressed (PS) part. * minor clean up. * fixed typo in convergence message for Newton and Picard solvers. * changed how PostUpdateState() is used in implicit solver. Now parsing Picard particle solver parameters in ImplicitSolverEM class. Using a new formula for the epsilon used in the finite-difference Jacobian action calculation that is suitable for large absolute norms of the solution vector. * Picard method for particle is now being used. PS-JFNK works. * moved WarpXImplicitOps.cpp from Source/Evolve/ to Source/FieldSolvers/ImplicitSolvers/ * minor cleanup. PostUpdateState() ==> UpdateWarpXState(). * Moved the particle convergence check into its own function. * added increment function to WarpXSolverVec class. * removed some commented out lines in JacobianFunctionJFNK.H * removed a_tol condition for print message when maximum iterations reached in nonlinear solvers. Newton solver iter is base zero now. * cleaned up picard method for self-consistent particle update. added ablastr warning message for particles that don't converge after the maximum number of iterations. * fixed small bug in PicardSolver related to absolute tolerance check for convegence. Default absolute tolerance values are set to zero. * the mask used to compute the dot product of a WarpXSolverVec is now owned by the WarpXSolverVec class rather then the time solver class. It is a static member to avoid multiple definitions. * defined accessors for Efield_fp and Bfield_fp Vectors owned by WarpX. ImplicitSolver is no longer a friend class of WarpX. Small tidy for PicardSolver.H. * SemiImplicitEM and ThetaImplicitEM are now their own independent derived classes from the base ImplicitSolver class. * added algorithm descriptions to the top of the SemiImplicitEM.cpp and ThetaImplicitEM.cpp files. * updated appropriate files in Examples and Regression folders to reflect new changes. * updating docs. * JacobianFunctionJFNK.H ==> JacobianFunctionMF.H * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * need to call clear on the static vector m_dotMask owned by the WarpXSovlerVec class to avoid malloc_consolidate(): unaligned fastbin chunk detected error messages at simulation finish. * moved WarpXSovlerVec class from ../ImplicitSolvers/Utils/ to ../ImplicitSolvers/. Utils directory is deleted. * ImplicitPushXP: GPU Support for Convergence Test * cleaning up. * Atomic: Fix type of `1` * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * bug fix. * Update Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H Co-authored-by: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> * Remove Zero * adding Copyright lines to top of files. Removed commented code. * Prevent calling the implicit solver if not initialized * More robust if conditions * set implicit verbose to warpx verbose * Update Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp * Simplify call to updates of E * changed benchmarks_json file names as needed. * using warpx.verbose * clang-tidying * changed header names. * clang-tyding * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * more clang-tyding * clang tidy again * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * UpdateElectricFiled ==> SetElectricFieldAndApplyBCs * clang tidy * more clang tidy * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * prohibiting copy and move constructors for solver classes. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed bug in move constructor in WarpXSovlerVec. * slight refactoring of ThetaImplicitEM class. * reducing divisions in Picard method for particles. * small cosmetic changes to implicit solvers. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * removed commented out code. * updating Docs and adding briefs. * Fix HIP compilation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix const * fixed indent. updated comments. * New Python test: Particle-Boundary interaction (#4729) * enable the diagnostic of ParticleScraping in Python * Update picmi.py * Update picmi.py * new test * python update * modification of the script * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update PICMI_inputs_rz.py * json * update * Update PICMI_inputs_rz.py * Update particle_boundary_interaction.json * Update PICMI_inputs_rz.py * Update PICMI_inputs_rz.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update PICMI_inputs_rz.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix hanging script in parallel * Make the test executable * Update analysis script * Update particle_containers.py * Update PICMI_inputs_rz.py * Update analysis.py * Update analysis.py * Update particle_containers.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Remi Lehe Co-authored-by: Axel Huebl * Adding normal components to regular boundary buffer (#4742) * first draft * adding normal only * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Add function to set domain boundary potentials from Python (#4740) * add function to set domain boundary potentials from Python * switch function arguments to form `potential_[lo/hi]_[x/y/z]` and add to docs * clean up `ablastr/fields` (#4753) * move PoissonInterpCPtoFP to Interpolate.H * concatenate nested namespaces * Split clang-tidy CI test into 4 to improve performances (#4747) * split clang-tidy checks to improve performances * rename folders and tests * fix concurrency * Simplify --------- Co-authored-by: Axel Huebl * Replace links to learn git (#4758) * Replace links to learn git * Bugfix in `fields.py` for GPU run without `cupy` (#4750) * Bugfix in `fields.py` for GPU run without `cupy` * apply suggestion from code review * Release 24.03 (#4759) * AMReX: 24.03 * pyAMReX: 24.03 * WarpX: 24.03 * Implement stair-case Yee solver with EB in RZ geometry (#2707) * Allow compilation with RZ EB * Do not push cells for RZ Yee solver, when covered with EB * Fix compilation errors * Fix additional compilation errors * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix additional compilation errors * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add automated test * Add automated test * Fix path in tests * Enable parser in RZ * Update example script * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Clean-up PR * Initialize EB quantities * Modified EM field initialization in 2D with EB * Typo fix * Typo fix * Ignoring unused variables correctly * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Correct condition for updating E * Correct update of B * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update B push * Update input script * Revert "Update input script" This reverts commit 5087485db606f77806c08849c51af9be635ca622. * Update initialization * Updated test * Move test to a different folder * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add test for WarpX-test.ini * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix path for tests * Update test description * Update test metadata * Add checksum file * Revert changes * Revert changes * Change lx to lr * Revert "Change lx to lr" This reverts commit be3039af229d5dee509c8de0a712addf099a2b82. * Change lx to lr --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: lgiacome * AMReX/pyAMReX/PICSAR: Weekly Update (#4763) * AMReX: Weekly Update * pyAMReX: Weekly Update * clean up (#4761) * Fix compilation * updating some function names to contain Implicit. * fixed bug that caused segfault on GMRES restart. * parsing GMRES restart length from input file. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update field accessor * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * removed +, -, and * operators from WarpXSolverVec class. These operators encourage inefficient vector arithmetic. * fix/workaround to field accessor issue. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * changing implicit-scheme related header files in WarpX.H and WarpX.cpp * WarpX::max_particle_iterations ==> WarpX::max_particle_its_in_implicit_scheme and WarpX::particle_tolerance ==> WarpX:particle_tol_in_implicit_scheme * updating docs. * updating docs again. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * adding comments about the template parameters Vec and Ops used in the NonlinearSolver class. * adding comments. addressing comments from PR. * Ensure that laser particles can converge in the implicit solver * Add braces to make clang-tidy happy * moving nonlinear solver parameters to base ImplicitSolver class * mirrors not to be used with implicit schemes. * moved more to base implicit solver class. adding comments. * removed some WarpXSolverVec functions. Updated comments. * clang tidy complains when removing default copy constructor in WarpXSolverVec.H * amrex::ExecOnFinalize(WarpXSolverVec::clearDotMask) * WarpXSolverVec (const WarpXSolverVec&) = delete * updating briefs for nonlinear solvers. * adding loop over levels. * static cast amrex::Vector.size() to int * updating docs for nonlinear solvers. * adding gmres.restart_length to docs. * fixed typos in docs. * Removed PreRHSOp() call from nonlinear solvers. * clang tidy. * Prohibit = operator for WarpXSolverVec. Using Copy() instead. * Document PICMI function `LoadInitialField` * updating comments in WarpXImplicitOps.cpp * moved static member m_dotMask definition to the header file with inline added to the declaration. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed indent. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl Co-authored-by: Debojyoti Ghosh Co-authored-by: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Co-authored-by: Remi Lehe Co-authored-by: Weiqun Zhang Co-authored-by: Eya D <81635404+EyaDammak@users.noreply.github.com> Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Co-authored-by: Arianna Formenti Co-authored-by: Luca Fedeli Co-authored-by: lgiacome --- Docs/source/usage/parameters.rst | 145 ++++-- Examples/Tests/Implicit/analysis_1d.py | 2 +- ..._vandb_2d.py => analysis_vandb_jfnk_2d.py} | 0 Examples/Tests/Implicit/inputs_1d | 14 +- .../Tests/Implicit/inputs_1d_semiimplicit | 14 +- .../{inputs_vandb_2d => inputs_vandb_jfnk_2d} | 30 +- ...d.json => ThetaImplicitJFNK_VandB_2d.json} | 0 ...rd_1d.json => ThetaImplicitPicard_1d.json} | 0 Regression/WarpX-tests.ini | 8 +- Source/Evolve/CMakeLists.txt | 1 - Source/Evolve/Make.package | 1 - Source/Evolve/WarpXEvolve.cpp | 6 +- Source/Evolve/WarpXOneStepImplicitPicard.cpp | 423 ------------------ Source/FieldSolver/CMakeLists.txt | 1 + .../ImplicitSolvers/CMakeLists.txt | 10 + .../ImplicitSolvers/ImplicitSolver.H | 145 ++++++ .../ImplicitSolvers/ImplicitSolverLibrary.H | 13 + .../FieldSolver/ImplicitSolvers/Make.package | 6 + .../ImplicitSolvers/SemiImplicitEM.H | 80 ++++ .../ImplicitSolvers/SemiImplicitEM.cpp | 117 +++++ .../ImplicitSolvers/ThetaImplicitEM.H | 124 +++++ .../ImplicitSolvers/ThetaImplicitEM.cpp | 171 +++++++ .../ImplicitSolvers/WarpXImplicitOps.cpp | 352 +++++++++++++++ .../ImplicitSolvers/WarpXSolverVec.H | 233 ++++++++++ .../ImplicitSolvers/WarpXSolverVec.cpp | 51 +++ Source/FieldSolver/Make.package | 1 + Source/Initialization/WarpXInitData.cpp | 26 ++ Source/Make.WarpX | 3 +- Source/NonlinearSolvers/CMakeLists.txt | 6 + Source/NonlinearSolvers/JacobianFunctionMF.H | 234 ++++++++++ Source/NonlinearSolvers/Make.package | 1 + Source/NonlinearSolvers/NewtonSolver.H | 345 ++++++++++++++ Source/NonlinearSolvers/NonlinearSolver.H | 87 ++++ .../NonlinearSolvers/NonlinearSolverLibrary.H | 15 + Source/NonlinearSolvers/PicardSolver.H | 217 +++++++++ Source/Particles/LaserParticleContainer.H | 2 +- Source/Particles/LaserParticleContainer.cpp | 46 +- .../Particles/PhysicalParticleContainer.cpp | 185 +++++--- Source/Particles/Pusher/UpdatePosition.H | 42 ++ Source/Utils/WarpXAlgorithmSelection.H | 4 +- Source/Utils/WarpXAlgorithmSelection.cpp | 8 +- Source/WarpX.H | 63 ++- Source/WarpX.cpp | 67 ++- 43 files changed, 2717 insertions(+), 582 deletions(-) rename Examples/Tests/Implicit/{analysis_vandb_2d.py => analysis_vandb_jfnk_2d.py} (100%) rename Examples/Tests/Implicit/{inputs_vandb_2d => inputs_vandb_jfnk_2d} (78%) rename Regression/Checksum/benchmarks_json/{ImplicitPicard_VandB_2d.json => ThetaImplicitJFNK_VandB_2d.json} (100%) rename Regression/Checksum/benchmarks_json/{ImplicitPicard_1d.json => ThetaImplicitPicard_1d.json} (100%) delete mode 100644 Source/Evolve/WarpXOneStepImplicitPicard.cpp create mode 100644 Source/FieldSolver/ImplicitSolvers/CMakeLists.txt create mode 100644 Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H create mode 100644 Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H create mode 100644 Source/FieldSolver/ImplicitSolvers/Make.package create mode 100644 Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H create mode 100644 Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp create mode 100644 Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H create mode 100644 Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp create mode 100644 Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp create mode 100644 Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H create mode 100644 Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp create mode 100644 Source/NonlinearSolvers/CMakeLists.txt create mode 100644 Source/NonlinearSolvers/JacobianFunctionMF.H create mode 100644 Source/NonlinearSolvers/Make.package create mode 100644 Source/NonlinearSolvers/NewtonSolver.H create mode 100644 Source/NonlinearSolvers/NonlinearSolver.H create mode 100644 Source/NonlinearSolvers/NonlinearSolverLibrary.H create mode 100644 Source/NonlinearSolvers/PicardSolver.H diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index f3a74f969a0..fb53e81012f 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -86,37 +86,122 @@ Overall simulation parameters * ``explicit``: Use an explicit solver, such as the standard FDTD or PSATD - * ``implicit_picard``: Use an implicit solver with exact energy conservation that uses a Picard iteration to solve the system. - Note that this method is for demonstration only. It is inefficient and does not work well when - :math:`\omega_{pe} \Delta t` is close to or greater than one. - The method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. - The version implemented is an updated version that is relativistically correct, including the relativistic gamma factor for the particles. - For exact energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, - and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in - which case charge will also be conserved). - - * ``semi_implicit_picard``: Use an energy conserving semi-implicit solver that uses a Picard iteration to solve the system. - Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. It is inefficient and does not work well or at all when :math:`\omega_{pe} \Delta t` is close to or greater than one. + * ``theta_implicit_em``: Use a fully implicit electromagnetic solver with a time-biasing parameter theta bound between 0.5 and 1.0. Exact energy conservation is achieved using theta = 0.5. Maximal damping of high-k modes is obtained using theta = 1.0. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed (PS) JNFK. + The algorithm itself is numerical stable for large time steps. That is, it does not require time steps that resolve the plasma period or the CFL condition for light waves. However, the practicality of using a large time step depends on the nonlinear solver. Note that the Picard solver is for demonstration only. It is inefficient and will most like not converge when + :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relavent physics. + The Picard method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. + The PS-JFNK method is described in `Angus et al., An implicit particle code with exact energy and charge conservation for electromagnetic studies of dense plasmas `__ . (The version implemented in WarpX is an updated version that includes the relativistic gamma factor for the particles.) Also see `Chen et al., An energy- and charge-conserving, implicit, electrostatic particle-in-cell algorithm. `__ . + Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. + + * ``semi_implicit_em``: Use an approximately energy conserving semi-implicit electromagnetic solver. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed JFNK. + Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. The Picard solver for this method can only be expected to work well when :math:`\omega_{pe} \Delta t` is less than one. The method is described in `Chen et al., A semi-implicit, energy- and charge-conserving particle-in-cell algorithm for the relativistic Vlasov-Maxwell equations `__. - For energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, - and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in - which case charge will also be conserved). - -* ``algo.max_picard_iterations`` (`integer`, default: 10) - When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the maximum number of Picard - itearations that are done each time step. - -* ``algo.picard_iteration_tolerance`` (`float`, default: 1.e-7) - When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the convergence tolerance of - the iterations, the maximum of the relative change of the L2 norm of the field from one iteration to the next. - If this is set to zero, the maximum number of iterations will always be done with the change only calculated on the last - iteration (for a slight optimization). - -* ``algo.require_picard_convergence`` (`bool`, default: 1) - When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets whether the iteration each step - is required to converge. - If it is required, an abort is raised if it does not converge and the code then exits. - If not, then a warning is issued and the calculation continues. + Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. + +* ``implicit_evolve.theta`` (`float`, default: 0.5) + When `algo.evolve_scheme = theta_implicit_em`, the fields used on the RHS of the equations for the implicit advance + are computed as (1-theta)*E_{n} + theta*E_{n+1}. theta is bound between 0.5 and 1. The default value of theta = 0.5 + is needed for exact energy conservation. For theta > 0.5, high-k modes will be damped and the method will not be + exactly energy conserving, but the solver may perform better. + +* ``implicit_evolve.nonlinear_solver`` (`string`, default: None) + When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em`, this sets the nonlinear solver used + to advance the field-particle system in time. Options are `picard` or `newton`. + +* ``implicit_evolve.max_particle_iterations`` (`integer`, default: 21) + When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + , this sets the maximum number of iterations for the method used to obtain a self-consistent update of the particles at + each iteration in the JFNK process. + +* ``implicit_evolve.particle_tolerance`` (`float`, default: 1.e-10) + When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + , this sets the relative tolerance for the iterative method used to obtain a self-consistent update of the particles at + each iteration in the JFNK process. + +* ``picard.verbose`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = picard`, this sets the verbosity of the Picard solver. If true, then information + on the nonlinear error are printed to screen at each nonlinear iteration. + +* ``picard.require_convergence`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = picard`, this sets whether the Picard method is required to converge at each + time step. If it is required, an abort is raised if it does not converge and the code then exits. If not, then a warning + is issued and the calculation continues. + +* ``picard.maximum_iterations`` (`int`, default: 100) + When `implicit_evolve.nonlinear_solver = picard`, this sets the maximum iterations used by the Picard method. If + `picard.require_convergence = false`, the solution is considered converged if the iteration count reaches this value, + but a warning is issued. If `picard.require_convergence = true`, then an abort is raised if the iteration count reaches + this value. + +* ``picard.relative_tolerance`` (`float`, default: 1.0e-6) + When `implicit_evolve.nonlinear_solver = picard`, this sets the relative tolerance used by the Picard method for determining + convergence. The absolute error for the Picard method is the L2 norm of the difference of the solution vector between + two successive iterations. The relative error is the absolute error after iteration k > 1 divided by the absolute error + after the first iteration. The Picard method is considered converged when the relative error is below the relative tolerance. + This is the preferred means of determining convergence. + +* ``picard.absolute_tolerance`` (`float`, default: 0.0) + When `implicit_evolve.nonlinear_solver = picard`, this sets the absolute tolerance used by the Picard method for determining + convergence. The default value is 0.0, which means that the absolute tolerance is not used to determine converence. The + solution vector in the nonlinear solvers are in physical units rather than normalized ones. Thus, the absolute scale + of the problem can vary over many orders and magnitude depending on the problem. The relative tolerance is the preferred + means of determining convergence. + +* ``newton.verbose`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = newton`, this sets the verbosity of the Newton solver. If true, then information + on the nonlinear error are printed to screen at each nonlinear iteration. + +* ``newton.require_convergence`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = newton`, this sets whether the Newton method is required to converge at each + time step. If it is required, an abort is raised if it does not converge and the code then exits. If not, then a warning + is issued and the calculation continues. + +* ``newton.maximum_iterations`` (`int`, default: 100) + When `implicit_evolve.nonlinear_solver = newton`, this sets the maximum iterations used by the Newton method. If + `newton.require_convergence = false`, the solution is considered converged if the iteration count reaches this value, + but a warning is issued. If `newton.require_convergence = true`, then an abort is raised if the iteration count reaches + this value. + +* ``newton.relative_tolerance`` (`float`, default: 1.0e-6) + When `implicit_evolve.nonlinear_solver = newton`, this sets the relative tolerance used by the Newton method for determining + convergence. The absolute error for the Newton method is the L2 norm of the residual vector. The relative error is the + absolute error divided by the L2 norm of the initial residual associated with the initial guess. The Newton method is + considered converged when the relative error is below the relative tolerance. This is the preferred means of determining + convergence. + +* ``newton.absolute_tolerance`` (`float`, default: 0.0) + When `implicit_evolve.nonlinear_solver = newton`, this sets the absolute tolerance used by the Newton method for determining + convergence. The default value is 0.0, which means that the absolute tolerance is not used to determine converence. The + residual vector in the nonlinear solvers are in physical units rather than normalized ones. Thus, the absolute scale + of the problem can vary over many orders and magnitude depending on the problem. The relative tolerance is the preferred + means of determining convergence. + +* ``gmres.verbose_int`` (`int`, default: 2) + When `implicit_evolve.nonlinear_solver = newton`, this sets the verbosity of the AMReX::GMRES linear solver. The default + value of 2 gives maximumal verbosity and information about the residual are printed to the screen at each GMRES iteration. + +* ``gmres.restart_length`` (`int`, default: 30) + When `implicit_evolve.nonlinear_solver = newton`, this sets the iteration number at which to do a restart in AMReX::GMRES. + This parameter is used to save memory on building the Krylov subspace basis vectors for linear systems that are ill-conditioned + and require many iterations to converge. + +* ``gmres.relative_tolerance`` (`float`, default: 1.0e-4) + When `implicit_evolve.nonlinear_solver = newton`, this sets the relative tolerance used to determine convergence of the + AMReX::GMRES linear solver used to compute the Newton step in the JNFK process. The absolute error is the L2 norm of the + residual vector. The relative error is the absolute error divided by the L2 norm of the initial residual (typically equal + to the norm of the nonlinear residual from the end of the previous Newton iteration). The linear solver is considered + converged when the relative error is below the relative tolerance. This is the preferred means of determining convergence. + +* ``gmres.absolute_tolerance`` (`float`, default: 0.0) + When `implicit_evolve.nonlinear_solver = newton`, this sets the absolute tolerance used to determine converence of the + GMRES linear solver used to compute the Newton step in the JNFK process. The default value is 0.0, which means that the + absolute tolerance is not used to determine converence. The residual vector in the nonlinear/linear solvers are in physical + units rather than normalized ones. Thus, the absolute scale of the problem can vary over many orders and magnitude depending + on the problem. The relative tolerance is the preferred means of determining convergence. + +* ``gmres.maximum_iterations`` (`int`, default: 1000) + When `implicit_evolve.nonlinear_solver = newton`, this sets the maximum iterations used by the GMRES linear solver. The + solution to the linear system is considered converged if the iteration count reaches this value. * ``warpx.do_electrostatic`` (`string`) optional (default `none`) Specifies the electrostatic mode. When turned on, instead of updating diff --git a/Examples/Tests/Implicit/analysis_1d.py b/Examples/Tests/Implicit/analysis_1d.py index 0f00010a505..0e20b925df5 100755 --- a/Examples/Tests/Implicit/analysis_1d.py +++ b/Examples/Tests/Implicit/analysis_1d.py @@ -31,7 +31,7 @@ if re.match('SemiImplicitPicard_1d', fn): tolerance_rel = 2.5e-5 -elif re.match('ImplicitPicard_1d', fn): +elif re.match('ThetaImplicitPicard_1d', fn): # This case should have near machine precision conservation of energy tolerance_rel = 1.e-14 diff --git a/Examples/Tests/Implicit/analysis_vandb_2d.py b/Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py similarity index 100% rename from Examples/Tests/Implicit/analysis_vandb_2d.py rename to Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d index 465d9dd6965..3e57689b723 100644 --- a/Examples/Tests/Implicit/inputs_1d +++ b/Examples/Tests/Implicit/inputs_1d @@ -31,10 +31,18 @@ boundary.particle_hi = periodic ############ NUMERICS ########### ################################# +warpx.verbose = 1 warpx.const_dt = dt -algo.evolve_scheme = implicit_picard -algo.max_picard_iterations = 31 -algo.picard_iteration_tolerance = 0. +algo.evolve_scheme = theta_implicit_em + +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 31 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + algo.current_deposition = esirkepov algo.field_gathering = energy-conserving algo.particle_shape = 2 diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit index 4008a559588..07460e08be8 100644 --- a/Examples/Tests/Implicit/inputs_1d_semiimplicit +++ b/Examples/Tests/Implicit/inputs_1d_semiimplicit @@ -31,10 +31,18 @@ boundary.particle_hi = periodic ############ NUMERICS ########### ################################# +warpx.verbose = 1 warpx.const_dt = dt -algo.evolve_scheme = semi_implicit_picard -algo.max_picard_iterations = 5 -algo.picard_iteration_tolerance = 0. +algo.evolve_scheme = semi_implicit_em + +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 5 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + algo.current_deposition = esirkepov algo.field_gathering = energy-conserving algo.particle_shape = 2 diff --git a/Examples/Tests/Implicit/inputs_vandb_2d b/Examples/Tests/Implicit/inputs_vandb_jfnk_2d similarity index 78% rename from Examples/Tests/Implicit/inputs_vandb_2d rename to Examples/Tests/Implicit/inputs_vandb_jfnk_2d index 33ce964710a..393a9d90330 100644 --- a/Examples/Tests/Implicit/inputs_vandb_2d +++ b/Examples/Tests/Implicit/inputs_vandb_jfnk_2d @@ -38,10 +38,32 @@ warpx.const_dt = dt warpx.use_filter = 0 algo.maxwell_solver = Yee -algo.evolve_scheme = "implicit_picard" -algo.require_picard_convergence = 0 -algo.max_picard_iterations = 25 -algo.picard_iteration_tolerance = 0.0 #1.0e-12 +algo.evolve_scheme = "theta_implicit_em" +#algo.evolve_scheme = "semi_implicit_em" + +implicit_evolve.theta = 0.5 +implicit_evolve.max_particle_iterations = 21 +implicit_evolve.particle_tolerance = 1.0e-12 + +#implicit_evolve.nonlinear_solver = "picard" +#picard.verbose = true +#picard.max_iterations = 25 +#picard.relative_tolerance = 0.0 #1.0e-12 +#picard.absolute_tolerance = 0.0 #1.0e-24 +#picard.require_convergence = false + +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 20 +newton.relative_tolerance = 1.0e-12 +newton.absolute_tolerance = 0.0 +newton.require_convergence = false + +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-8 +gmres.absolute_tolerance = 0.0 + algo.particle_pusher = "boris" #algo.particle_pusher = "higuera" diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json b/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json rename to Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d.json diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/ThetaImplicitPicard_1d.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json rename to Regression/Checksum/benchmarks_json/ThetaImplicitPicard_1d.json diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 6ee7ad3929d..e2d69be9c60 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4666,7 +4666,7 @@ particleTypes = electrons outputFile = Point_of_contact_EB_rz_plt analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py -[ImplicitPicard_1d] +[ThetaImplicitPicard_1d] buildDir = . inputFile = Examples/Tests/Implicit/inputs_1d runtime_params = warpx.abort_on_warning_threshold=high @@ -4683,9 +4683,9 @@ doVis = 0 compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py -[ImplicitPicard_VandB_2d] +[ThetaImplicitJFNK_VandB_2d] buildDir = . -inputFile = Examples/Tests/Implicit/inputs_vandb_2d +inputFile = Examples/Tests/Implicit/inputs_vandb_jfnk_2d runtime_params = warpx.abort_on_warning_threshold=high dim = 2 addToCompileString = @@ -4698,7 +4698,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 1 -analysisRoutine = Examples/Tests/Implicit/analysis_vandb_2d.py +analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py [SemiImplicitPicard_1d] buildDir = . diff --git a/Source/Evolve/CMakeLists.txt b/Source/Evolve/CMakeLists.txt index a84d6e1e42b..3ef0dae5e8e 100644 --- a/Source/Evolve/CMakeLists.txt +++ b/Source/Evolve/CMakeLists.txt @@ -4,6 +4,5 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE WarpXEvolve.cpp WarpXComputeDt.cpp - WarpXOneStepImplicitPicard.cpp ) endforeach() diff --git a/Source/Evolve/Make.package b/Source/Evolve/Make.package index a9a877475b9..275b8bfde5a 100644 --- a/Source/Evolve/Make.package +++ b/Source/Evolve/Make.package @@ -1,5 +1,4 @@ CEXE_sources += WarpXEvolve.cpp CEXE_sources += WarpXComputeDt.cpp -CEXE_sources += WarpXOneStepImplicitPicard.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Evolve diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 881967a0479..4ac50483ad3 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -125,10 +125,8 @@ WarpX::Evolve (int numsteps) ExecutePythonCallback("particleinjection"); - // TODO: move out - if (evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard) { - OneStep_ImplicitPicard(cur_time); + if (m_implicit_solver) { + m_implicit_solver->OneStep(cur_time, dt[0], step); } else if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) diff --git a/Source/Evolve/WarpXOneStepImplicitPicard.cpp b/Source/Evolve/WarpXOneStepImplicitPicard.cpp deleted file mode 100644 index 6df3cd0d4e6..00000000000 --- a/Source/Evolve/WarpXOneStepImplicitPicard.cpp +++ /dev/null @@ -1,423 +0,0 @@ -/* Copyright 2022 David Grote - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#include "WarpX.H" - -#include "BoundaryConditions/PML.H" -#include "Diagnostics/MultiDiagnostics.H" -#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" -#include "Evolve/WarpXDtType.H" -#include "Evolve/WarpXPushType.H" -#ifdef WARPX_USE_FFT -# ifdef WARPX_DIM_RZ -# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" -# else -# include "FieldSolver/SpectralSolver/SpectralSolver.H" -# endif -#endif -#include "Parallelization/GuardCellManager.H" -#include "Particles/MultiParticleContainer.H" -#include "Particles/ParticleBoundaryBuffer.H" -#include "Python/callbacks.H" -#include "Utils/TextMsg.H" -#include "Utils/WarpXAlgorithmSelection.H" -#include "Utils/WarpXUtil.H" -#include "Utils/WarpXConst.H" -#include "Utils/WarpXProfilerWrapper.H" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -void -WarpX::EvolveImplicitPicardInit (int lev) -{ - - if (lev == 0) { - // Add space to save the positions and velocities at the start of the time steps - for (auto const& pc : *mypc) { -#if (AMREX_SPACEDIM >= 2) - pc->AddRealComp("x_n"); -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - pc->AddRealComp("y_n"); -#endif - pc->AddRealComp("z_n"); - pc->AddRealComp("ux_n"); - pc->AddRealComp("uy_n"); - pc->AddRealComp("uz_n"); - } - } - - // Initialize MultiFabs to hold the E and B fields at the start of the time steps - // Only one refinement level is supported - const int nlevs_max = maxLevel() + 1; - Efield_n.resize(nlevs_max); - Efield_save.resize(nlevs_max); - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - Bfield_n.resize(nlevs_max); - Bfield_save.resize(nlevs_max); - } - - // The Efield_n and Bfield_n will hold the fields at the start of the time step. - // This is needed since in each iteration the fields are advanced from the values - // at the start of the step. - // The Efield_save and Bfield_save will hold the fields from the previous iteration, - // to check the change in the fields after the iterations to check for convergence. - // The Efiel_fp and Bfield_fp will hole the n+theta during the iterations and then - // advance to the n+1 time level after the iterations complete. - AllocInitMultiFabFromModel(Efield_n[lev][0], *Efield_fp[0][0], lev, "Efield_n[0]"); - AllocInitMultiFabFromModel(Efield_n[lev][1], *Efield_fp[0][1], lev, "Efield_n[1]"); - AllocInitMultiFabFromModel(Efield_n[lev][2], *Efield_fp[0][2], lev, "Efield_n[2]"); - AllocInitMultiFabFromModel(Efield_save[lev][0], *Efield_fp[0][0], lev, "Efield_save[0]"); - AllocInitMultiFabFromModel(Efield_save[lev][1], *Efield_fp[0][1], lev, "Efield_save[1]"); - AllocInitMultiFabFromModel(Efield_save[lev][2], *Efield_fp[0][2], lev, "Efield_save[2]"); - - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - AllocInitMultiFabFromModel(Bfield_n[lev][0], *Bfield_fp[0][0], lev, "Bfield_n[0]"); - AllocInitMultiFabFromModel(Bfield_n[lev][1], *Bfield_fp[0][1], lev, "Bfield_n[1]"); - AllocInitMultiFabFromModel(Bfield_n[lev][2], *Bfield_fp[0][2], lev, "Bfield_n[2]"); - AllocInitMultiFabFromModel(Bfield_save[lev][0], *Bfield_fp[0][0], lev, "Bfield_save[0]"); - AllocInitMultiFabFromModel(Bfield_save[lev][1], *Bfield_fp[0][1], lev, "Bfield_save[1]"); - AllocInitMultiFabFromModel(Bfield_save[lev][2], *Bfield_fp[0][2], lev, "Bfield_save[2]"); - } - -} - -void -WarpX::OneStep_ImplicitPicard(amrex::Real cur_time) -{ - using namespace amrex::literals; - - // We have E^{n}. - // Particles have p^{n} and x^{n}. - // With full implicit, B^{n} - // With semi-implicit, B^{n-1/2} - - // Save the values at the start of the time step, - // copying particle data to x_n etc. - for (auto const& pc : *mypc) { - SaveParticlesAtImplicitStepStart (*pc, 0); - } - - // Save the fields at the start of the step - amrex::MultiFab::Copy(*Efield_n[0][0], *Efield_fp[0][0], 0, 0, ncomps, Efield_fp[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_n[0][1], *Efield_fp[0][1], 0, 0, ncomps, Efield_fp[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_n[0][2], *Efield_fp[0][2], 0, 0, ncomps, Efield_fp[0][2]->nGrowVect()); - - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - amrex::MultiFab::Copy(*Bfield_n[0][0], *Bfield_fp[0][0], 0, 0, ncomps, Bfield_fp[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_n[0][1], *Bfield_fp[0][1], 0, 0, ncomps, Bfield_fp[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_n[0][2], *Bfield_fp[0][2], 0, 0, ncomps, Bfield_fp[0][2]->nGrowVect()); - } else if (evolve_scheme == EvolveScheme::SemiImplicitPicard) { - // This updates Bfield_fp so it holds the new B at n+1/2 - EvolveB(dt[0], DtType::Full); - // WarpX::sync_nodal_points is used to avoid instability - FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); - } - - // Start the iterations - amrex::Real deltaE = 1._rt; - amrex::Real deltaB = 1._rt; - int iteration_count = 0; - while (iteration_count < max_picard_iterations && - (deltaE > picard_iteration_tolerance || deltaB > picard_iteration_tolerance)) { - iteration_count++; - - // Advance the particle positions by 1/2 dt, - // particle velocities by dt, then take average of old and new v, - // deposit currents, giving J at n+1/2 - // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration. - const bool skip_current = false; - const PushType push_type = PushType::Implicit; - PushParticlesandDeposit(cur_time, skip_current, push_type); - - SyncCurrentAndRho(); - - if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { - // Save the E at n+1/2 from the previous iteration so that the change - // in this iteration can be calculated - amrex::MultiFab::Copy(*Efield_save[0][0], *Efield_fp[0][0], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Efield_save[0][1], *Efield_fp[0][1], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Efield_save[0][2], *Efield_fp[0][2], 0, 0, ncomps, 0); - } - - // Copy Efield_n into Efield_fp since EvolveE updates Efield_fp in place - amrex::MultiFab::Copy(*Efield_fp[0][0], *Efield_n[0][0], 0, 0, ncomps, Efield_n[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_fp[0][1], *Efield_n[0][1], 0, 0, ncomps, Efield_n[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_fp[0][2], *Efield_n[0][2], 0, 0, ncomps, Efield_n[0][2]->nGrowVect()); - - // Updates Efield_fp so it holds the new E at n+1/2 - EvolveE(0.5_rt*dt[0]); - // WarpX::sync_nodal_points is used to avoid instability - FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyEfieldBoundary(0, PatchType::fine); - - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { - // Save the B at n+1/2 from the previous iteration so that the change - // in this iteration can be calculated - amrex::MultiFab::Copy(*Bfield_save[0][0], *Bfield_fp[0][0], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Bfield_save[0][1], *Bfield_fp[0][1], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Bfield_save[0][2], *Bfield_fp[0][2], 0, 0, ncomps, 0); - } - - // Copy Bfield_n into Bfield_fp since EvolveB updates Bfield_fp in place - amrex::MultiFab::Copy(*Bfield_fp[0][0], *Bfield_n[0][0], 0, 0, ncomps, Bfield_n[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_fp[0][1], *Bfield_n[0][1], 0, 0, ncomps, Bfield_n[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_fp[0][2], *Bfield_n[0][2], 0, 0, ncomps, Bfield_n[0][2]->nGrowVect()); - - // This updates Bfield_fp so it holds the new B at n+1/2 - EvolveB(0.5_rt*dt[0], DtType::Full); - // WarpX::sync_nodal_points is used to avoid instability - FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); - } - - // The B field update needs - if (num_mirrors>0){ - applyMirrors(cur_time); - // E : guard cells are NOT up-to-date from the mirrors - // B : guard cells are NOT up-to-date from the mirrors - } - - if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { - // Calculate the change in E and B from this iteration - // deltaE = abs(Enew - Eold)/max(abs(Enew)) - Efield_save[0][0]->minus(*Efield_fp[0][0], 0, ncomps, 0); - Efield_save[0][1]->minus(*Efield_fp[0][1], 0, ncomps, 0); - Efield_save[0][2]->minus(*Efield_fp[0][2], 0, ncomps, 0); - const amrex::Real maxE0 = std::max(1._rt, Efield_fp[0][0]->norm0(0, 0)); - const amrex::Real maxE1 = std::max(1._rt, Efield_fp[0][1]->norm0(0, 0)); - const amrex::Real maxE2 = std::max(1._rt, Efield_fp[0][2]->norm0(0, 0)); - const amrex::Real deltaE0 = Efield_save[0][0]->norm0(0, 0)/maxE0; - const amrex::Real deltaE1 = Efield_save[0][1]->norm0(0, 0)/maxE1; - const amrex::Real deltaE2 = Efield_save[0][2]->norm0(0, 0)/maxE2; - deltaE = std::max(std::max(deltaE0, deltaE1), deltaE2); - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - Bfield_save[0][0]->minus(*Bfield_fp[0][0], 0, ncomps, 0); - Bfield_save[0][1]->minus(*Bfield_fp[0][1], 0, ncomps, 0); - Bfield_save[0][2]->minus(*Bfield_fp[0][2], 0, ncomps, 0); - const amrex::Real maxB0 = std::max(1._rt, Bfield_fp[0][0]->norm0(0, 0)); - const amrex::Real maxB1 = std::max(1._rt, Bfield_fp[0][1]->norm0(0, 0)); - const amrex::Real maxB2 = std::max(1._rt, Bfield_fp[0][2]->norm0(0, 0)); - const amrex::Real deltaB0 = Bfield_save[0][0]->norm0(0, 0)/maxB0; - const amrex::Real deltaB1 = Bfield_save[0][1]->norm0(0, 0)/maxB1; - const amrex::Real deltaB2 = Bfield_save[0][2]->norm0(0, 0)/maxB2; - deltaB = std::max(std::max(deltaB0, deltaB1), deltaB2); - } else { - deltaB = 0.; - } - amrex::Print() << "Max delta " << iteration_count << " " << deltaE << " " << deltaB << "\n"; - } - - // Now, the particle positions and velocities and the Efield_fp and Bfield_fp hold - // the new values at n+1/2 - } - - amrex::Print() << "Picard iterations = " << iteration_count << ", Eerror = " << deltaE << ", Berror = " << deltaB << "\n"; - if (picard_iteration_tolerance > 0. && iteration_count == max_picard_iterations) { - std::stringstream convergenceMsg; - convergenceMsg << "The Picard implicit solver failed to converge after " << iteration_count << " iterations, with Eerror = " << deltaE << ", Berror = " << deltaB << " with a tolerance of " << picard_iteration_tolerance; - if (require_picard_convergence) { - WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); - } else { - ablastr::warn_manager::WMRecordWarning("PicardSolver", convergenceMsg.str()); - } - } - - // Advance particles to step n+1 - for (auto const& pc : *mypc) { - FinishImplicitParticleUpdate(*pc, 0); - } - - // Advance fields to step n+1 - // WarpX::sync_nodal_points is used to avoid instability - FinishImplicitFieldUpdate(Efield_fp, Efield_n); - FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - FinishImplicitFieldUpdate(Bfield_fp, Bfield_n); - FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - } - -} - -void -WarpX::SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev) -{ - -#ifdef AMREX_USE_OMP -#pragma omp parallel -#endif - { - - auto particle_comps = pc.getParticleComps(); - - for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { - - const auto getPosition = GetParticlePosition(pti); - - auto& attribs = pti.GetAttribs(); - amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - -#if (AMREX_SPACEDIM >= 2) - amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); -#endif - amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); - amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); - amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); - amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); - - const long np = pti.numParticles(); - - amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) - { - amrex::ParticleReal xp, yp, zp; - getPosition(ip, xp, yp, zp); - -#if (AMREX_SPACEDIM >= 2) - x_n[ip] = xp; -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - y_n[ip] = yp; -#endif - z_n[ip] = zp; - - ux_n[ip] = ux[ip]; - uy_n[ip] = uy[ip]; - uz_n[ip] = uz[ip]; - - }); - - } - } -} - -void -WarpX::FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev) -{ - using namespace amrex::literals; - -#ifdef AMREX_USE_OMP -#pragma omp parallel -#endif - { - - auto particle_comps = pc.getParticleComps(); - - for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { - - const auto getPosition = GetParticlePosition(pti); - const auto setPosition = SetParticlePosition(pti); - - auto& attribs = pti.GetAttribs(); - amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - -#if (AMREX_SPACEDIM >= 2) - amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); -#endif - amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); - amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); - amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); - amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); - - const long np = pti.numParticles(); - - amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) - { - amrex::ParticleReal xp, yp, zp; - getPosition(ip, xp, yp, zp); - -#if (AMREX_SPACEDIM >= 2) - xp = 2._rt*xp - x_n[ip]; -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - yp = 2._rt*yp - y_n[ip]; -#endif - zp = 2._rt*zp - z_n[ip]; - - ux[ip] = 2._rt*ux[ip] - ux_n[ip]; - uy[ip] = 2._rt*uy[ip] - uy_n[ip]; - uz[ip] = 2._rt*uz[ip] - uz_n[ip]; - - setPosition(ip, xp, yp, zp); - }); - - } - } -} - -void -WarpX::FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Field_fp, - amrex::Vector, 3 > >& Field_n) -{ - using namespace amrex::literals; - - for (int lev = 0; lev <= finest_level; ++lev) { - -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for ( amrex::MFIter mfi(*Field_fp[lev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { - - amrex::Array4 const& Fx = Field_fp[lev][0]->array(mfi); - amrex::Array4 const& Fy = Field_fp[lev][1]->array(mfi); - amrex::Array4 const& Fz = Field_fp[lev][2]->array(mfi); - - amrex::Array4 const& Fx_n = Field_n[lev][0]->array(mfi); - amrex::Array4 const& Fy_n = Field_n[lev][1]->array(mfi); - amrex::Array4 const& Fz_n = Field_n[lev][2]->array(mfi); - - amrex::Box const tbx = mfi.tilebox(Field_fp[lev][0]->ixType().toIntVect()); - amrex::Box const tby = mfi.tilebox(Field_fp[lev][1]->ixType().toIntVect()); - amrex::Box const tbz = mfi.tilebox(Field_fp[lev][2]->ixType().toIntVect()); - - amrex::ParallelFor( - tbx, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) - { - Fx(i,j,k,n) = 2._rt*Fx(i,j,k,n) - Fx_n(i,j,k,n); - }, - tby, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) - { - Fy(i,j,k,n) = 2._rt*Fy(i,j,k,n) - Fy_n(i,j,k,n); - }, - tbz, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) - { - Fz(i,j,k,n) = 2._rt*Fz(i,j,k,n) - Fz_n(i,j,k,n); - }); - } - } -} diff --git a/Source/FieldSolver/CMakeLists.txt b/Source/FieldSolver/CMakeLists.txt index cf741aa6e01..859117eb214 100644 --- a/Source/FieldSolver/CMakeLists.txt +++ b/Source/FieldSolver/CMakeLists.txt @@ -11,6 +11,7 @@ endforeach() add_subdirectory(FiniteDifferenceSolver) add_subdirectory(MagnetostaticSolver) +add_subdirectory(ImplicitSolvers) if(WarpX_FFT) add_subdirectory(SpectralSolver) endif() diff --git a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt new file mode 100644 index 00000000000..6e16f19084c --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt @@ -0,0 +1,10 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + SemiImplicitEM.cpp + ThetaImplicitEM.cpp + WarpXImplicitOps.cpp + WarpXSolverVec.cpp + ) +endforeach() diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H new file mode 100644 index 00000000000..88ad6a058fd --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H @@ -0,0 +1,145 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef Implicit_Solver_H_ +#define Implicit_Solver_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" +#include "NonlinearSolvers/NonlinearSolverLibrary.H" + +#include +#include + +/** + * \brief Base class for implicit time solvers. The base functions are those + * needed by an implicit solver to be used through WarpX and those needed + * to use the nonlinear solvers, such as Picard or Newton (i.e., JFNK). + */ + +class WarpX; +class ImplicitSolver +{ +public: + + ImplicitSolver() = default; + + virtual ~ImplicitSolver() = default; + + // Prohibit Move and Copy operations + ImplicitSolver(const ImplicitSolver&) = delete; + ImplicitSolver& operator=(const ImplicitSolver&) = delete; + ImplicitSolver(ImplicitSolver&&) = delete; + ImplicitSolver& operator=(ImplicitSolver&&) = delete; + + // + // the following routines are called by WarpX + // + + /** + * \brief Read user-provided parameters that control the implicit solver. + * Allocate internal arrays for intermediate field values needed by the solver. + */ + virtual void Define ( WarpX* a_WarpX ) = 0; + + [[nodiscard]] bool IsDefined () const { return m_is_defined; } + + virtual void PrintParameters () const = 0; + + void GetParticleSolverParams (int& a_max_particle_iter, + amrex::ParticleReal& a_particle_tol ) const + { + a_max_particle_iter = m_max_particle_iterations; + a_particle_tol = m_particle_tolerance; + } + + /** + * \brief Advance fields and particles by one time step using the specified implicit algorithm + */ + virtual void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) = 0; + + // + // the following routines are called by the linear and nonlinear solvers + // + + /** + * \brief Computes the RHS of the equation corresponding to the specified implicit algorithm. + * The discrete equations corresponding to numerical integration of ODEs are often + * written in the form U = b + RHS(U), where U is the variable to be solved for (e.g., + * the solution at the next time step), b is a constant (i.e., the solution from the + * previous time step), and RHS(U) is the right-hand-side of the equation. Iterative + * solvers, such as Picard and Newton, and higher-order Runge-Kutta methods, need to + * compute RHS(U) multiple times for different values of U. Thus, a routine that + * returns this value is needed. + * e.g., Ebar - E^n = cvac^2*0.5*dt*(curl(Bbar) - mu0*Jbar(Ebar,Bbar)) + * Here, U = Ebar, b = E^n, and the expression on the right is RHS(U). + */ + virtual void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) = 0; + +protected: + + /** + * \brief Pointer back to main WarpX class + */ + WarpX* m_WarpX; + + bool m_is_defined = false; + + /** + * \brief Nonlinear solver type and object + */ + NonlinearSolverType m_nlsolver_type; + std::unique_ptr> m_nlsolver; + + /** + * \brief tolerance used by the iterative method used to obtain a self-consistent + * update of the particle positions and velocities for given E and B on the grid + */ + amrex::ParticleReal m_particle_tolerance = 1.0e-10; + + /** + * \brief maximum iterations for the iterative method used to obtain a self-consistent + * update of the particle positions and velocities for given E and B on the grid + */ + int m_max_particle_iterations = 21; + + /** + * \brief parse nonlinear solver parameters (if one is used) + */ + void parseNonlinearSolverParams( const amrex::ParmParse& pp ) + { + + std::string nlsolver_type_str; + pp.get("nonlinear_solver", nlsolver_type_str); + + if (nlsolver_type_str=="picard") { + m_nlsolver_type = NonlinearSolverType::Picard; + m_nlsolver = std::make_unique>(); + m_max_particle_iterations = 1; + m_particle_tolerance = 0.0; + } + else if (nlsolver_type_str=="newton") { + m_nlsolver_type = NonlinearSolverType::Newton; + m_nlsolver = std::make_unique>(); + pp.query("max_particle_iterations", m_max_particle_iterations); + pp.query("particle_tolerance", m_particle_tolerance); + } + else { + WARPX_ABORT_WITH_MESSAGE( + "invalid nonlinear_solver specified. Valid options are picard and newton."); + } + + } + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H new file mode 100644 index 00000000000..423957ef061 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H @@ -0,0 +1,13 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef IMPLICIT_SOLVER_LIBRARY_H_ +#define IMPLICIT_SOLVER_LIBRARY_H_ + +#include "SemiImplicitEM.H" // IWYU pragma: export +#include "ThetaImplicitEM.H" // IWYU pragma: export + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/Make.package b/Source/FieldSolver/ImplicitSolvers/Make.package new file mode 100644 index 00000000000..a4543f94dd3 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/Make.package @@ -0,0 +1,6 @@ +CEXE_sources += SemiImplicitEM.cpp +CEXE_sources += ThetaImplicitEM.cpp +CEXE_sources += WarpXImplicitOps.cpp +CEXE_sources += WarpXSolverVec.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver/ImplicitSolvers diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H new file mode 100644 index 00000000000..6e3e5db2c74 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H @@ -0,0 +1,80 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef SEMI_IMPLICIT_EM_H_ +#define SEMI_IMPLICIT_EM_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +#include +#include +#include + +#include "ImplicitSolver.H" + +/** @file + * Semi-implicit electromagnetic time solver class. The electric field and the + * particles are implicitly coupled in this algorithm, but the magnetic field + * is advanced in the standard explicit leap-frog manner (hence semi-implicit). + * + * The time stencil is + * Eg^{n+1} = Eg^n + c^2*dt*( curlBg^{n+1/2} - mu0*Jg^{n+1/2} ) + * Bg^{n+3/2} = Bg^{n+1/2} - dt*curlEg^{n+1} + * xp^{n+1} = xp^n + dt*up^{n+1/2}/(0.5(gammap^n + gammap^{n+1})) + * up^{n+1} = up^n + dt*qp/mp*(Ep^{n+1/2} + up^{n+1/2}/gammap^{n+1/2} x Bp^{n+1/2}) + * where f^{n+1/2} = (f^{n+1} + f^n)/2.0, for all but Bg, which lives at half steps + * + * This algorithm is approximately energy conserving. It is exactly energy conserving + * using a non-standard definition for the magnetic field energy. The advantage of + * this method over the exactly energy-conserving theta-implicit EM method is that + * light wave dispersion is captured much better. However, the CFL condition for light + * waves has to be satisifed for numerical stability (and for the modified definition + * of the magnetic field energy to be well-posed). + * + * See G. Chen, L. Chacon, L. Yin, B.J. Albright, D.J. Stark, R.F. Bird, + * "A semi-implicit energy- and charge-conserving particle-in-cell algorithm for the + * relativistic Vlasov-Maxwell equations.", JCP 407 (2020). + */ + +class SemiImplicitEM : public ImplicitSolver +{ +public: + + SemiImplicitEM() = default; + + ~SemiImplicitEM() override = default; + + // Prohibit Move and Copy operations + SemiImplicitEM(const SemiImplicitEM&) = delete; + SemiImplicitEM& operator=(const SemiImplicitEM&) = delete; + SemiImplicitEM(SemiImplicitEM&&) = delete; + SemiImplicitEM& operator=(SemiImplicitEM&&) = delete; + + void Define ( WarpX* a_WarpX ) override; + + void PrintParameters () const override; + + void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) override; + + void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) override; + +private: + + /** + * \brief Solver vectors for E and Eold + */ + WarpXSolverVec m_E, m_Eold; + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp new file mode 100644 index 00000000000..897bd5c07f3 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp @@ -0,0 +1,117 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "SemiImplicitEM.H" +#include "WarpX.H" + +using namespace warpx::fields; +using namespace amrex::literals; + +void SemiImplicitEM::Define ( WarpX* a_WarpX ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !m_is_defined, + "SemiImplicitEM object is already defined!"); + + // Retain a pointer back to main WarpX class + m_WarpX = a_WarpX; + + // Define E and Eold vectors + m_E.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_Eold.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + + // Need to define the WarpXSolverVec owned dot_mask to do dot + // product correctly for linear and nonlinear solvers + const amrex::Vector& Geom = m_WarpX->Geom(); + m_E.SetDotMask(Geom); + + // Parse implicit solver parameters + const amrex::ParmParse pp("implicit_evolve"); + parseNonlinearSolverParams( pp ); + + // Define the nonlinear solver + m_nlsolver->Define(m_E, this); + m_is_defined = true; + +} + +void SemiImplicitEM::PrintParameters () const +{ + if (!m_WarpX->Verbose()) { return; } + amrex::Print() << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "----------- SEMI IMPLICIT EM SOLVER PARAMETERS ------------" << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << std::endl; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << std::endl; + if (m_nlsolver_type==NonlinearSolverType::Picard) { + amrex::Print() << "Nonlinear solver type: Picard" << std::endl; + } + else if (m_nlsolver_type==NonlinearSolverType::Newton) { + amrex::Print() << "Nonlinear solver type: Newton" << std::endl; + } + m_nlsolver->PrintParams(); + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << std::endl; +} + +void SemiImplicitEM::OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) +{ + amrex::ignore_unused(a_step); + + // Fields have E^{n}, B^{n-1/2} + // Particles have p^{n} and x^{n}. + + // Save the values at the start of the time step, + m_WarpX->SaveParticlesAtImplicitStepStart ( ); + + // Save the fields at the start of the step + m_Eold.Copy( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_E.Copy(m_Eold); // initial guess for E + + // Compute Bfield at time n+1/2 + m_WarpX->EvolveB(a_dt, DtType::Full); + m_WarpX->ApplyMagneticFieldBCs(); + + const amrex::Real half_time = a_time + 0.5_rt*a_dt; + + // Solve nonlinear system for E at t_{n+1/2} + // Particles will be advanced to t_{n+1/2} + m_nlsolver->Solve( m_E, m_Eold, half_time, a_dt ); + + // Update WarpX owned Efield_fp to t_{n+1/2} + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + + // Advance particles from time n+1/2 to time n+1 + m_WarpX->FinishImplicitParticleUpdate(); + + // Advance E fields from time n+1/2 to time n+1 + // Eg^{n+1} = 2.0*E_g^{n+1/2} - E_g^n + m_E.linComb( 2._rt, m_E, -1._rt, m_Eold ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + +} + +void SemiImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + // update WarpX-owned Efield_fp using current state of E from + // the nonlinear solver at time n+theta + m_WarpX->SetElectricFieldAndApplyBCs( a_E ); + + // Self consistently update particle positions and velocities using the + // current state of the fields E and B. Deposit current density at time n+1/2. + m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + + // RHS = cvac^2*0.5*dt*( curl(B^{n+1/2}) - mu0*J^{n+1/2} ) + m_WarpX->ImplicitComputeRHSE(0.5_rt*a_dt, a_RHS); +} diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H new file mode 100644 index 00000000000..009c2c7e546 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H @@ -0,0 +1,124 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef THETA_IMPLICIT_EM_H_ +#define THETA_IMPLICIT_EM_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +#include +#include +#include + +#include "ImplicitSolver.H" + +/** @file + * Theta-implicit electromagnetic time solver class. This is a fully implicit + * algorithm where both the fields and particles are treated implicitly. + * + * The time stencil is + * Eg^{n+1} = Eg^n + c^2*dt*( curlBg^{n+theta} - mu0*Jg^{n+1/2} ) + * Bg^{n+1} = Bg^n - dt*curlEg^{n+theta} + * xp^{n+1} = xp^n + dt*up^{n+1/2}/(0.5*(gammap^n + gammap^{n+1})) + * up^{n+1} = up^n + dt*qp/mp*(Ep^{n+theta} + up^{n+1/2}/gammap^{n+1/2} x Bp^{n+theta}) + * where f^{n+theta} = (1.0-theta)*f^{n} + theta*f^{n+1} with 0.5 <= theta <= 1.0 + * + * The user-specified time-biasing parameter theta used for the fields on the RHS is bound + * between 0.5 and 1.0. The algorithm is exactly energy conserving for theta = 0.5. + * Signifcant damping of high-k modes will occur as theta approaches 1.0. The algorithm is + * numerially stable for any time step. I.e., the CFL condition for light waves does not + * have to be satisifed and the time step is not limited by the plasma period. However, how + * efficiently the algorithm can use large time steps depends strongly on the nonlinear solver. + * Furthermore, the time step should always be such that particles do not travel outside the + * ghost region of the box they live in, which is an MPI-related limitation. The time step + * is always limited by the need to resolve the appropriate physics. + * + * See S. Markidis, G. Lapenta, "The energy conserving particle-in-cell method." JCP 230 (2011). + * + * See G. Chen, L. Chacon, D.C. Barnes, "An energy- and charge-conserving, implicit, + * elctrostatic particle-in-cell algorithm." JCP 230 (2011). + * + * See J.R. Angus, A. Link, A. Friedman, D. Ghosh, J. D. Johnson, "On numerical energy + * conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo + * algorithm for Coulomb collisions.", JCP 456 (2022). + * + * See J.R. Angus, W. Farmer, A. Friedman, D. Ghosh, D. Grote, D. Larson, A. Link, "An + * implicit particle code with exact energy and charge conservation for electromagnetic studies + * of dense plasmas.", JCP 491 (2023). + */ + +class ThetaImplicitEM : public ImplicitSolver +{ +public: + + ThetaImplicitEM() = default; + + ~ThetaImplicitEM() override = default; + + // Prohibit Move and Copy operations + ThetaImplicitEM(const ThetaImplicitEM&) = delete; + ThetaImplicitEM& operator=(const ThetaImplicitEM&) = delete; + ThetaImplicitEM(ThetaImplicitEM&&) = delete; + ThetaImplicitEM& operator=(ThetaImplicitEM&&) = delete; + + void Define ( WarpX* a_WarpX ) override; + + void PrintParameters () const override; + + void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) override; + + void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) override; + + [[nodiscard]] amrex::Real theta () const { return m_theta; } + +private: + + /** + * \brief Time-biasing parameter for fields used on RHS to advance system + */ + amrex::Real m_theta = 0.5; + + /** + * \brief Solver vectors to be used in the nonlinear solver to solve for the + * electric field E. The main logic for determining which variables should be + * WarpXSolverVec type is that it must have the same size and have the same + * centering of the data as the variable being solved for, which is E here. + * For example, if using a Yee grid then a container for curlB could be a + * WarpXSovlerVec, but magnetic field B should not be. + */ + WarpXSolverVec m_E, m_Eold; + + /** + * \brief B is a derived variable from E. Need to save Bold to update B during + * the iterative nonlinear solve for E. Bold is owned here, but only used by WarpX. + * It is not used directly by the nonlinear solver, nor is it the same size as the + * solver vector (size E), and so it should not be WarpXSolverVec type. + */ + amrex::Vector, 3 > > m_Bold; + + /** + * \brief Update the E and B fields owned by WarpX + */ + void UpdateWarpXFields ( const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt ); + + /** + * \brief Nonlinear solver is for the time-centered values of E. After + * the solver, need to use m_E and m_Eold to compute E^{n+1} + */ + void FinishFieldUpdate ( amrex::Real a_new_time ); + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp new file mode 100644 index 00000000000..026c509c3ba --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -0,0 +1,171 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "FieldSolver/Fields.H" +#include "ThetaImplicitEM.H" +#include "WarpX.H" + +using namespace warpx::fields; +using namespace amrex::literals; + +void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !m_is_defined, + "ThetaImplicitEM object is already defined!"); + + // Retain a pointer back to main WarpX class + m_WarpX = a_WarpX; + + // Define E and Eold vectors + m_E.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_Eold.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + + // Need to define the WarpXSolverVec owned dot_mask to do dot + // product correctly for linear and nonlinear solvers + const amrex::Vector& Geom = m_WarpX->Geom(); + m_E.SetDotMask(Geom); + + // Define Bold MultiFab + const int num_levels = 1; + m_Bold.resize(num_levels); // size is number of levels + for (int lev = 0; lev < num_levels; ++lev) { + for (int n=0; n<3; n++) { + const amrex::MultiFab& Bfp = m_WarpX->getField( FieldType::Bfield_fp,lev,n); + m_Bold[lev][n] = std::make_unique( Bfp.boxArray(), + Bfp.DistributionMap(), + Bfp.nComp(), + Bfp.nGrowVect() ); + } + } + + // Parse theta-implicit solver specific parameters + const amrex::ParmParse pp("implicit_evolve"); + pp.query("theta", m_theta); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_theta>=0.5 && m_theta<=1.0, + "theta parameter for theta implicit time solver must be between 0.5 and 1.0"); + + // Parse nonlinear solver parameters + parseNonlinearSolverParams( pp ); + + // Define the nonlinear solver + m_nlsolver->Define(m_E, this); + m_is_defined = true; + +} + +void ThetaImplicitEM::PrintParameters () const +{ + if (!m_WarpX->Verbose()) { return; } + amrex::Print() << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "----------- THETA IMPLICIT EM SOLVER PARAMETERS -----------" << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "Time-bias parameter theta: " << m_theta << std::endl; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << std::endl; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << std::endl; + if (m_nlsolver_type==NonlinearSolverType::Picard) { + amrex::Print() << "Nonlinear solver type: Picard" << std::endl; + } + else if (m_nlsolver_type==NonlinearSolverType::Newton) { + amrex::Print() << "Nonlinear solver type: Newton" << std::endl; + } + m_nlsolver->PrintParams(); + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << std::endl; +} + +void ThetaImplicitEM::OneStep ( const amrex::Real a_time, + const amrex::Real a_dt, + const int a_step ) +{ + amrex::ignore_unused(a_step); + + // Fields have E^{n} and B^{n} + // Particles have p^{n} and x^{n}. + + // Save the values at the start of the time step, + m_WarpX->SaveParticlesAtImplicitStepStart ( ); + + // Save the fields at the start of the step + m_Eold.Copy( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_E.Copy(m_Eold); // initial guess for E + + const int num_levels = static_cast(m_Bold.size()); + for (int lev = 0; lev < num_levels; ++lev) { + for (int n=0; n<3; n++) { + const amrex::MultiFab& Bfp = m_WarpX->getField(FieldType::Bfield_fp,lev,n); + amrex::MultiFab& Bold = *m_Bold[lev][n]; + amrex::MultiFab::Copy(Bold, Bfp, 0, 0, 1, Bold.nGrowVect()); + } + } + + const amrex::Real theta_time = a_time + m_theta*a_dt; + + // Solve nonlinear system for E at t_{n+theta} + // Particles will be advanced to t_{n+1/2} + m_nlsolver->Solve( m_E, m_Eold, theta_time, a_dt ); + + // Update WarpX owned Efield_fp and Bfield_fp to t_{n+theta} + UpdateWarpXFields( m_E, theta_time, a_dt ); + + // Advance particles from time n+1/2 to time n+1 + m_WarpX->FinishImplicitParticleUpdate(); + + // Advance E and B fields from time n+theta to time n+1 + const amrex::Real new_time = a_time + a_dt; + FinishFieldUpdate( new_time ); + +} + +void ThetaImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + // update WarpX-owned Efield_fp and Bfield_fp using current state of E from + // the nonlinear solver at time n+theta + UpdateWarpXFields( a_E, a_time, a_dt ); + + // Self consistently update particle positions and velocities using the + // current state of the fields E and B. Deposit current density at time n+1/2. + m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + + // RHS = cvac^2*m_theta*dt*( curl(B^{n+theta}) - mu0*J^{n+1/2} ) + m_WarpX->ImplicitComputeRHSE(m_theta*a_dt, a_RHS); +} + +void ThetaImplicitEM::UpdateWarpXFields ( const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt ) +{ + amrex::ignore_unused(a_time); + + // Update Efield_fp owned by WarpX + m_WarpX->SetElectricFieldAndApplyBCs( a_E ); + + // Update Bfield_fp owned by WarpX + m_WarpX->UpdateMagneticFieldAndApplyBCs( m_Bold, m_theta*a_dt ); + +} + +void ThetaImplicitEM::FinishFieldUpdate ( amrex::Real a_new_time ) +{ + amrex::ignore_unused(a_new_time); + + // Eg^{n+1} = (1/theta)*E_g^{n+theta} + (1-1/theta)*E_g^n + // Bg^{n+1} = (1/theta)*B_g^{n+theta} + (1-1/theta)*B_g^n + + const amrex::Real c0 = 1._rt/m_theta; + const amrex::Real c1 = 1._rt - c0; + m_E.linComb( c0, m_E, c1, m_Eold ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + m_WarpX->FinishMagneticFieldAndApplyBCs( m_Bold, m_theta ); + +} diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp new file mode 100644 index 00000000000..7b95abade3e --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -0,0 +1,352 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "WarpX.H" + +#include "BoundaryConditions/PML.H" +#include "Diagnostics/MultiDiagnostics.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" +#include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" +#ifdef WARPX_USE_PSATD +# ifdef WARPX_DIM_RZ +# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" +# else +# include "FieldSolver/SpectralSolver/SpectralSolver.H" +# endif +#endif +#include "Parallelization/GuardCellManager.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleBoundaryBuffer.H" +#include "Python/callbacks.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" +#include "Utils/WarpXUtil.H" +#include "Utils/WarpXConst.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +void +WarpX::ImplicitPreRHSOp ( amrex::Real a_cur_time, + amrex::Real a_full_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + using namespace amrex::literals; + amrex::ignore_unused( a_full_dt, a_nl_iter, a_from_jacobian ); + + // Advance the particle positions by 1/2 dt, + // particle velocities by dt, then take average of old and new v, + // deposit currents, giving J at n+1/2 + // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration. + const bool skip_current = false; + const PushType push_type = PushType::Implicit; + PushParticlesandDeposit(a_cur_time, skip_current, push_type); + + SyncCurrentAndRho(); + +} + +void +WarpX::SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ) +{ + const amrex::Vector, 3 > >& Evec = a_E.getVec(); + amrex::MultiFab::Copy(*Efield_fp[0][0], *Evec[0][0], 0, 0, ncomps, Evec[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][1], *Evec[0][1], 0, 0, ncomps, Evec[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][2], *Evec[0][2], 0, 0, ncomps, Evec[0][2]->nGrowVect()); + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyEfieldBoundary(0, PatchType::fine); +} + +void +WarpX::UpdateMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_thetadt ) +{ + amrex::MultiFab::Copy(*Bfield_fp[0][0], *a_Bn[0][0], 0, 0, ncomps, a_Bn[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][1], *a_Bn[0][1], 0, 0, ncomps, a_Bn[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][2], *a_Bn[0][2], 0, 0, ncomps, a_Bn[0][2]->nGrowVect()); + EvolveB(a_thetadt, DtType::Full); + ApplyMagneticFieldBCs(); +} + +void +WarpX::FinishMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_theta ) +{ + FinishImplicitField(Bfield_fp, a_Bn, a_theta); + ApplyMagneticFieldBCs(); +} + +void +WarpX::ApplyMagneticFieldBCs() +{ + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); +} + +void +WarpX::SaveParticlesAtImplicitStepStart ( ) +{ + // The implicit advance routines require the particle velocity + // and position values at the beginning of the step to compute the + // time-centered position and velocity needed for the implicit stencil. + // Thus, we need to save this information. + + for (auto const& pc : *mypc) { + + for (int lev = 0; lev <= finest_level; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc->getParticleComps(); + + for (WarpXParIter pti(*pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + x_n[ip] = xp; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + y_n[ip] = yp; +#endif + z_n[ip] = zp; + + ux_n[ip] = ux[ip]; + uy_n[ip] = uy[ip]; + uz_n[ip] = uz[ip]; + + }); + + } + } + + } + + } + +} + +void +WarpX::FinishImplicitParticleUpdate () +{ + using namespace amrex::literals; + + // The implicit advance routines use the time-centered position and + // momentum to advance the system in time. Thus, at the end of the + // step we need to transform the particle postion and momentum from + // time n+1/2 to time n+1. This is done here. + + for (auto const& pc : *mypc) { + + for (int lev = 0; lev <= finest_level; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc->getParticleComps(); + + for (WarpXParIter pti(*pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + const auto setPosition = SetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + xp = 2._rt*xp - x_n[ip]; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + yp = 2._rt*yp - y_n[ip]; +#endif + zp = 2._rt*zp - z_n[ip]; + + ux[ip] = 2._rt*ux[ip] - ux_n[ip]; + uy[ip] = 2._rt*uy[ip] - uy_n[ip]; + uz[ip] = 2._rt*uz[ip] - uz_n[ip]; + + setPosition(ip, xp, yp, zp); + }); + + } + } + + } + + } + +} + +void +WarpX::FinishImplicitField( amrex::Vector, 3 > >& Field_fp, + const amrex::Vector, 3 > >& Field_n, + amrex::Real theta ) +{ + using namespace amrex::literals; + + // The implicit field advance routines use the fields at time n+theta + // with 0.5 <= theta <= 1.0. Thus, at the end of the step we need to + // transform the fields from time n+theta to time n+1. This is done here. + + for (int lev = 0; lev <= finest_level; ++lev) { + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*Field_fp[lev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + + amrex::Array4 const& Fx = Field_fp[lev][0]->array(mfi); + amrex::Array4 const& Fy = Field_fp[lev][1]->array(mfi); + amrex::Array4 const& Fz = Field_fp[lev][2]->array(mfi); + + amrex::Array4 const& Fx_n = Field_n[lev][0]->array(mfi); + amrex::Array4 const& Fy_n = Field_n[lev][1]->array(mfi); + amrex::Array4 const& Fz_n = Field_n[lev][2]->array(mfi); + + amrex::Box const& tbx = mfi.tilebox(Field_n[lev][0]->ixType().toIntVect()); + amrex::Box const& tby = mfi.tilebox(Field_n[lev][1]->ixType().toIntVect()); + amrex::Box const& tbz = mfi.tilebox(Field_n[lev][2]->ixType().toIntVect()); + + const amrex::Real c0 = 1._rt/theta; + const amrex::Real c1 = 1._rt - c0; + + amrex::ParallelFor( + tbx, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fx(i,j,k,n) = c0*Fx(i,j,k,n) + c1*Fx_n(i,j,k,n); + }, + tby, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fy(i,j,k,n) = c0*Fy(i,j,k,n) + c1*Fy_n(i,j,k,n); + }, + tbz, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fz(i,j,k,n) = c0*Fz(i,j,k,n) + c1*Fz_n(i,j,k,n); + }); + } + } +} + +void +WarpX::ImplicitComputeRHSE (amrex::Real a_dt, WarpXSolverVec& a_Erhs_vec) +{ + for (int lev = 0; lev <= finest_level; ++lev) + { + ImplicitComputeRHSE(lev, a_dt, a_Erhs_vec); + } +} + +void +WarpX::ImplicitComputeRHSE (int lev, amrex::Real a_dt, WarpXSolverVec& a_Erhs_vec) +{ + WARPX_PROFILE("WarpX::ImplicitComputeRHSE()"); + ImplicitComputeRHSE(lev, PatchType::fine, a_dt, a_Erhs_vec); + if (lev > 0) + { + ImplicitComputeRHSE(lev, PatchType::coarse, a_dt, a_Erhs_vec); + } +} + +void +WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, WarpXSolverVec& a_Erhs_vec) +{ + // set RHS to zero value + a_Erhs_vec.getVec()[lev][0]->setVal(0.0); + a_Erhs_vec.getVec()[lev][1]->setVal(0.0); + a_Erhs_vec.getVec()[lev][2]->setVal(0.0); + + // Compute Efield_rhs in regular cells by calling EvolveE. Because + // a_Erhs_vec is set to zero above, calling EvolveE below results in + // a_Erhs_vec storing only the RHS of the update equation. I.e., + // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) + if (patch_type == PatchType::fine) { + m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getVec()[lev], Bfield_fp[lev], + current_fp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], + F_fp[lev], lev, a_dt ); + } else { + m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getVec()[lev], Bfield_cp[lev], + current_cp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], + F_cp[lev], lev, a_dt ); + } + + // Compute Efield_rhs in PML cells by calling EvolveEPML + if (do_pml && pml[lev]->ok()) { + amrex::Abort("PML not yet implemented with implicit solvers."); + } + +} diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H new file mode 100644 index 00000000000..89a0b82b700 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -0,0 +1,233 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WarpXSolverVec_H_ +#define WarpXSolverVec_H_ + +#include "Utils/TextMsg.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/** + * \brief This is a wrapper class around a Vector of array of pointers to MultiFabs that + * contains basic math operators and functionality needed to interact with nonlinear + * solvers in WarpX and linear solvers in AMReX, such as GMRES. + * + * The size of the Vector is the number of amr levels. Hardcoded for 1 right now. The + * size of the array is the number of MultiFabs. It is hardcoded for 3 right now as it + * is only used for the electric field in the implicit electromagnetic time solvers. In + * the future, the array size can be made a template parameter so that this class can + * be used for other solver vectors, such as electrostatic (array size 1) or Darwin (array size 4). + */ + +class WarpXSolverVec +{ +public: + + WarpXSolverVec() = default; + + WarpXSolverVec(const WarpXSolverVec&) = delete; + + ~WarpXSolverVec() = default; + + using value_type = amrex::Real; + using RT = value_type; + + [[nodiscard]] inline bool IsDefined () const { return m_is_defined; } + + inline + void Define (const WarpXSolverVec& a_vec) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_vec.IsDefined(), + "WarpXSolverVec::Define(a_vec) called with undefined a_vec"); + Define( a_vec.getVec() ); + } + + inline + void Define ( const amrex::Vector, 3 > >& a_solver_vec ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !IsDefined(), + "WarpXSolverVec::Define() called on undefined WarpXSolverVec"); + m_field_vec.resize(m_num_amr_levels); + const int lev = 0; + for (int n=0; n<3; n++) { + const amrex::MultiFab& mf_model = *a_solver_vec[lev][n]; + m_field_vec[lev][n] = std::make_unique( mf_model.boxArray(), mf_model.DistributionMap(), + mf_model.nComp(), amrex::IntVect::TheZeroVector() ); + } + m_is_defined = true; + } + + void SetDotMask( const amrex::Vector& a_Geom ); + [[nodiscard]] RT dotProduct( const WarpXSolverVec& a_X ) const; + + inline + void Copy ( const amrex::Vector, 3 > >& a_solver_vec ) + { + AMREX_ASSERT_WITH_MESSAGE( + IsDefined(), + "WarpXSolverVec::Copy() called on undefined WarpXSolverVec"); + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n = 0; n < 3; ++n) { + amrex::MultiFab::Copy(*m_field_vec[lev][n], *a_solver_vec[lev][n], 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); + } + } + } + + inline + void Copy ( const WarpXSolverVec& a_vec ) + { + AMREX_ASSERT_WITH_MESSAGE( + a_vec.IsDefined(), + "WarpXSolverVec::Copy(a_vec) called with undefined a_vec"); + if (!IsDefined()) { Define(a_vec); } + const amrex::Vector, 3 > >& field_vec = a_vec.getVec(); + Copy(field_vec); + } + + // Prohibit Copy assignment operator + WarpXSolverVec& operator= ( const WarpXSolverVec& a_vec ) = delete; + + // Move assignment operator + WarpXSolverVec(WarpXSolverVec&&) noexcept = default; + WarpXSolverVec& operator= ( WarpXSolverVec&& a_vec ) noexcept + { + if (this != &a_vec) { + m_field_vec = std::move(a_vec.m_field_vec); + m_is_defined = true; + } + return *this; + } + + inline + void operator+= ( const WarpXSolverVec& a_vec ) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->plus(*(a_vec.getVec()[lev][n]), 0, 1, 0); + } + } + } + + inline + void operator-= (const WarpXSolverVec& a_vec) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->minus(*(a_vec.getVec()[lev][n]), 0, 1, 0); + } + } + } + + /** + * \brief Y = a*X + b*Y + */ + inline + void linComb (const RT a, const WarpXSolverVec& X, const RT b, const WarpXSolverVec& Y) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + amrex::MultiFab::LinComb(*m_field_vec[lev][n], a, *X.getVec()[lev][n], 0, + b, *Y.getVec()[lev][n], 0, + 0, 1, 0); + } + } + } + + /** + * \brief Increment Y by a*X (Y += a*X) + */ + void increment (const WarpXSolverVec& X, const RT a) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + amrex::MultiFab::Saxpy( *m_field_vec[lev][n], a, *X.getVec()[lev][n], + 0, 0, 1, amrex::IntVect::TheZeroVector() ); + } + } + } + + /** + * \brief Scale Y by a (Y *= a) + */ + inline + void scale (RT a_a) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->mult(a_a, 0, 1); + } + } + } + + inline + void zero () { setVal(0.0); } + + inline + void setVal ( const RT a_val ) + { + AMREX_ASSERT_WITH_MESSAGE( + IsDefined(), + "WarpXSolverVec::ones() called on undefined WarpXSolverVec"); + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->setVal(a_val); + } + } + } + + [[nodiscard]] inline RT norm2 () const + { + auto const norm = dotProduct(*this); + return std::sqrt(norm); + } + + [[nodiscard]] const amrex::Vector, 3 > >& getVec() const {return m_field_vec;} + amrex::Vector, 3 > >& getVec() {return m_field_vec;} + + // clearDotMask() must be called by the highest class that owns WarpXSolverVec() + // after it is done being used ( typically in the destructor ) to avoid the + // following error message after the simulation finishes: + // malloc_consolidate(): unaligned fastbin chunk detected + static void clearDotMask() { m_dotMask.clear(); } + +private: + + bool m_is_defined = false; + amrex::Vector, 3 > > m_field_vec; + + static constexpr int m_ncomp = 1; + static constexpr int m_num_amr_levels = 1; + + inline static bool m_dot_mask_defined = false; + inline static amrex::Vector,3>> m_dotMask; + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp new file mode 100644 index 00000000000..b181f038fb5 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -0,0 +1,51 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +void WarpXSolverVec::SetDotMask( const amrex::Vector& a_Geom ) +{ + if (m_dot_mask_defined) { return; } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + IsDefined(), + "WarpXSolverVec::SetDotMask() called from undefined instance "); + + m_dotMask.resize(m_num_amr_levels); + for ( int n = 0; n < 3; n++) { + const amrex::BoxArray& grids = m_field_vec[0][n]->boxArray(); + const amrex::MultiFab tmp( grids, m_field_vec[0][n]->DistributionMap(), + 1, 0, amrex::MFInfo().SetAlloc(false) ); + const amrex::Periodicity& period = a_Geom[0].periodicity(); + m_dotMask[0][n] = tmp.OwnerMask(period); + } + m_dot_mask_defined = true; + + // If the function below is not called, then the following + // error message occurs after the simulation finishes: + // malloc_consolidate(): unaligned fastbin chunk detected + amrex::ExecOnFinalize(WarpXSolverVec::clearDotMask); +} + +[[nodiscard]] amrex::Real WarpXSolverVec::dotProduct ( const WarpXSolverVec& a_X ) const +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_dot_mask_defined, + "WarpXSolverVec::dotProduct called with m_dotMask not yet defined"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_X.IsDefined(), + "WarpXSolverVec::dotProduct(a_X) called with undefined a_X"); + amrex::Real result = 0.0; + const int lev = 0; + const bool local = true; + for (int n = 0; n < 3; ++n) { + auto rtmp = amrex::MultiFab::Dot( *m_dotMask[lev][n], + *m_field_vec[lev][n], 0, + *a_X.getVec()[lev][n], 0, 1, 0, local); + result += rtmp; + } + amrex::ParallelAllReduce::Sum(result, amrex::ParallelContext::CommunicatorSub()); + return result; +} diff --git a/Source/FieldSolver/Make.package b/Source/FieldSolver/Make.package index 3b88f4627e5..a8af4c2de97 100644 --- a/Source/FieldSolver/Make.package +++ b/Source/FieldSolver/Make.package @@ -7,5 +7,6 @@ ifeq ($(USE_FFT),TRUE) endif include $(WARPX_HOME)/Source/FieldSolver/FiniteDifferenceSolver/Make.package include $(WARPX_HOME)/Source/FieldSolver/MagnetostaticSolver/Make.package +include $(WARPX_HOME)/Source/FieldSolver/ImplicitSolvers/Make.package VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index bd40c5e0cd4..68b38153f8d 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -501,6 +501,9 @@ WarpX::InitData () CheckGuardCells(); PrintMainPICparameters(); + if (m_implicit_solver) { + m_implicit_solver->PrintParameters(); + } WriteUsedInputsFile(); if (restart_chkfile.empty()) @@ -568,10 +571,33 @@ WarpX::InitFromScratch () AmrCore::InitFromScratch(time); // This will call MakeNewLevelFromScratch + if (m_implicit_solver) { + + m_implicit_solver->Define(this); + m_implicit_solver->GetParticleSolverParams( max_particle_its_in_implicit_scheme, + particle_tol_in_implicit_scheme ); + + // Add space to save the positions and velocities at the start of the time steps + for (auto const& pc : *mypc) { +#if (AMREX_SPACEDIM >= 2) + pc->AddRealComp("x_n"); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + pc->AddRealComp("y_n"); +#endif + pc->AddRealComp("z_n"); + pc->AddRealComp("ux_n"); + pc->AddRealComp("uy_n"); + pc->AddRealComp("uz_n"); + } + + } + mypc->AllocData(); mypc->InitData(); InitPML(); + } void diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 29f047c14e6..519eed52820 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -76,6 +76,7 @@ include $(WARPX_HOME)/Source/BoundaryConditions/Make.package include $(WARPX_HOME)/Source/Diagnostics/Make.package include $(WARPX_HOME)/Source/EmbeddedBoundary/Make.package include $(WARPX_HOME)/Source/FieldSolver/Make.package +include $(WARPX_HOME)/Source/NonlinearSolvers/Make.package include $(WARPX_HOME)/Source/Filter/Make.package include $(WARPX_HOME)/Source/Fluids/Make.package include $(WARPX_HOME)/Source/Initialization/Make.package @@ -90,7 +91,7 @@ include $(AMREX_HOME)/Src/Base/Make.package include $(AMREX_HOME)/Src/Particle/Make.package include $(AMREX_HOME)/Src/Boundary/Make.package include $(AMREX_HOME)/Src/AmrCore/Make.package -include $(AMREX_HOME)/Src/LinearSolvers/MLMG/Make.package +include $(AMREX_HOME)/Src/LinearSolvers/Make.package ifeq ($(USE_SENSEI_INSITU),TRUE) include $(AMREX_HOME)/Src/Amr/Make.package diff --git a/Source/NonlinearSolvers/CMakeLists.txt b/Source/NonlinearSolvers/CMakeLists.txt new file mode 100644 index 00000000000..938cdaad547 --- /dev/null +++ b/Source/NonlinearSolvers/CMakeLists.txt @@ -0,0 +1,6 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + ) +endforeach() diff --git a/Source/NonlinearSolvers/JacobianFunctionMF.H b/Source/NonlinearSolvers/JacobianFunctionMF.H new file mode 100644 index 00000000000..823523df23c --- /dev/null +++ b/Source/NonlinearSolvers/JacobianFunctionMF.H @@ -0,0 +1,234 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef JacobianFunctionMF_H_ +#define JacobianFunctionMF_H_ + +/** + * \brief This is a linear function class for computing the action of a + * Jacobian on a vector using a matrix-free finite-difference method. + * This class has all of the required functions to be used as the + * linear operator template parameter in AMReX_GMRES. + */ + +template +class JacobianFunctionMF +{ + public: + + using RT = typename T::value_type; + + JacobianFunctionMF() = default; + ~JacobianFunctionMF() = default; + + // Default move and copy operations + JacobianFunctionMF(const JacobianFunctionMF&) = default; + JacobianFunctionMF& operator=(const JacobianFunctionMF&) = default; + JacobianFunctionMF(JacobianFunctionMF&&) noexcept = default; + JacobianFunctionMF& operator=(JacobianFunctionMF&&) noexcept = default; + + void apply ( T& a_dF, const T& a_dU ); + + inline + void precond ( T& a_U, const T& a_X ) + { + if (m_usePreCond) { a_U.zero(); } + else { a_U.Copy(a_X); } + } + + inline + void updatePreCondMat ( const T& a_X ) + { + amrex::ignore_unused(a_X); + } + + inline + void create ( T& a_Z, const T& a_U ) + { + a_Z.define(a_U); + } + + T makeVecLHS () const; + T makeVecRHS () const; + + inline + void assign( T& a_Z, const T& a_U ) { + a_Z.Copy(a_U); + } + + inline + void increment( T& a_Z, const T& a_U, RT a_scale ) + { + a_Z.increment(a_U,a_scale); + } + + inline + void scale ( T& a_U, RT a_scale ) + { + a_U.scale(a_scale); + } + + inline + void linComb ( T& a_U, RT a, const T& X, RT b, const T& Y ) + { + a_U.linComb( a, X, b, Y ); + } + + inline + void setToZero ( T& a_U ) + { + a_U.zero(); + } + + inline + void setVal ( T& a_U, RT a_val ) + { + a_U.setVal(a_val); + } + + inline + RT dotProduct( const T& a_X, const T& a_Y ) + { + return( a_X.dotProduct(a_Y) ); + } + + inline + RT norm2( const T& a_U ) + { + return ( a_U.norm2() ); + } + + [[nodiscard]] inline + bool isDefined() const { return m_is_defined; } + + inline + void setBaseSolution ( const T& a_U ) + { + m_Y0.Copy(a_U); + m_normY0 = norm2(m_Y0); + } + + inline + void setBaseRHS ( const T& a_R ) + { + m_R0.Copy(a_R); + } + + inline + void setJFNKEps ( RT a_eps ) + { + m_epsJFNK = a_eps; + } + + inline + void setIsLinear ( bool a_isLinear ) + { + m_is_linear = a_isLinear; + } + + inline + void curTime ( RT a_time ) + { + m_cur_time = a_time; + } + + inline + void curTimeStep ( RT a_dt ) + { + m_dt = a_dt; + } + + void define( const T&, Ops* ); + + private: + + bool m_is_defined = false; + bool m_is_linear = false; + bool m_usePreCond = false; + RT m_epsJFNK = RT(1.0e-6); + RT m_normY0; + RT m_cur_time, m_dt; + std::string m_pc_type; + + T m_Z, m_Y0, m_R0, m_R; + Ops* m_ops; + +}; + +template +void JacobianFunctionMF::define ( const T& a_U, + Ops* a_ops ) +{ + m_Z.Define(a_U); + m_Y0.Define(a_U); + m_R0.Define(a_U); + m_R.Define(a_U); + + m_ops = a_ops; + + m_is_defined = true; +} + +template +auto JacobianFunctionMF::makeVecRHS () const -> T +{ + T Vec; + Vec.Define(m_R); + return Vec; +} + +template +auto JacobianFunctionMF::makeVecLHS () const -> T +{ + T Vec; + Vec.Define(m_R); + return Vec; +} + +template +void JacobianFunctionMF::apply (T& a_dF, const T& a_dU) +{ + BL_PROFILE("JacobianFunctionMF::apply()"); + using namespace amrex::literals; + RT const normY = norm2(a_dU); // always 1 when called from GMRES + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + isDefined(), + "JacobianFunction::apply() called on undefined JacobianFunction"); + + if (normY < 1.0e-15) { a_dF.zero(); } + else { + + RT eps; + if (m_is_linear) { + eps = 1.0_rt; + } else { + /* eps = error_rel * sqrt(1 + ||Y0||) / ||dU|| + * M. Pernice and H. F. Walker, "NITSOL: A Newton Iterative Solver for + * Nonlinear Systems", SIAM J. Sci. Stat. Comput., 1998, vol 19, + * pp. 302--318. */ + if (m_normY0==0.0) { eps = m_epsJFNK * norm2(m_R0) / normY; } + else { + // m_epsJFNK * sqrt(1.0 + m_normY0) / normY + // above commonly used form not recommend for poorly scaled Y0 + eps = m_epsJFNK * m_normY0 / normY; + } + } + const RT eps_inv = 1.0_rt/eps; + + m_Z.linComb( 1.0, m_Y0, eps, a_dU ); // Z = Y0 + eps*dU + m_ops->ComputeRHS(m_R, m_Z, m_cur_time, m_dt, -1, true ); + + // F(Y) = Y - b - R(Y) ==> dF = dF/dY*dU = [1 - dR/dY]*dU + // = dU - (R(Z)-R(Y0))/eps + a_dF.linComb( 1.0, a_dU, eps_inv, m_R0 ); + a_dF.increment(m_R,-eps_inv); + + } + +} + +#endif diff --git a/Source/NonlinearSolvers/Make.package b/Source/NonlinearSolvers/Make.package new file mode 100644 index 00000000000..4f2dcfa9617 --- /dev/null +++ b/Source/NonlinearSolvers/Make.package @@ -0,0 +1 @@ +VPATH_LOCATIONS += $(WARPX_HOME)/Source/NonlinearSolvers diff --git a/Source/NonlinearSolvers/NewtonSolver.H b/Source/NonlinearSolvers/NewtonSolver.H new file mode 100644 index 00000000000..93ad432208a --- /dev/null +++ b/Source/NonlinearSolvers/NewtonSolver.H @@ -0,0 +1,345 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef NEWTON_SOLVER_H_ +#define NEWTON_SOLVER_H_ + +#include "NonlinearSolver.H" +#include "JacobianFunctionMF.H" + +#include +#include +#include "Utils/TextMsg.H" + +#include + +/** + * \brief Newton method to solve nonlinear equation of form: + * F(U) = U - b - R(U) = 0. U is the solution vector, b is a constant, + * and R(U) is some nonlinear function of U, which is computed in the + * ComputeRHS() Ops function. + */ + +template +class NewtonSolver : public NonlinearSolver +{ +public: + + NewtonSolver() = default; + + ~NewtonSolver() override = default; + + // Prohibit Move and Copy operations + NewtonSolver(const NewtonSolver&) = delete; + NewtonSolver& operator=(const NewtonSolver&) = delete; + NewtonSolver(NewtonSolver&&) noexcept = delete; + NewtonSolver& operator=(NewtonSolver&&) noexcept = delete; + + void Define ( const Vec& a_U, + Ops* a_ops ) override; + + void Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const override; + + void GetSolverParams ( amrex::Real& a_rtol, + amrex::Real& a_atol, + int& a_maxits ) override + { + a_rtol = m_rtol; + a_atol = m_atol; + a_maxits = m_maxits; + } + + inline void CurTime ( amrex::Real a_time ) const + { + m_cur_time = a_time; + m_linear_function->curTime( a_time ); + } + + inline void CurTimeStep ( amrex::Real a_dt ) const + { + m_dt = a_dt; + m_linear_function->curTimeStep( a_dt ); + } + + void PrintParams () const override + { + amrex::Print() << "Newton verbose: " << (this->m_verbose?"true":"false") << std::endl; + amrex::Print() << "Newton max iterations: " << m_maxits << std::endl; + amrex::Print() << "Newton relative tolerance: " << m_rtol << std::endl; + amrex::Print() << "Newton absolute tolerance: " << m_atol << std::endl; + amrex::Print() << "Newton require convergence: " << (m_require_convergence?"true":"false") << std::endl; + amrex::Print() << "GMRES verbose: " << m_gmres_verbose_int << std::endl; + amrex::Print() << "GMRES restart length: " << m_gmres_restart_length << std::endl; + amrex::Print() << "GMRES max iterations: " << m_gmres_maxits << std::endl; + amrex::Print() << "GMRES relative tolerance: " << m_gmres_rtol << std::endl; + amrex::Print() << "GMRES absolute tolerance: " << m_gmres_atol << std::endl; + } + +private: + + /** + * \brief Intermediate Vec containers used by the solver. + */ + mutable Vec m_dU, m_F, m_R; + + /** + * \brief Pointer to Ops class. + */ + Ops* m_ops = nullptr; + + /** + * \brief Flag to determine whether convergence is required. + */ + bool m_require_convergence = true; + + /** + * \brief Relative tolerance for the Newton solver. + */ + amrex::Real m_rtol = 1.0e-6; + + /** + * \brief Absolute tolerance for the Newton solver. + */ + amrex::Real m_atol = 0.; + + /** + * \brief Maximum iterations for the Newton solver. + */ + int m_maxits = 100; + + /** + * \brief Relative tolerance for GMRES. + */ + amrex::Real m_gmres_rtol = 1.0e-4; + + /** + * \brief Absolute tolerance for GMRES. + */ + amrex::Real m_gmres_atol = 0.; + + /** + * \brief Maximum iterations for GMRES. + */ + int m_gmres_maxits = 1000; + + /** + * \brief Verbosity flag for GMRES. + */ + int m_gmres_verbose_int = 2; + + /** + * \brief Restart iteration for GMRES. + */ + int m_gmres_restart_length = 30; + + mutable amrex::Real m_cur_time, m_dt; + mutable bool m_update_pc = false; + mutable bool m_update_pc_init = false; + + /** + * \brief The linear function used by GMRES to compute A*v. + * In the contect of JFNK, A = dF/dU (i.e., system Jacobian) + */ + std::unique_ptr> m_linear_function; + + /** + * \brief The linear solver (GMRES) object. + */ + std::unique_ptr>> m_linear_solver; + + void ParseParameters (); + + /** + * \brief Compute the nonlinear residual: F(U) = U - b - R(U). + */ + void EvalResidual ( Vec& a_F, + const Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt, + int a_iter ) const; + +}; + +template +void NewtonSolver::Define ( const Vec& a_U, + Ops* a_ops ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !this->m_is_defined, + "Newton nonlinear solver object is already defined!"); + + ParseParameters(); + + m_dU.Define(a_U); + m_F.Define(a_U); // residual function F(U) = U - b - R(U) = 0 + m_R.Define(a_U); // right hand side function R(U) + + m_ops = a_ops; + + m_linear_function = std::make_unique>(); + m_linear_function->define(m_F, m_ops); + + m_linear_solver = std::make_unique>>(); + m_linear_solver->define(*m_linear_function); + m_linear_solver->setVerbose( m_gmres_verbose_int ); + m_linear_solver->setRestartLength( m_gmres_restart_length ); + m_linear_solver->setMaxIters( m_gmres_maxits ); + + this->m_is_defined = true; + +} + +template +void NewtonSolver::ParseParameters () +{ + const amrex::ParmParse pp_newton("newton"); + pp_newton.query("verbose", this->m_verbose); + pp_newton.query("absolute_tolerance", m_atol); + pp_newton.query("relative_tolerance", m_rtol); + pp_newton.query("max_iterations", m_maxits); + pp_newton.query("require_convergence", m_require_convergence); + + const amrex::ParmParse pp_gmres("gmres"); + pp_gmres.query("verbose_int", m_gmres_verbose_int); + pp_gmres.query("restart_length", m_gmres_restart_length); + pp_gmres.query("absolute_tolerance", m_gmres_atol); + pp_gmres.query("relative_tolerance", m_gmres_rtol); + pp_gmres.query("max_iterations", m_gmres_maxits); +} + +template +void NewtonSolver::Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const +{ + BL_PROFILE("NewtonSolver::Solve()"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + this->m_is_defined, + "NewtonSolver::Solve() called on undefined object"); + using namespace amrex::literals; + + // + // Newton routine to solve nonlinear equation of form: + // F(U) = U - b - R(U) = 0 + // + + CurTime(a_time); + CurTimeStep(a_dt); + + amrex::Real norm_abs = 0.; + amrex::Real norm0 = 1._rt; + amrex::Real norm_rel = 0.; + + int iter; + for (iter = 0; iter < m_maxits;) { + + // Compute residual: F(U) = U - b - R(U) + EvalResidual(m_F, a_U, a_b, a_time, a_dt, iter); + + // Compute norm of the residual + norm_abs = m_F.norm2(); + if (iter == 0) { + if (norm_abs > 0.) { norm0 = norm_abs; } + else { norm0 = 1._rt; } + } + norm_rel = norm_abs/norm0; + + // Check for convergence criteria + if (this->m_verbose || iter == m_maxits) { + amrex::Print() << "Newton: iteration = " << std::setw(3) << iter << ", norm = " + << std::scientific << std::setprecision(5) << norm_abs << " (abs.), " + << std::scientific << std::setprecision(5) << norm_rel << " (rel.)" << "\n"; + } + + if (norm_abs < m_rtol) { + amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter + << ". Satisfied absolute tolerance " << m_atol << std::endl; + break; + } + + if (norm_rel < m_rtol) { + amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter + << ". Satisfied relative tolerance " << m_rtol << std::endl; + break; + } + + if (norm_abs > 100._rt*norm0) { + amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter + << ". SOLVER DIVERGED! relative tolerance = " << m_rtol << std::endl; + std::stringstream convergenceMsg; + convergenceMsg << "Newton: exiting at iteration " << std::setw(3) << iter << + ". SOLVER DIVERGED! absolute norm = " << norm_abs << + " has increased by 100X from that after first iteration."; + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } + + // Solve linear system for Newton step [Jac]*dU = F + m_dU.zero(); + m_linear_solver->solve( m_dU, m_F, m_gmres_rtol, m_gmres_atol ); + + // Update solution + a_U -= m_dU; + + iter++; + if (iter >= m_maxits) { + amrex::Print() << "Newton: exiting at iter = " << std::setw(3) << iter + << ". Maximum iteration reached: iter = " << m_maxits << std::endl; + break; + } + + } + + if (m_rtol > 0. && iter == m_maxits) { + std::stringstream convergenceMsg; + convergenceMsg << "Newton solver failed to converge after " << iter << + " iterations. Relative norm is " << norm_rel << + " and the relative tolerance is " << m_rtol << + ". Absolute norm is " << norm_abs << + " and the absolute tolerance is " << m_atol; + if (this->m_verbose) { amrex::Print() << convergenceMsg.str() << std::endl; } + if (m_require_convergence) { + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } else { + ablastr::warn_manager::WMRecordWarning("NewtonSolver", convergenceMsg.str()); + } + } + +} + +template +void NewtonSolver::EvalResidual ( Vec& a_F, + const Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt, + int a_iter ) const +{ + + m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, a_iter, false ); + + // set base U and R(U) for matrix-free Jacobian action calculation + m_linear_function->setBaseSolution(a_U); + m_linear_function->setBaseRHS(m_R); + + // update preconditioner + if (m_update_pc || m_update_pc_init) { + m_linear_function->updatePreCondMat(a_U); + } + m_update_pc_init = false; + + // Compute residual: F(U) = U - b - R(U) + a_F.Copy(a_U); + a_F -= m_R; + a_F -= a_b; + +} + +#endif diff --git a/Source/NonlinearSolvers/NonlinearSolver.H b/Source/NonlinearSolvers/NonlinearSolver.H new file mode 100644 index 00000000000..5587826474c --- /dev/null +++ b/Source/NonlinearSolvers/NonlinearSolver.H @@ -0,0 +1,87 @@ +#ifndef WARPX_NONLINEAR_SOLVER_H_ +#define WARPX_NONLINEAR_SOLVER_H_ + +#include +#include + +#include + +#include +#include +#include + +/** + * \brief Top-level class for the nonlinear solver + * + * This class is templated on a vector class Vec, and an operator class Ops. + * + * The Ops class must have the following function: + * ComputeRHS( R_vec, U_vec, time, dt, nl_iter, from_jacobian ), + * where U_vec and R_vec are of type Vec. + * + * The Vec class must have basic math operators, such as Copy, +=, -=, + * increment(), linComb(), scale(), etc.. See WarpXSolverVec.H for an example. + */ + +template +class NonlinearSolver +{ +public: + + NonlinearSolver() = default; + + virtual ~NonlinearSolver() = default; + + // Prohibit Move and Copy operations + NonlinearSolver(const NonlinearSolver&) = delete; + NonlinearSolver& operator=(const NonlinearSolver&) = delete; + NonlinearSolver(NonlinearSolver&&) noexcept = delete; + NonlinearSolver& operator=(NonlinearSolver&&) noexcept = delete; + + /** + * \brief Read user-provided parameters that control the nonlinear solver. + * Allocate intermediate data containers needed by the solver. For Newton, + * setup the linear solver for computing the Newton step. + */ + virtual void Define ( const Vec&, + Ops* ) = 0; + + /** + * \brief Check if the nonlinear solver has been defined. + */ + [[nodiscard]] bool IsDefined () const { return m_is_defined; } + + /** + * \brief Solve the specified nonlinear equation for U. + * Picard: U = b + R(U). + * Newton: F(U) = U - b - R(U) = 0. + */ + virtual void Solve ( Vec&, + const Vec&, + amrex::Real, + amrex::Real ) const = 0; + + /** + * \brief Print parameters used by the nonlinear solver. + */ + virtual void PrintParams () const = 0; + + /** + * \brief Return the convergence parameters used by the nonlinear solver. + */ + virtual void GetSolverParams (amrex::Real&, amrex::Real&, int&) = 0; + + /** + * \brief Allow caller to dynamically change the verbosity flag. For + * example, one may want to only print solver information every 100 steps. + */ + void Verbose ( bool a_verbose ) { m_verbose = a_verbose; } + +protected: + + bool m_is_defined = false; + mutable bool m_verbose = true; + +}; + +#endif diff --git a/Source/NonlinearSolvers/NonlinearSolverLibrary.H b/Source/NonlinearSolvers/NonlinearSolverLibrary.H new file mode 100644 index 00000000000..5803d6ca753 --- /dev/null +++ b/Source/NonlinearSolvers/NonlinearSolverLibrary.H @@ -0,0 +1,15 @@ +#ifndef NONLINEAR_SOLVER_LIBRARY_H_ +#define NONLINEAR_SOLVER_LIBRARY_H_ + +#include "PicardSolver.H" // IWYU pragma: export +#include "NewtonSolver.H" // IWYU pragma: export + +/** + * \brief struct to select the nonlinear solver for implicit schemes + */ +enum NonlinearSolverType { + Picard = 0, + Newton = 1 +}; + +#endif diff --git a/Source/NonlinearSolvers/PicardSolver.H b/Source/NonlinearSolvers/PicardSolver.H new file mode 100644 index 00000000000..f05b9a106e6 --- /dev/null +++ b/Source/NonlinearSolvers/PicardSolver.H @@ -0,0 +1,217 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef PICARD_SOLVER_H_ +#define PICARD_SOLVER_H_ + +#include "NonlinearSolver.H" + +#include +#include "Utils/TextMsg.H" + +#include + +/** + * \brief Picard fixed-point iteration method to solve nonlinear + * equation of form: U = b + R(U). U is the solution vector. b + * is a constant. R(U) is some nonlinear function of U, which + * is computed in the Ops function ComputeRHS(). + */ + +template +class PicardSolver : public NonlinearSolver +{ +public: + + PicardSolver() = default; + + ~PicardSolver() override = default; + + // Prohibit Move and Copy operations + PicardSolver(const PicardSolver&) = delete; + PicardSolver& operator=(const PicardSolver&) = delete; + PicardSolver(PicardSolver&&) noexcept = delete; + PicardSolver& operator=(PicardSolver&&) noexcept = delete; + + void Define ( const Vec& a_U, + Ops* a_ops ) override; + + void Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const override; + + void GetSolverParams ( amrex::Real& a_rtol, + amrex::Real& a_atol, + int& a_maxits ) override + { + a_rtol = m_rtol; + a_atol = m_atol; + a_maxits = m_maxits; + } + + void PrintParams () const override + { + amrex::Print() << "Picard max iterations: " << m_maxits << std::endl; + amrex::Print() << "Picard relative tolerance: " << m_rtol << std::endl; + amrex::Print() << "Picard absolute tolerance: " << m_atol << std::endl; + amrex::Print() << "Picard require convergence: " << (m_require_convergence?"true":"false") << std::endl; + } + +private: + + /** + * \brief Intermediate Vec containers used by the solver. + */ + mutable Vec m_Usave, m_R; + + /** + * \brief Pointer to Ops class. + */ + Ops* m_ops = nullptr; + + /** + * \brief Flag to determine whether convergence is required. + */ + bool m_require_convergence = true; + + /** + * \brief Relative tolerance for the Picard nonlinear solver + */ + amrex::Real m_rtol = 1.0e-6; + + /** + * \brief Absolute tolerance for the Picard nonlinear solver + */ + amrex::Real m_atol = 0.; + + /** + * \brief Maximum iterations for the Picard nonlinear solver + */ + int m_maxits = 100; + + void ParseParameters( ); + +}; + +template +void PicardSolver::Define ( const Vec& a_U, + Ops* a_ops ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !this->m_is_defined, + "Picard nonlinear solver object is already defined!"); + + ParseParameters(); + + m_Usave.Define(a_U); + m_R.Define(a_U); + + m_ops = a_ops; + + this->m_is_defined = true; + +} + +template +void PicardSolver::ParseParameters () +{ + const amrex::ParmParse pp_picard("picard"); + pp_picard.query("verbose", this->m_verbose); + pp_picard.query("absolute_tolerance", m_atol); + pp_picard.query("relative_tolerance", m_rtol); + pp_picard.query("max_iterations", m_maxits); + pp_picard.query("require_convergence", m_require_convergence); + +} + +template +void PicardSolver::Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const +{ + BL_PROFILE("PicardSolver::Solve()"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + this->m_is_defined, + "PicardSolver::Solve() called on undefined object"); + using namespace amrex::literals; + + // + // Picard fixed-point iteration method to solve nonlinear + // equation of form: U = b + R(U) + // + + amrex::Real norm_abs = 0.; + amrex::Real norm0 = 1._rt; + amrex::Real norm_rel = 0.; + + int iter; + for (iter = 0; iter < m_maxits;) { + + // Save previous state for norm calculation + m_Usave.Copy(a_U); + + // Update the solver state (a_U = a_b + m_R) + m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, iter, false ); + a_U.Copy(a_b); + a_U += m_R; + + // Compute the step norm and update iter + m_Usave -= a_U; + norm_abs = m_Usave.norm2(); + if (iter == 0) { + if (norm_abs > 0.) { norm0 = norm_abs; } + else { norm0 = 1._rt; } + } + norm_rel = norm_abs/norm0; + iter++; + + // Check for convergence criteria + if (this->m_verbose || iter == m_maxits) { + amrex::Print() << "Picard: iter = " << std::setw(3) << iter << ", norm = " + << std::scientific << std::setprecision(5) << norm_abs << " (abs.), " + << std::scientific << std::setprecision(5) << norm_rel << " (rel.)" << "\n"; + } + + if (norm_abs < m_atol) { + amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter + << ". Satisfied absolute tolerance " << m_atol << std::endl; + break; + } + + if (norm_rel < m_rtol) { + amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter + << ". Satisfied relative tolerance " << m_rtol << std::endl; + break; + } + + if (iter >= m_maxits) { + amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter + << ". Maximum iteration reached: iter = " << m_maxits << std::endl; + break; + } + + } + + if (m_rtol > 0. && iter == m_maxits) { + std::stringstream convergenceMsg; + convergenceMsg << "Picard solver failed to converge after " << iter << + " iterations. Relative norm is " << norm_rel << + " and the relative tolerance is " << m_rtol << + ". Absolute norm is " << norm_abs << + " and the absolute tolerance is " << m_atol; + if (this->m_verbose) { amrex::Print() << convergenceMsg.str() << std::endl; } + if (m_require_convergence) { + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } else { + ablastr::warn_manager::WMRecordWarning("PicardSolver", convergenceMsg.str()); + } + } + +} + +#endif diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index fac94ff20a3..197cb897602 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -94,7 +94,7 @@ public: amrex::ParticleReal * AMREX_RESTRICT puzp, amrex::ParticleReal const * AMREX_RESTRICT pwp, amrex::Real const * AMREX_RESTRICT amplitude, - amrex::Real dt); + amrex::Real dt, PushType push_type=PushType::Explicit); protected: diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 7b735053f0b..0693a13c1f9 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -653,7 +653,7 @@ LaserParticleContainer::Evolve (int lev, // Calculate the corresponding momentum and position for the particles update_laser_particle(pti, static_cast(np), uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), wp.dataPtr(), - amplitude_E.dataPtr(), dt); + amplitude_E.dataPtr(), dt, push_type ); WARPX_PROFILE_VAR_STOP(blp_pp); // Current Deposition @@ -851,7 +851,7 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, ParticleReal * AMREX_RESTRICT const puzp, ParticleReal const * AMREX_RESTRICT const pwp, Real const * AMREX_RESTRICT const amplitude, - const Real dt) + const Real dt, PushType push_type) { const auto GetPosition = GetParticlePosition(pti); auto SetPosition = SetParticlePosition(pti); @@ -863,6 +863,26 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, const Real tmp_nvec_1 = m_nvec[1]; const Real tmp_nvec_2 = m_nvec[2]; + // When using the implicit solver, this function is called multiple times per timestep + // (within the linear and nonlinear solver). Thus, the position of the particles needs to be reset + // to the initial position (at the beginning of the timestep), before updating the particle position +#if (AMREX_SPACEDIM >= 2) + ParticleReal* x_n = nullptr; + if (push_type == PushType::Implicit) { + x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); + } +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal* y_n = nullptr; + if (push_type == PushType::Implicit) { + y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); + } +#endif + ParticleReal* z_n = nullptr; + if (push_type == PushType::Implicit) { + z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + } + // Copy member variables to tmp copies for GPU runs. const Real tmp_mobility = m_mobility; const Real gamma_boost = WarpX::gamma_boost; @@ -894,15 +914,33 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, puzp[i] = gamma * vz; // Push the the particle positions - ParticleReal x, y, z; - GetPosition(i, x, y, z); + + // When using the implicit solver, this function is called multiple times per timestep + // (within the linear and nonlinear solver). Thus, the position of the particles needs to be reset + // to the initial position (at the beginning of the timestep), before updating the particle position + + ParticleReal x=0., y=0., z=0.; + if (push_type == PushType::Explicit) { + GetPosition(i, x, y, z); + } + #if !defined(WARPX_DIM_1D_Z) + if (push_type == PushType::Implicit) { + x = x_n[i]; + } x += vx * dt; #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + if (push_type == PushType::Implicit) { + y = y_n[i]; + } y += vy * dt; #endif + if (push_type == PushType::Implicit) { + z = z_n[i]; + } z += vz * dt; + SetPosition(i, x, y, z); } ); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 3d134393f06..e2be4f948ca 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -61,6 +61,7 @@ #include #include #include +#include #include #include #include @@ -3118,6 +3119,12 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const int qed_runtime_flag = no_qed; #endif + const int max_iterations = WarpX::max_particle_its_in_implicit_scheme; + const amrex::ParticleReal particle_tolerance = WarpX::particle_tol_in_implicit_scheme; + + amrex::Gpu::Buffer unconverged_particles({0}); + amrex::Long* unconverged_particles_ptr = unconverged_particles.data(); + // Using this version of ParallelFor with compile time options // improves performance when qed or external EB are not used by reducing // register pressure. @@ -3129,107 +3136,163 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, { // Position advance starts from the position at the start of the step // but uses the most recent velocity. + #if (AMREX_SPACEDIM >= 2) amrex::ParticleReal xp = x_n[ip]; const amrex::ParticleReal xp_n = x_n[ip]; #else - amrex::ParticleReal xp = 0._rt; + const amrex::ParticleReal xp = 0._rt; const amrex::ParticleReal xp_n = 0._rt; #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) amrex::ParticleReal yp = y_n[ip]; const amrex::ParticleReal yp_n = y_n[ip]; #else - amrex::ParticleReal yp = 0._rt; + const amrex::ParticleReal yp = 0._rt; const amrex::ParticleReal yp_n = 0._rt; #endif amrex::ParticleReal zp = z_n[ip]; const amrex::ParticleReal zp_n = z_n[ip]; - UpdatePositionImplicit(xp, yp, zp, ux_n[ip], uy_n[ip], uz_n[ip], ux[ip], uy[ip], uz[ip], 0.5_rt*dt); - setPosition(ip, xp, yp, zp); + amrex::ParticleReal dxp, dxp_save; + amrex::ParticleReal dyp, dyp_save; + amrex::ParticleReal dzp, dzp_save; + auto idxg2 = static_cast(1._rt/(dx[0]*dx[0])); + auto idyg2 = static_cast(1._rt/(dx[1]*dx[1])); + auto idzg2 = static_cast(1._rt/(dx[2]*dx[2])); - amrex::ParticleReal Exp = Ex_external_particle; - amrex::ParticleReal Eyp = Ey_external_particle; - amrex::ParticleReal Ezp = Ez_external_particle; - amrex::ParticleReal Bxp = Bx_external_particle; - amrex::ParticleReal Byp = By_external_particle; - amrex::ParticleReal Bzp = Bz_external_particle; + amrex::ParticleReal step_norm = 1._prt; + for (int iter=0; iter(ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 1, - m, q, pusher_algo, do_crr, -#ifdef WARPX_QED - t_chi_max, + if (!do_sync) #endif - dt); - } -#ifdef WARPX_QED - else { - if constexpr (qed_control == has_qed) { - doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + { + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, +#ifdef WARPX_QED t_chi_max, +#endif dt); } - } +#ifdef WARPX_QED + else { + if constexpr (qed_control == has_qed) { + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 1, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); + } + } #endif #ifdef WARPX_QED - [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; - [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; - [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc - if constexpr (qed_control == has_qed) { - if (local_has_quantum_sync) { - evolve_opt(ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp,Bxp, Byp, Bzp, - dt, p_optical_depth_QSR[ip]); + [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + if constexpr (qed_control == has_qed) { + if (local_has_quantum_sync) { + evolve_opt(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp,Bxp, Byp, Bzp, + dt, p_optical_depth_QSR[ip]); + } } - } #else amrex::ignore_unused(qed_control); #endif - // Take average to get the time centered value - ux[ip] = 0.5_rt*(ux[ip] + ux_n[ip]); - uy[ip] = 0.5_rt*(uy[ip] + uy_n[ip]); - uz[ip] = 0.5_rt*(uz[ip] + uz_n[ip]); + // Take average to get the time centered value + ux[ip] = 0.5_rt*(ux[ip] + ux_n[ip]); + uy[ip] = 0.5_rt*(uy[ip] + uy_n[ip]); + uz[ip] = 0.5_rt*(uz[ip] + uz_n[ip]); + + iter++; + + // particle did not converge + if ( iter > 1 && iter == max_iterations ) { +#if !defined(AMREX_USE_GPU) + std::stringstream convergenceMsg; + convergenceMsg << "Picard solver for particle failed to converge after " << + iter << " iterations. " << std::endl; + convergenceMsg << "Position step norm is " << step_norm << + " and the tolerance is " << particle_tolerance << std::endl; + convergenceMsg << " ux = " << ux[ip] << ", uy = " << uy[ip] << ", uz = " << uz[ip] << std::endl; + convergenceMsg << " xp = " << xp << ", yp = " << yp << ", zp = " << zp; + ablastr::warn_manager::WMRecordWarning("ImplicitPushXP", convergenceMsg.str()); +#endif + + // write signaling flag: how many particles did not converge? + amrex::Gpu::Atomic::Add(unconverged_particles_ptr, amrex::Long(1)); + } + + } // end Picard iterations }); + + auto const num_unconverged_particles = *(unconverged_particles.copyToHost()); + if (num_unconverged_particles > 0) { + ablastr::warn_manager::WMRecordWarning("ImplicitPushXP", + "Picard solver for " + + std::to_string(num_unconverged_particles) + + " particles failed to converge after " + + std::to_string(max_iterations) + " iterations." + ); + } } void diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H index 6e2e82632ee..89c2de88e47 100644 --- a/Source/Particles/Pusher/UpdatePosition.H +++ b/Source/Particles/Pusher/UpdatePosition.H @@ -85,4 +85,46 @@ void UpdatePositionImplicit(amrex::ParticleReal& x, amrex::ParticleReal& y, amre z += uz * inv_gamma * dt; } +/** \brief Check particle position for convergence. This is used by the theta-implicit + * and semi-implicit time solvers to obtain a self-consistent time-centered update + * of the particles for given electric and magnetic fields on the grid. + */ +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void PositionNorm( amrex::ParticleReal dxp, amrex::ParticleReal dyp, amrex::ParticleReal dzp, + amrex::ParticleReal& dxp_save, amrex::ParticleReal& dyp_save, amrex::ParticleReal& dzp_save, + amrex::ParticleReal idxg2, amrex::ParticleReal idyg2, amrex::ParticleReal idzg2, + amrex::ParticleReal& step_norm, const int iter ) +{ + using namespace amrex::literals; + +#if defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(dxp, dxp_save, idxg2); +#endif +#if !defined(WARPX_DIM_3D) + amrex::ignore_unused(dyp, dyp_save, idyg2); +#endif + + if (iter==0) { step_norm = 1.0_prt; } + else { + step_norm = (dzp - dzp_save)*(dzp - dzp_save)*idzg2; +#if !defined(WARPX_DIM_1D_Z) + step_norm += (dxp - dxp_save)*(dxp - dxp_save)*idxg2; +#endif +#if defined(WARPX_DIM_3D) + step_norm += (dyp - dyp_save)*(dyp - dyp_save)*idyg2; +#elif defined(WARPX_DIM_RZ) + step_norm += (dyp - dyp_save)*(dyp - dyp_save)*idxg2; +#endif + step_norm = std::sqrt(step_norm); + } + dzp_save = dzp; +#if !defined(WARPX_DIM_1D_Z) + dxp_save = dxp; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + dyp_save = dyp; +#endif + +} + #endif // WARPX_PARTICLES_PUSHER_UPDATEPOSITION_H_ diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index e067ed03a80..527b7766895 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -29,8 +29,8 @@ struct MediumForEM { struct EvolveScheme { enum { Explicit = 0, - ImplicitPicard = 1, - SemiImplicitPicard = 2 + ThetaImplicitEM = 1, + SemiImplicitEM = 2 }; }; diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 2449ea2f8dd..89784c3b86c 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -24,10 +24,10 @@ // and corresponding integer for use inside the code const std::map evolve_scheme_to_int = { - {"explicit", EvolveScheme::Explicit }, - {"implicit_picard", EvolveScheme::ImplicitPicard }, - {"semi_implicit_picard", EvolveScheme::SemiImplicitPicard }, - {"default", EvolveScheme::Explicit } + {"explicit", EvolveScheme::Explicit }, + {"theta_implicit_em", EvolveScheme::ThetaImplicitEM }, + {"semi_implicit_em", EvolveScheme::SemiImplicitEM }, + {"default", EvolveScheme::Explicit } }; const std::map grid_to_int = { diff --git a/Source/WarpX.H b/Source/WarpX.H index df06a1723ca..a0a1379d9e2 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -41,6 +41,8 @@ #include "FieldSolver/Fields.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" +#include "FieldSolver/ImplicitSolvers/ImplicitSolver.H" +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" #include "Filter/BilinearFilter.H" #include "Parallelization/GuardCellManager.H" #include "Utils/Parser/IntervalsParser.H" @@ -113,11 +115,27 @@ public: void Evolve (int numsteps = -1); - void EvolveImplicitPicardInit (int lev); - void SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev); - void FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev); - void FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Efield_fp, - amrex::Vector, 3 > >& Efield_n); + // + // Functions used by implicit solvers + // + void ImplicitPreRHSOp ( amrex::Real cur_time, + amrex::Real a_full_dt, + int a_nl_iter, + bool a_from_jacobian ); + void SaveParticlesAtImplicitStepStart (); + void FinishImplicitParticleUpdate (); + void SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ); + void UpdateMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_thetadt ); + void ApplyMagneticFieldBCs (); + void FinishMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_theta ); + void FinishImplicitField ( amrex::Vector, 3 > >& Field_fp, + const amrex::Vector, 3 > >& Field_n, + amrex::Real theta ); + void ImplicitComputeRHSE ( amrex::Real dt, WarpXSolverVec& a_Erhs_vec); + void ImplicitComputeRHSE (int lev, amrex::Real dt, WarpXSolverVec& a_Erhs_vec); + void ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real dt, WarpXSolverVec& a_Erhs_vec); MultiParticleContainer& GetPartContainer () { return *mypc; } MultiFluidContainer& GetFluidContainer () { return *myfl; } @@ -159,14 +177,12 @@ public: static short particle_pusher_algo; //! Integer that corresponds to the type of Maxwell solver (Yee, CKC, PSATD, ECT) static short electromagnetic_solver_id; - //! Integer that corresponds to the evolve scheme (explicit, implicit_picard, semi_implicit_picard) + //! Integer that corresponds to the evolve scheme (explicit, semi_implicit_em, theta_implicit_em) static short evolve_scheme; - //! The maximum number of Picard iterations to do each time step - static int max_picard_iterations; - //! The tolerance for the Picard iteration convergence - static amrex::Real picard_iteration_tolerance; - //! Flags whether the Picard iterations are required to converge - static bool require_picard_convergence; + //! Maximum iterations used for self-consistent particle update in implicit particle-suppressed evolve schemes + static int max_particle_its_in_implicit_scheme; + //! Relative tolerance used for self-consistent particle update in implicit particle-suppressed evolve schemes + static amrex::ParticleReal particle_tol_in_implicit_scheme; /** Records a number corresponding to the load balance cost update strategy * being used (0 or 1 corresponding to timers or heuristic). */ @@ -506,6 +522,17 @@ public: [[nodiscard]] const amrex::MultiFab& getField(warpx::fields::FieldType field_type, int lev, int direction = 0) const; + /** + * \brief + * Get a constant reference to the specified vector field on the different MR levels + * + * \param field_type[in] the field type + * + * \return a vector (which one element per MR level) of arrays of three pointers (for 3 vector components) amrex::MultiFab* containing the field data + */ + [[nodiscard]] const amrex::Vector,3>>& + getMultiLevelField(warpx::fields::FieldType field_type) const; + [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} @@ -1268,8 +1295,6 @@ private: void OneStep_nosub (amrex::Real cur_time); void OneStep_sub1 (amrex::Real cur_time); - void OneStep_ImplicitPicard(amrex::Real cur_time); - /** * \brief Perform one PIC iteration, with the multiple J deposition per time step */ @@ -1477,12 +1502,6 @@ private: amrex::Vector, 3 > > Efield_avg_fp; amrex::Vector, 3 > > Bfield_avg_fp; - // Implicit, fields at start of step and from the previous iteration - amrex::Vector, 3 > > Efield_n; - amrex::Vector, 3 > > Bfield_n; - amrex::Vector, 3 > > Efield_save; - amrex::Vector, 3 > > Bfield_save; - // Memory buffers for computing magnetostatic fields // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order amrex::Vector, 3 > > vector_potential_fp_nodal; @@ -1936,6 +1955,10 @@ private: amrex::Vector> m_fdtd_solver_fp; amrex::Vector> m_fdtd_solver_cp; + + // implicit solver object + std::unique_ptr m_implicit_solver; + }; #endif diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 512ed7fe9d0..d3f91002ef4 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -41,6 +41,8 @@ #include "Utils/WarpXProfilerWrapper.H" #include "Utils/WarpXUtil.H" +#include "FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H" + #include #include @@ -113,9 +115,8 @@ short WarpX::field_gathering_algo; short WarpX::particle_pusher_algo; short WarpX::electromagnetic_solver_id; short WarpX::evolve_scheme; -int WarpX::max_picard_iterations = 10; -Real WarpX::picard_iteration_tolerance = 1.e-7; -bool WarpX::require_picard_convergence = true; +int WarpX::max_particle_its_in_implicit_scheme = 21; +ParticleReal WarpX::particle_tol_in_implicit_scheme = 1.e-10; short WarpX::psatd_solution_type; short WarpX::J_in_time; short WarpX::rho_in_time; @@ -1195,6 +1196,21 @@ WarpX::ReadParameters () particle_pusher_algo = static_cast(GetAlgorithmInteger(pp_algo, "particle_pusher")); evolve_scheme = static_cast(GetAlgorithmInteger(pp_algo, "evolve_scheme")); + // check for implicit evolve scheme + if (evolve_scheme == EvolveScheme::SemiImplicitEM) { + m_implicit_solver = std::make_unique(); + } + else if (evolve_scheme == EvolveScheme::ThetaImplicitEM) { + m_implicit_solver = std::make_unique(); + } + + // implicit evolve schemes not setup to use mirrors + if (evolve_scheme == EvolveScheme::SemiImplicitEM || + evolve_scheme == EvolveScheme::ThetaImplicitEM) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( num_mirrors == 0, + "Mirrors cannot be used with Implicit evolve schemes."); + } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo != CurrentDepositionAlgo::Esirkepov || !do_current_centering, @@ -1231,8 +1247,8 @@ WarpX::ReadParameters () if (current_deposition_algo == CurrentDepositionAlgo::Villasenor) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard, + evolve_scheme == EvolveScheme::SemiImplicitEM || + evolve_scheme == EvolveScheme::ThetaImplicitEM, "Villasenor current deposition can only" "be used with Implicit evolve schemes."); } @@ -1293,11 +1309,8 @@ WarpX::ReadParameters () macroscopic_solver_algo = GetAlgorithmInteger(pp_algo,"macroscopic_sigma_method"); } - if (evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard) { - utils::parser::queryWithParser(pp_algo, "max_picard_iterations", max_picard_iterations); - utils::parser::queryWithParser(pp_algo, "picard_iteration_tolerance", picard_iteration_tolerance); - utils::parser::queryWithParser(pp_algo, "require_picard_convergence", require_picard_convergence); + if (evolve_scheme == EvolveScheme::SemiImplicitEM || + evolve_scheme == EvolveScheme::ThetaImplicitEM) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo == CurrentDepositionAlgo::Esirkepov || @@ -2177,11 +2190,6 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d AllocLevelMFs(lev, ba, dm, guard_cells.ng_alloc_EB, guard_cells.ng_alloc_J, guard_cells.ng_alloc_Rho, guard_cells.ng_alloc_F, guard_cells.ng_alloc_G, aux_is_nodal); - if (evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard) { - EvolveImplicitPicardInit(lev); - } - m_accelerator_lattice[lev] = std::make_unique(); m_accelerator_lattice[lev]->InitElementFinder(lev, ba, dm); @@ -3513,3 +3521,32 @@ WarpX::getField(FieldType field_type, const int lev, const int direction) const { return *getFieldPointer(field_type, lev, direction); } + +const amrex::Vector,3>>& +WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const +{ + switch(field_type) + { + case FieldType::Efield_aux : + return Efield_aux; + case FieldType::Bfield_aux : + return Bfield_aux; + case FieldType::Efield_fp : + return Efield_fp; + case FieldType::Bfield_fp : + return Bfield_fp; + case FieldType::current_fp : + return current_fp; + case FieldType::current_fp_nodal : + return current_fp_nodal; + case FieldType::Efield_cp : + return Efield_cp; + case FieldType::Bfield_cp : + return Bfield_cp; + case FieldType::current_cp : + return current_cp; + default: + WARPX_ABORT_WITH_MESSAGE("Invalid field type"); + return Efield_fp; + } +} From fd8b4b885c62e0c1f5e794907a1a124579179b4e Mon Sep 17 00:00:00 2001 From: Avigdor Veksler <124003120+aveksler1@users.noreply.github.com> Date: Tue, 4 Jun 2024 20:34:08 -0700 Subject: [PATCH 12/87] Add upper bound for weight of product particles from particle resampling (#4969) * add upper bound for weight parameter on particle merging * increase readability, fix clang-tidy test * rework from ignoring particles above some maximum weight to instead resample particles such that they stay under a target weight * avoid implicit capture of this, cleanup * remove leftover code Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * clarify docstring * clarifying comment --------- Co-authored-by: Avigdor Veksler Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- Python/pywarpx/picmi.py | 5 +++++ .../Resampling/VelocityCoincidenceThinning.H | 1 + .../Resampling/VelocityCoincidenceThinning.cpp | 15 +++++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 34c21c2a34a..61fad85ec94 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -137,6 +137,9 @@ class Species(picmistandard.PICMI_Species): Cells with fewer particles than this number will be skipped during resampling. + warpx_resampling_algorithm_target_weight: float + Weight that the product particles from resampling will not exceed. + warpx_resampling_trigger_intervals: bool, default=0 Timesteps at which to resample @@ -249,6 +252,7 @@ def init(self, kw): self.resampling_min_ppc = kw.pop('warpx_resampling_min_ppc', None) self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) + self.resampling_algorithm_target_weight = kw.pop('warpx_resampling_algorithm_target_weight', None) self.resampling_algorithm_velocity_grid_type = kw.pop('warpx_resampling_algorithm_velocity_grid_type', None) self.resampling_algorithm_delta_ur = kw.pop('warpx_resampling_algorithm_delta_ur', None) self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) @@ -298,6 +302,7 @@ def species_initialize_inputs(self, layout, resampling_min_ppc=self.resampling_min_ppc, resampling_trigger_intervals=self.resampling_trigger_intervals, resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc, + resampling_algorithm_target_weight=self.resampling_algorithm_target_weight, resampling_algorithm_velocity_grid_type=self.resampling_algorithm_velocity_grid_type, resampling_algorithm_delta_ur=self.resampling_algorithm_delta_ur, resampling_algorithm_n_theta=self.resampling_algorithm_n_theta, diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.H b/Source/Particles/Resampling/VelocityCoincidenceThinning.H index b8d67d198a4..bb325734777 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.H +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.H @@ -200,5 +200,6 @@ private: int m_ntheta, m_nphi; amrex::ParticleReal m_delta_ur; amrex::Vector m_delta_u; + amrex::ParticleReal m_cluster_weight = std::numeric_limits::max(); }; #endif // WARPX_VELOCITY_COINCIDENCE_THINNING_H_ diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp index db4ddb801c6..2880ace200d 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp @@ -24,6 +24,14 @@ VelocityCoincidenceThinning::VelocityCoincidenceThinning (const std::string& spe "Resampling min_ppc should be greater than or equal to 1" ); + amrex::ParticleReal target_weight = 0; + if (utils::parser::queryWithParser( + pp_species_name, "resampling_algorithm_target_weight", target_weight + )) { + // factor of 2 since each cluster is reduced to 2 particles + m_cluster_weight = target_weight * 2.0_prt; + } + std::string velocity_grid_type_str = "spherical"; pp_species_name.query( "resampling_algorithm_velocity_grid_type", velocity_grid_type_str @@ -84,7 +92,7 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, auto *const cell_offsets = bins.offsetsPtr(); const auto min_ppc = m_min_ppc; - + const auto cluster_weight = m_cluster_weight; const auto mass = pc->getMass(); // check if species mass > 0 @@ -207,10 +215,13 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, ux[part_idx], uy[part_idx], uz[part_idx], mass ); - // check if this is the last particle in the current momentum bin + // check if this is the last particle in the current momentum bin, + // or if the next particle would push the current cluster weight + // to exceed the maximum specified cluster weight if ( (i == cell_stop - 1) || (momentum_bin_number_data[sorted_indices_data[i]] != momentum_bin_number_data[sorted_indices_data[i + 1]]) + || (total_weight + w[indices[sorted_indices_data[i+1]]] > cluster_weight) ) { // check if the bin has more than 2 particles in it if ( particles_in_bin > 2 && total_weight > std::numeric_limits::min() ){ From 67419c6ad3edbbbfafb4d5e2ecbca4cece524bbe Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 5 Jun 2024 19:15:10 -0700 Subject: [PATCH 13/87] Add initialization of extra attribs to picmi (#4972) * add initialization of extra attribs to picmi * avoid new `Bucket` * update CI checksum * update ionization test analysis to always include runtime attribute * Update Examples Further, Fix CI * update docstring; update checksum values * allow user defined attributes to be added to `ParticleDiagnostic` output * more CI fixes * revert adding `z_orig` to inputs_2d_bf_rt --------- Co-authored-by: Axel Huebl --- .../laser_acceleration/PICMI_inputs_3d.py | 4 ++- .../laser_acceleration/inputs_3d | 2 +- .../laser_ion/PICMI_inputs_2d.py | 5 ++-- Examples/Tests/ionization/PICMI_inputs_2d.py | 8 +++--- .../Tests/ionization/analysis_ionization.py | 9 ++++--- Examples/Tests/ionization/inputs_2d_bf_rt | 2 +- Python/pywarpx/picmi.py | 25 +++++++++++++++++++ .../Python_LaserAcceleration.json | 4 ++- .../benchmarks_json/Python_LaserIonAcc2d.json | 4 ++- .../benchmarks_json/Python_ionization.json | 20 ++++++++------- 10 files changed, 59 insertions(+), 24 deletions(-) diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py index ac3398e43fc..13bf492e203 100755 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py +++ b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py @@ -55,7 +55,9 @@ electrons = picmi.Species( particle_type = 'electron', name = 'electrons', - initial_distribution = uniform_distribution) + initial_distribution = uniform_distribution, + warpx_add_int_attributes = {'regionofinterest': "(z>12.0e-6) * (z<13.0e-6)"}, + warpx_add_real_attributes = {'initialenergy': "ux*ux + uy*uy + uz*uz"}) # Particles: beam electrons q_tot = 1e-12 diff --git a/Examples/Physics_applications/laser_acceleration/inputs_3d b/Examples/Physics_applications/laser_acceleration/inputs_3d index bdcfd7676a4..fb8810dd34b 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_3d +++ b/Examples/Physics_applications/laser_acceleration/inputs_3d @@ -55,7 +55,7 @@ electrons.do_continuous_injection = 1 electrons.addIntegerAttributes = regionofinterest electrons.attribute.regionofinterest(x,y,z,ux,uy,uz,t) = "(z>12.0e-6) * (z<13.0e-6)" electrons.addRealAttributes = initialenergy -electrons.attribute.initialenergy(x,y,z,ux,uy,uz,t) = " ux*ux + uy*uy + uz*uz" +electrons.attribute.initialenergy(x,y,z,ux,uy,uz,t) = "ux*ux + uy*uy + uz*uz" ################################# ############ LASER ############# diff --git a/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py index 844501992c3..9f7a2aacfca 100755 --- a/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py +++ b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py @@ -91,18 +91,17 @@ rms_velocity=[c*ux_th, 0., c*uz_th] # thermal velocity spread in m/s ) -# TODO: add additional attributes orig_x and orig_z electrons = picmi.Species( particle_type='electron', name='electrons', initial_distribution=slab_with_ramp_dist_electrons, ) -# TODO: add additional attributes orig_x and orig_z hydrogen = picmi.Species( particle_type='proton', name='hydrogen', - initial_distribution=slab_with_ramp_dist_hydrogen + initial_distribution=slab_with_ramp_dist_hydrogen, + warpx_add_real_attributes = {"orig_x": "x", "orig_z": "z"} ) # Laser diff --git a/Examples/Tests/ionization/PICMI_inputs_2d.py b/Examples/Tests/ionization/PICMI_inputs_2d.py index a076361bf50..802bf5435ac 100644 --- a/Examples/Tests/ionization/PICMI_inputs_2d.py +++ b/Examples/Tests/ionization/PICMI_inputs_2d.py @@ -47,12 +47,14 @@ fill_in = True) electrons = picmi.Species( particle_type = 'electron', - name = 'electrons') + name = 'electrons', + warpx_add_real_attributes = {'orig_z': 'z'}) ions = picmi.Species( particle_type = 'N', name = 'ions', charge_state = 2, - initial_distribution = uniform_distribution) + initial_distribution = uniform_distribution, + warpx_add_real_attributes = {'orig_z': 'z'}) # Field ionization nitrogen_ionization = picmi.FieldIonization( @@ -88,7 +90,7 @@ name = 'diag1', period = 10000, species = [electrons, ions], - data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting'], + data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting', 'orig_z'], write_dir = '.', warpx_file_prefix = 'Python_ionization_plt') field_diag = picmi.FieldDiagnostic( diff --git a/Examples/Tests/ionization/analysis_ionization.py b/Examples/Tests/ionization/analysis_ionization.py index 95732b03e36..90657915b50 100755 --- a/Examples/Tests/ionization/analysis_ionization.py +++ b/Examples/Tests/ionization/analysis_ionization.py @@ -52,7 +52,7 @@ ad = ds.all_data() # Plot ions with ionization levels - species = 'ions'; + species = 'ions' xi = ad[species, 'particle_position_x'].v zi = ad[species, 'particle_position_y'].v ii = ad[species, 'particle_ionizationLevel'].v @@ -75,7 +75,7 @@ plt.colorbar() # Plot electrons - species = 'electrons'; + species = 'electrons' if species in [x[0] for x in ds.field_list]: xe = ad[species, 'particle_position_x'].v ze = ad[species, 'particle_position_y'].v @@ -96,10 +96,11 @@ # Check that the user runtime component (if it exists) worked as expected try: orig_z = ad['electrons', 'particle_orig_z'].v - assert np.all( (orig_z > 0) & (orig_z < 1.5e-5) ) + print(f"orig_z: min = {np.min(orig_z)}, max = {np.max(orig_z)}") + assert np.all( (orig_z > 0.0) & (orig_z < 1.5e-5) ) print('particle_orig_z has reasonable values') except yt.utilities.exceptions.YTFieldNotFound: - pass # Some of the tested script to not have the quantity orig_z + pass # The backtransformed diagnostic version of the test does not have orig_z test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/ionization/inputs_2d_bf_rt b/Examples/Tests/ionization/inputs_2d_bf_rt index 52af5d0d40f..8bcb66595d2 100644 --- a/Examples/Tests/ionization/inputs_2d_bf_rt +++ b/Examples/Tests/ionization/inputs_2d_bf_rt @@ -48,7 +48,7 @@ electrons.zmax = 50.e-6 electrons.profile = constant electrons.density = 2. electrons.momentum_distribution_type = at_rest -electrons.do_continuous_injection=1 +electrons.do_continuous_injection = 1 lasers.names = laser1 laser1.profile = Gaussian diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 61fad85ec94..b981bc5e1d3 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -171,6 +171,14 @@ class Species(picmistandard.PICMI_Species): during grid-based merging, with `velocity_grid_type == "cartesian"`. If a single number is given the same du value will be used in all three directions. + + warpx_add_int_attributes: dict + Dictionary of extra integer particle attributes initialized from an + expression that is a function of the variables (x, y, z, ux, uy, uz, t). + + warpx_add_real_attributes: dict + Dictionary of extra real particle attributes initialized from an + expression that is a function of the variables (x, y, z, ux, uy, uz, t). """ def init(self, kw): @@ -261,6 +269,10 @@ def init(self, kw): if self.resampling_algorithm_delta_u is not None and np.size(self.resampling_algorithm_delta_u) == 1: self.resampling_algorithm_delta_u = [self.resampling_algorithm_delta_u]*3 + # extra particle attributes + self.extra_int_attributes = kw.pop('warpx_add_int_attributes', None) + self.extra_real_attributes = kw.pop('warpx_add_real_attributes', None) + def species_initialize_inputs(self, layout, initialize_self_fields = False, injection_plane_position = None, @@ -318,6 +330,16 @@ def species_initialize_inputs(self, layout, self.species.add_new_attr("reflection_model_zhi(E)", self.reflection_model_zhi) # self.species.add_new_attr("reflection_model_eb(E)", self.reflection_model_eb) + # extra particle attributes + if self.extra_int_attributes is not None: + self.species.addIntegerAttributes = self.extra_int_attributes.keys() + for attr, function in self.extra_int_attributes.items(): + self.species.add_new_attr('attribute.'+attr+'(x,y,z,ux,uy,uz,t)', function) + if self.extra_real_attributes is not None: + self.species.addRealAttributes = self.extra_real_attributes.keys() + for attr, function in self.extra_real_attributes.items(): + self.species.add_new_attr('attribute.'+attr+'(x,y,z,ux,uy,uz,t)', function) + pywarpx.Particles.particles_list.append(self.species) if self.initial_distribution is not None: @@ -2596,6 +2618,9 @@ def diagnostic_initialize_inputs(self): ) else: variables.add(dataname) + else: + # possibly add user defined attributes + variables.add(dataname) # --- Convert the set to a sorted list so that the order # --- is the same on all processors. diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json index 08969db023e..474e812480f 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json @@ -21,12 +21,14 @@ "particle_weight": 6241509.074460764 }, "electrons": { + "particle_initialenergy": 0.0, "particle_momentum_x": 1.7921232210868553e-20, "particle_momentum_y": 7.225819896136567e-20, "particle_momentum_z": 4.2317254599358777e-20, "particle_position_x": 0.713912262116188, "particle_position_y": 0.7150340887578024, "particle_position_z": 1.31757706006908, + "particle_regionofinterest": 1936.0, "particle_weight": 12926557617.187498 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json index baaf29bec59..d2fd4841cc4 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json @@ -23,6 +23,8 @@ "particle_weight": 2.6507336926909222e+17 }, "hydrogen": { + "particle_origX": 0.008198291015625001, + "particle_origZ": 0.0365664599609375, "particle_position_x": 0.008197892199782453, "particle_position_y": 0.0, "particle_position_z": 0.0365646600930625, @@ -31,4 +33,4 @@ "particle_momentum_z": 1.0873094324185116e-18, "particle_weight": 2.703612070965676e+17 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Python_ionization.json b/Regression/Checksum/benchmarks_json/Python_ionization.json index 31f426aa362..a5e65fcf765 100644 --- a/Regression/Checksum/benchmarks_json/Python_ionization.json +++ b/Regression/Checksum/benchmarks_json/Python_ionization.json @@ -10,21 +10,23 @@ "jy": 0.0, "jz": 1.3483401471475687e-07 }, - "electrons": { - "particle_momentum_x": 4.4206237143449475e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.6361297302081026e-18, - "particle_position_x": 0.11009154442846772, - "particle_position_y": 0.6414658436421568, - "particle_weight": 3.4450781249999996e-10 - }, "ions": { "particle_ionizationLevel": 72897.0, "particle_momentum_x": 1.76132401934254e-18, "particle_momentum_y": 0.0, "particle_momentum_z": 3.644887053263054e-23, + "particle_orig_z": 0.128, "particle_position_x": 0.03200001189420337, "particle_position_y": 0.1280000046901387, "particle_weight": 9.999999999999999e-11 + }, + "electrons": { + "particle_momentum_x": 4.4206237143449475e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.6361297302081026e-18, + "particle_orig_z": 0.4305565137391907, + "particle_position_x": 0.11009154442846772, + "particle_position_y": 0.6414658436421568, + "particle_weight": 3.4450781249999996e-10 } -} \ No newline at end of file +} From dc2c2f690c85eb648036c0710fd773972b9bfde4 Mon Sep 17 00:00:00 2001 From: David Grote Date: Fri, 7 Jun 2024 11:16:23 -0700 Subject: [PATCH 14/87] Add temperature diagnostic (#4942) * Add temperature diagnostic * Add to CI test * Add input to picmi * Add comment on the method * const cleanup * Fix particle array declaration * Update benchmark for CI --- Docs/source/usage/parameters.rst | 4 +- Examples/Tests/collision/inputs_3d | 2 +- Python/pywarpx/picmi.py | 3 + .../benchmarks_json/collisionXYZ.json | 4 +- .../ComputeDiagFunctors/CMakeLists.txt | 1 + .../ComputeDiagFunctors/Make.package | 1 + .../ComputeDiagFunctors/TemperatureFunctor.H | 35 ++++ .../TemperatureFunctor.cpp | 165 ++++++++++++++++++ Source/Diagnostics/Diagnostics.H | 2 + Source/Diagnostics/Diagnostics.cpp | 24 +++ Source/Diagnostics/FullDiagnostics.cpp | 17 ++ 11 files changed, 255 insertions(+), 3 deletions(-) create mode 100644 Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H create mode 100644 Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index fb53e81012f..a3ca703d15e 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2693,7 +2693,9 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``.fields_to_plot`` (list of `strings`, optional) Fields written to output. - Possible scalar fields: ``part_per_cell`` ``rho`` ``phi`` ``F`` ``part_per_grid`` ``divE`` ``divB`` and ``rho_``, where ```` must match the name of one of the available particle species. Note that ``phi`` will only be written out when do_electrostatic==labframe. Also, note that for ``.diag_type = BackTransformed``, the only scalar field currently supported is ``rho``. + Possible scalar fields: ``part_per_cell`` ``rho`` ``phi`` ``F`` ``part_per_grid`` ``divE`` ``divB`` ``rho_`` and ``T_``, where ```` must match the name of one of the available particle species. + ``T_`` is the temperature in eV. + Note that ``phi`` will only be written out when do_electrostatic==labframe. Also, note that for ``.diag_type = BackTransformed``, the only scalar field currently supported is ``rho``. Possible vector field components in Cartesian geometry: ``Ex`` ``Ey`` ``Ez`` ``Bx`` ``By`` ``Bz`` ``jx`` ``jy`` ``jz``. Possible vector field components in RZ geometry: ``Er`` ``Et`` ``Ez`` ``Br`` ``Bt`` ``Bz`` ``jr`` ``jt`` ``jz``. The default is ``.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz`` in Cartesian geometry and ``.fields_to_plot = Er Et Ez Br Bt Bz jr jt jz`` in RZ geometry. diff --git a/Examples/Tests/collision/inputs_3d b/Examples/Tests/collision/inputs_3d index 3cc06061bed..ed413ba2776 100644 --- a/Examples/Tests/collision/inputs_3d +++ b/Examples/Tests/collision/inputs_3d @@ -75,7 +75,7 @@ collision3.ndt = 10 diagnostics.diags_names = diag1 diag_parser_filter diag_uniform_filter diag_random_filter diag1.intervals = 10 diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz +diag1.fields_to_plot = Ex Ey Ez Bx By Bz T_electron T_ion ## diag_parser_filter is a diag used to test the particle filter function. diag_parser_filter.intervals = 150:150: diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index b981bc5e1d3..0048cb1e51b 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2411,6 +2411,9 @@ def diagnostic_initialize_inputs(self): elif dataname.startswith('rho_'): # Adds rho_species diagnostic fields_to_plot.add(dataname) + elif dataname.startswith('T_'): + # Adds T_species diagnostic + fields_to_plot.add(dataname) elif dataname == 'dive': fields_to_plot.add('divE') elif dataname == 'divb': diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 927848745a8..6e4b9abf965 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -5,6 +5,8 @@ "Bz": 0.0, "Ex": 0.0, "Ey": 0.0, - "Ez": 0.0 + "Ez": 0.0, + "T_electron": 362230.52300397365, + "T_ion": 338312.83502136066 } } diff --git a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt index 5e0eeaab73a..2a5cc87c0cb 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt +++ b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt @@ -13,5 +13,6 @@ foreach(D IN LISTS WarpX_DIMS) BackTransformFunctor.cpp BackTransformParticleFunctor.cpp ParticleReductionFunctor.cpp + TemperatureFunctor.cpp ) endforeach() diff --git a/Source/Diagnostics/ComputeDiagFunctors/Make.package b/Source/Diagnostics/ComputeDiagFunctors/Make.package index fd1624b8708..0fd618748e3 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/Make.package +++ b/Source/Diagnostics/ComputeDiagFunctors/Make.package @@ -9,5 +9,6 @@ CEXE_sources += RhoFunctor.cpp CEXE_sources += BackTransformFunctor.cpp CEXE_sources += BackTransformParticleFunctor.cpp CEXE_sources += ParticleReductionFunctor.cpp +CEXE_sources += TemperatureFunctor.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Diagnostics/ComputeDiagFunctors diff --git a/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H new file mode 100644 index 00000000000..f6c425e74d5 --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H @@ -0,0 +1,35 @@ +#ifndef WARPX_TEMPERATUREFUNCTOR_H_ +#define WARPX_TEMPERATUREFUNCTOR_H_ + +#include "ComputeDiagFunctor.H" + +#include + +/** + * \brief Functor to calculate per-cell averages of particle temperature + */ +class TemperatureFunctor final : public ComputeDiagFunctor +{ +public: + /** \brief Constructor + * \param[in] lev level of multifab + * \param[in] crse_ratio for interpolating field values from simulation MultiFabs + to the output diagnostic MultiFab mf_dst + * \param[in] ispec index of the species over which to calculate the temperature + * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab + */ + TemperatureFunctor(int lev, amrex::IntVect crse_ratio, int ispec, int ncomp=1); + + /** \brief Compute the temperature in each grid cell. + * + * \param[out] mf_dst output MultiFab where the result is written + * \param[in] dcomp first component of mf_dst in which cell-centered + * data is stored + */ + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; +private: + int const m_lev; /**< level on which mf_src is defined */ + int const m_ispec; /**< index of species to average over */ +}; + +#endif // WARPX_TEMPERATUREFUNCTOR_H_ diff --git a/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp new file mode 100644 index 00000000000..c42f8970d5e --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp @@ -0,0 +1,165 @@ + +#include "TemperatureFunctor.H" + +#include "Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/Parser/ParserUtils.H" +#include "WarpX.H" + +#include + +#include +#include +#include +#include +#include + +TemperatureFunctor::TemperatureFunctor (const int lev, + const amrex::IntVect crse_ratio, const int ispec, const int ncomp) + : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev), m_ispec(ispec) +{ + // Write only in one output component. + AMREX_ALWAYS_ASSERT(ncomp == 1); +} + +void +TemperatureFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const int /*i_buffer*/) const +{ + using namespace amrex::literals; + auto& warpx = WarpX::GetInstance(); + + // Guard cell is set to 1 for generality. However, for a cell-centered + // output Multifab, mf_dst, the guard-cell data is not needed especially considering + // the operations performend in the CoarsenAndInterpolate function. + constexpr int ng = 1; + + // Temporary cell-centered, multi-component MultiFab for storing particles sums and result + amrex::MultiFab sum_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 7, ng); + + auto& pc = warpx.GetPartContainer().GetParticleContainer(m_ispec); + amrex::Real const mass = pc.getMass(); // Note, implicit conversion from ParticleReal + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(mass > 0., + "The temperature diagnostic can not be calculated for a massless species."); + + // Calculate the averages in two steps, first the average velocity , then the + // average velocity squared >**2. This method is more robust than the + // single step using - **2 when >> u_rms. + ParticleToMesh(pc, sum_mf, m_lev, + [=] AMREX_GPU_DEVICE (const WarpXParticleContainer::SuperParticleType& p, + amrex::Array4 const& out_array, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi) + { + // Get position in AMReX convention to calculate corresponding index. + // Ideally this will be replaced with the AMReX NGP interpolator + // Always do x direction. + int ii = 0, jj = 0, kk = 0; + const amrex::ParticleReal x = p.pos(0); + const amrex::Real lx = (x - plo[0]) * dxi[0]; + ii = static_cast(amrex::Math::floor(lx)); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + const amrex::ParticleReal y = p.pos(1); + const amrex::Real ly = (y - plo[1]) * dxi[1]; + jj = static_cast(amrex::Math::floor(ly)); +#endif +#if defined(WARPX_DIM_3D) + const amrex::ParticleReal z = p.pos(2); + const amrex::Real lz = (z - plo[2]) * dxi[2]; + kk = static_cast(amrex::Math::floor(lz)); +#endif + + const amrex::ParticleReal w = p.rdata(PIdx::w); + const amrex::ParticleReal ux = p.rdata(PIdx::ux); + const amrex::ParticleReal uy = p.rdata(PIdx::uy); + const amrex::ParticleReal uz = p.rdata(PIdx::uz); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 0), (amrex::Real)(w)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 1), (amrex::Real)(w*ux)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 2), (amrex::Real)(w*uy)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 3), (amrex::Real)(w*uz)); + }); + + // Divide value by number of particles for average + for (amrex::MFIter mfi(sum_mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const amrex::Box& box = mfi.tilebox(); + amrex::Array4 const& out_array = sum_mf.array(mfi); + amrex::ParallelFor(box, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + if (out_array(i,j,k,0) > 0) { + const amrex::Real invsum = 1._rt/out_array(i,j,k,0); + out_array(i,j,k,1) *= invsum; + out_array(i,j,k,2) *= invsum; + out_array(i,j,k,3) *= invsum; + } + }); + } + + // Calculate the sum of the squares, subtracting the averages + // These loops must be written out since ParticleToMesh always zeros out the mf. + const auto plo = pc.Geom(m_lev).ProbLoArray(); + const auto dxi = pc.Geom(m_lev).InvCellSizeArray(); + for (WarpXParIter pti(pc, m_lev); pti.isValid(); ++pti) + { + const long np = pti.numParticles(); + amrex::ParticleReal* wp = pti.GetAttribs(PIdx::w).dataPtr(); + amrex::ParticleReal* uxp = pti.GetAttribs(PIdx::ux).dataPtr(); + amrex::ParticleReal* uyp = pti.GetAttribs(PIdx::uy).dataPtr(); + amrex::ParticleReal* uzp = pti.GetAttribs(PIdx::uz).dataPtr(); + + auto const GetPosition = GetParticlePosition(pti); + + amrex::Array4 const& out_array = sum_mf.array(pti); + + amrex::ParallelFor(np, + [=] AMREX_GPU_DEVICE (long ip) { + // --- Get particle quantities + amrex::ParticleReal xp, yp, zp; + GetPosition.AsStored(ip, xp, yp, zp); + + // Get position in AMReX convention to calculate corresponding index. + int ii = 0, jj = 0, kk = 0; + const amrex::Real lx = (xp - plo[0]) * dxi[0]; + ii = static_cast(amrex::Math::floor(lx)); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real lz = (zp - plo[1]) * dxi[1]; + jj = static_cast(amrex::Math::floor(lz)); +#elif defined(WARPX_DIM_3D) + const amrex::Real ly = (yp - plo[1]) * dxi[1]; + jj = static_cast(amrex::Math::floor(ly)); + const amrex::Real lz = (zp - plo[2]) * dxi[2]; + kk = static_cast(amrex::Math::floor(lz)); +#endif + + const amrex::ParticleReal w = wp[ip]; + const amrex::ParticleReal ux = uxp[ip] - out_array(ii, jj, kk, 1); + const amrex::ParticleReal uy = uyp[ip] - out_array(ii, jj, kk, 2); + const amrex::ParticleReal uz = uzp[ip] - out_array(ii, jj, kk, 3); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 4), (amrex::Real)(w*ux*ux)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 5), (amrex::Real)(w*uy*uy)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 6), (amrex::Real)(w*uz*uz)); + }); + } + + // Divide the squares by number of particles for average and calculate the temperature + for (amrex::MFIter mfi(sum_mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const amrex::Box& box = mfi.tilebox(); + amrex::Array4 const& out_array = sum_mf.array(mfi); + amrex::ParallelFor(box, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + if (out_array(i,j,k,0) > 0) { + const amrex::Real invsum = 1._rt/out_array(i,j,k,0); + out_array(i,j,k,4) *= invsum; + out_array(i,j,k,5) *= invsum; + out_array(i,j,k,6) *= invsum; + out_array(i,j,k,0) = mass*(out_array(i,j,k,4) + out_array(i,j,k,5) + out_array(i,j,k,6))/(3._rt*PhysConst::q_e); + } + }); + } + + // Coarsen and interpolate from sum_mf to the output diagnostic MultiFab, mf_dst. + ablastr::coarsen::sample::Coarsen(mf_dst, sum_mf, dcomp, 0, nComp(), 0, m_crse_ratio); + +} diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index c0d2a9f0d53..20550364fb7 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -304,6 +304,8 @@ protected: int m_num_buffers; /** Array of species indices that dump rho per species */ amrex::Vector m_rho_per_species_index; + /** Array of species indices that dump temperature per species */ + amrex::Vector m_T_per_species_index; /** Vector of particle buffer vectors for each snapshot */ amrex::Vector< amrex::Vector > > m_particles_buffer; /** Vector of pointers to functors to compute particle output per species*/ diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index 3b5daabaffa..f6e2da74127 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -278,6 +278,30 @@ Diagnostics::BaseReadParameters () + ".fields_to_plot does not match any species" ); } + // Check if m_varnames contains a string of the form T_ + if (var.rfind("T_", 0) == 0) { + // Extract species name from the string T_ + const std::string species = var.substr(var.find("T_") + 2); + // Boolean used to check if species name was misspelled + bool species_name_is_wrong = true; + // Loop over all species + for (int i = 0, n = int(m_all_species_names.size()); i < n; i++) { + // Check if species name extracted from the string T_ + // matches any of the species in the simulation + if (species == m_all_species_names[i]) { + // Store species index: will be used in TemperatureFunctor to dump + // T for this species + m_T_per_species_index.push_back(i); + species_name_is_wrong = false; + } + } + // If species name was misspelled, abort with error message + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !species_name_is_wrong, + "Input error: string " + var + " in " + m_diag_name + + ".fields_to_plot does not match any species" + ); + } } const bool checkpoint_compatibility = ( diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index b25f899e29d..55af73d6408 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -8,6 +8,7 @@ #include "ComputeDiagFunctors/PartPerCellFunctor.H" #include "ComputeDiagFunctors/PartPerGridFunctor.H" #include "ComputeDiagFunctors/ParticleReductionFunctor.H" +#include "ComputeDiagFunctors/TemperatureFunctor.H" #include "ComputeDiagFunctors/RhoFunctor.H" #include "Diagnostics/Diagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" @@ -186,6 +187,8 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) // Species index to loop over species that dump rho per species int i = 0; + // Species index to loop over species that dump temperature per species + int i_T_species = 0; const int ncomp = ncomp_multimodefab; // This function is called multiple times, for different values of `lev` // but the `varnames` need only be updated once. @@ -303,6 +306,13 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) AddRZModesToOutputNames(std::string("rho_") + m_all_species_names[m_rho_per_species_index[i]], ncomp); } i++; + } else if ( m_varnames_fields[comp].rfind("T_", 0) == 0 ){ + // Initialize temperature functor to dump temperature per species + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_T_per_species_index[i_T_species]); + if (update_varnames) { + AddRZModesToOutputNames(std::string("T_") + m_all_species_names[m_T_per_species_index[i_T_species]], ncomp); + } + i_T_species++; } else if ( m_varnames_fields[comp] == "F" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio, false, ncomp); @@ -637,6 +647,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // Species index to loop over species that dump rho per species int i = 0; + // Species index to loop over species that dump temperature per species + int i_T_species = 0; + const auto nvar = static_cast(m_varnames_fields.size()); const auto nspec = static_cast(m_pfield_species.size()); const auto ntot = static_cast(nvar + m_pfield_varnames.size() * nspec); @@ -666,6 +679,10 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // Initialize rho functor to dump rho per species m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, m_rho_per_species_index[i]); i++; + } else if ( m_varnames[comp].rfind("T_", 0) == 0 ){ + // Initialize temperature functor to dump temperature per species + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_T_per_species_index[i_T_species]); + i_T_species++; } else if ( m_varnames[comp] == "F" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ From e80ca983aa27b023b0528c0ed2875a5745ea6599 Mon Sep 17 00:00:00 2001 From: David Grote Date: Fri, 7 Jun 2024 17:54:08 -0700 Subject: [PATCH 15/87] Cleanup diagnostic dimension macros (#4973) * Clean up BeamRelavent diagnostics, using get_particle_position * In Field diagnostics, simplify calculation of dV * Clean up of ParticleExtrema * In WarpXOpenPMD, removed duplicate particle coordinate transformation with RZ * Added more amrex prefixes in ParticleExtrema * Fix const * Small cleanup in WarpXOpenPMD * Clean up calculation of dV * Cleanup call to CellSize * Fix in FieldEnergy --- .../Diagnostics/ReducedDiags/BeamRelevant.cpp | 54 +--- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 14 +- .../ReducedDiags/FieldMomentum.cpp | 12 +- .../ReducedDiags/ParticleExtrema.cpp | 271 ++++++------------ Source/Diagnostics/WarpXOpenPMD.cpp | 43 +-- 5 files changed, 119 insertions(+), 275 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index ae7d5230b4c..9b5a28a3516 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -175,15 +175,6 @@ void BeamRelevant::ComputeDiags (int step) // inverse of speed of light squared Real constexpr inv_c2 = 1.0_rt / (PhysConst::c * PhysConst::c); - // If 2D-XZ, p.pos(1) is z, rather than p.pos(2). -#if (defined WARPX_DIM_3D) - int const index_z = 2; -#elif (defined WARPX_DIM_XZ || defined WARPX_DIM_RZ) - int const index_z = 1; -#elif (defined WARPX_DIM_1D_Z) - int const index_z = 0; -#endif - // loop over species for (int i_s = 0; i_s < nSpecies; ++i_s) { @@ -212,26 +203,14 @@ void BeamRelevant::ComputeDiags (int step) const ParticleReal p_uy = p.rdata(PIdx::uy); const ParticleReal p_uz = p.rdata(PIdx::uz); const ParticleReal p_us = p_ux*p_ux + p_uy*p_uy + p_uz*p_uz; - const ParticleReal p_pos0 = p.pos(0); const ParticleReal p_w = p.rdata(PIdx::w); -#if defined(WARPX_DIM_3D) - const ParticleReal p_pos1 = p.pos(1); - const ParticleReal p_x_mean = p_pos0*p_w; - const ParticleReal p_y_mean = p_pos1*p_w; -#elif defined(WARPX_DIM_RZ) - const ParticleReal p_theta = p.rdata(PIdx::theta); - const ParticleReal p_x_mean = p_pos0*std::cos(p_theta)*p_w; - const ParticleReal p_y_mean = p_pos0*std::sin(p_theta)*p_w; -#elif defined(WARPX_DIM_XZ) - const ParticleReal p_x_mean = p_pos0*p_w; - const ParticleReal p_y_mean = 0; -#elif defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(p_pos0); - const ParticleReal p_x_mean = 0; - const ParticleReal p_y_mean = 0; -#endif - const ParticleReal p_z_mean = p.pos(index_z)*p_w; + ParticleReal p_x, p_y, p_z; + get_particle_position(p, p_x, p_y, p_z); + + const ParticleReal p_x_mean = p_x*p_w; + const ParticleReal p_y_mean = p_y*p_w; + const ParticleReal p_z_mean = p_z*p_w; const ParticleReal p_ux_mean = p_ux*p_w; const ParticleReal p_uy_mean = p_uy*p_w; @@ -292,25 +271,8 @@ void BeamRelevant::ComputeDiags (int step) const ParticleReal p_gm = std::sqrt(1.0_rt+p_us*inv_c2); const ParticleReal p_w = p.rdata(PIdx::w); -#if (defined WARPX_DIM_1D_Z) - const ParticleReal p_x = 0.0; - const ParticleReal p_y = 0.0; -#elif (defined WARPX_DIM_RZ) - const ParticleReal p_pos0 = p.pos(0); - const ParticleReal p_theta = p.rdata(PIdx::theta); - const ParticleReal p_x = p_pos0*std::cos(p_theta); - const ParticleReal p_y = p_pos0*std::sin(p_theta); -#elif (defined WARPX_DIM_XZ) - const ParticleReal p_pos0 = p.pos(0); - const ParticleReal p_x = p_pos0; - const ParticleReal p_y = 0.0; -#else - const ParticleReal p_pos0 = p.pos(0); - const ParticleReal p_pos1 = p.pos(1); - const ParticleReal p_x = p_pos0; - const ParticleReal p_y = p_pos1; -#endif - const ParticleReal p_z = p.pos(index_z); + ParticleReal p_x, p_y, p_z; + get_particle_position(p, p_x, p_y, p_z); const ParticleReal p_x_ms = (p_x-x_mean)*(p_x-x_mean)*p_w; const ParticleReal p_y_ms = (p_y-y_mean)*(p_y-y_mean)*p_w; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index f4b4e2a39a1..40ef1a088e6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -98,15 +98,9 @@ void FieldEnergy::ComputeDiags (int step) const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); - // get cell size - Geometry const & geom = warpx.Geom(lev); -#if defined(WARPX_DIM_1D_Z) - auto dV = geom.CellSize(0); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif + // get cell volume + const std::array &dx = WarpX::CellSize(lev); + const amrex::Real dV = dx[0]*dx[1]*dx[2]; #if defined(WARPX_DIM_RZ) amrex::Real const tmpEx = ComputeNorm2RZ(Ex, lev); @@ -119,6 +113,8 @@ void FieldEnergy::ComputeDiags (int step) amrex::Real const tmpBz = ComputeNorm2RZ(Bz, lev); amrex::Real const Bs = tmpBx + tmpBy + tmpBz; #else + Geometry const & geom = warpx.Geom(lev); + // compute E squared Real const tmpEx = Ex.norm2(0,geom.periodicity()); Real const tmpEy = Ey.norm2(0,geom.periodicity()); diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index f182acd5ba2..7eb16efecff 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -183,15 +183,9 @@ void FieldMomentum::ComputeDiags (int step) amrex::Real ExB_z = amrex::get<2>(r); amrex::ParallelDescriptor::ReduceRealSum({ExB_x,ExB_y,ExB_z}); - // Get cell size - amrex::Geometry const & geom = warpx.Geom(lev); -#if defined(WARPX_DIM_1D_Z) - auto dV = geom.CellSize(0); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif + // Get cell volume + const std::array &dx = WarpX::CellSize(lev); + const amrex::Real dV = dx[0]*dx[1]*dx[2]; // Save data (offset: 3 values for each refinement level) const int offset = lev*3; diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 0cc5429be7a..9adfd3f238c 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -51,7 +51,7 @@ #include #include -using namespace amrex; +using namespace amrex::literals; using namespace warpx::fields; // constructor @@ -59,7 +59,7 @@ ParticleExtrema::ParticleExtrema (const std::string& rd_name) : ReducedDiags{rd_name} { // read species name - const ParmParse pp_rd_name(rd_name); + const amrex::ParmParse pp_rd_name(rd_name); pp_rd_name.get("species",m_species_name); // get WarpX class object @@ -122,7 +122,7 @@ ParticleExtrema::ParticleExtrema (const std::string& rd_name) m_data.resize(all_diag_names.size()); - if (ParallelDescriptor::IOProcessor()) + if (amrex::ParallelDescriptor::IOProcessor()) { if ( m_write_header ) { @@ -165,16 +165,7 @@ void ParticleExtrema::ComputeDiags (int step) const auto species_names = mypc.GetSpeciesNames(); // inverse of speed of light squared - Real constexpr inv_c2 = 1.0_rt / (PhysConst::c * PhysConst::c); - - // If 2D-XZ, p.pos(1) is z, rather than p.pos(2). -#if (defined WARPX_DIM_3D) - int const index_z = 2; -#elif (defined WARPX_DIM_XZ || defined WARPX_DIM_RZ) - int const index_z = 1; -#elif (defined WARPX_DIM_1D_Z) - int const index_z = 0; -#endif + amrex::Real constexpr inv_c2 = 1.0_rt / (PhysConst::c * PhysConst::c); // loop over species for (int i_s = 0; i_s < nSpecies; ++i_s) @@ -193,172 +184,72 @@ void ParticleExtrema::ComputeDiags (int step) } using PType = typename WarpXParticleContainer::SuperParticleType; - - // xmin -#if (defined WARPX_DIM_RZ) - Real xmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::cos(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_1D_Z) - Real xmin = 0.0_rt; -#else - Real xmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0); }); -#endif - - // xmax -#if (defined WARPX_DIM_RZ) - Real xmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::cos(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_1D_Z) - Real xmax = 0.0_rt; -#else - Real xmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0); }); -#endif - - // ymin -#if (defined WARPX_DIM_RZ) - Real ymin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::sin(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_XZ || WARPX_DIM_1D_Z) - Real ymin = 0.0_rt; -#else - Real ymin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(1); }); -#endif - - // ymax -#if (defined WARPX_DIM_RZ) - Real ymax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::sin(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_XZ || WARPX_DIM_1D_Z) - Real ymax = 0.0_rt; -#else - Real ymax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(1); }); -#endif - - // zmin - Real zmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(index_z); }); - - // zmax - Real zmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(index_z); }); - - // uxmin - Real uxmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::ux); }); - - // uxmax - Real uxmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::ux); }); - - // uymin - Real uymin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uy); }); - - // uymax - Real uymax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uy); }); - - // uzmin - Real uzmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uz); }); - - // uzmax - Real uzmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uz); }); - - // gmin - Real gmin = 0.0_rt; - if ( is_photon ) { - gmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) + using OpMin = amrex::ReduceOpMin; + using OpMax = amrex::ReduceOpMax; + + amrex::ReduceOps reduce_ops; + auto posminmax = amrex::ParticleReduce>( + myspc, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(us*inv_c2); - }); - } else { - gmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) + amrex::ParticleReal x, y, z; + get_particle_position(p, x, y, z); + amrex::Real const w = p.rdata(PIdx::w); + return {w, x, y, z, w, x, y, z}; + }, + reduce_ops); + + amrex::Real wmin = amrex::get<0>(posminmax); + amrex::Real xmin = amrex::get<1>(posminmax); + amrex::Real ymin = amrex::get<2>(posminmax); + amrex::Real zmin = amrex::get<3>(posminmax); + amrex::Real wmax = amrex::get<4>(posminmax); + amrex::Real xmax = amrex::get<5>(posminmax); + amrex::Real ymax = amrex::get<6>(posminmax); + amrex::Real zmax = amrex::get<7>(posminmax); + + amrex::Real const gfactor = (is_photon ? 0._rt : 1._rt); + auto uminmax = amrex::ParticleReduce>( + myspc, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(1.0_rt + us*inv_c2); - }); - } - - // gmax - Real gmax = 0.0_rt; - if ( is_photon ) { - gmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(us*inv_c2); - }); - } else { - gmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(1.0_rt + us*inv_c2); - }); - } - - // wmin - Real wmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::w); }); - - // wmax - Real wmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::w); }); - - ParallelDescriptor::ReduceRealMin({xmin,ymin,zmin,uxmin,uymin,uzmin,gmin,wmin}); - ParallelDescriptor::ReduceRealMax({xmax,ymax,zmax,uxmax,uymax,uzmax,gmax,wmax}); + amrex::Real const ux = p.rdata(PIdx::ux); + amrex::Real const uy = p.rdata(PIdx::uy); + amrex::Real const uz = p.rdata(PIdx::uz); + amrex::Real const g = std::sqrt(gfactor + (ux*ux + uy*uy + uz*uz)*inv_c2); + return {g, ux, uy, uz, g, ux, uy, uz}; + }, + reduce_ops); + + amrex::Real gmin = amrex::get<0>(uminmax); + amrex::Real uxmin = amrex::get<1>(uminmax); + amrex::Real uymin = amrex::get<2>(uminmax); + amrex::Real uzmin = amrex::get<3>(uminmax); + amrex::Real gmax = amrex::get<4>(uminmax); + amrex::Real uxmax = amrex::get<5>(uminmax); + amrex::Real uymax = amrex::get<6>(uminmax); + amrex::Real uzmax = amrex::get<7>(uminmax); + + amrex::ParallelDescriptor::ReduceRealMin({xmin,ymin,zmin,uxmin,uymin,uzmin,gmin,wmin}); + amrex::ParallelDescriptor::ReduceRealMax({xmax,ymax,zmax,uxmax,uymax,uzmax,gmax,wmax}); #if (defined WARPX_QED) // get number of level (int) const auto level_number = WarpX::GetInstance().finestLevel(); // compute chimin and chimax - Real chimin_f = 0.0_rt; - Real chimax_f = 0.0_rt; + amrex::Real chimin_f = 0.0_rt; + amrex::Real chimax_f = 0.0_rt; if (myspc.DoQED()) { // declare chi arrays - std::vector chimin, chimax; + std::vector chimin, chimax; chimin.resize(level_number+1,0.0_rt); chimax.resize(level_number+1,0.0_rt); @@ -374,17 +265,17 @@ void ParticleExtrema::ComputeDiags (int step) { // define variables in preparation for field gathering const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - const GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // declare reduce_op - ReduceOps reduce_op; - ReduceData reduce_data(reduce_op); + amrex::ReduceOps reduce_op; + amrex::ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; // Loop over boxes @@ -408,28 +299,28 @@ void ParticleExtrema::ComputeDiags (int step) // define variables in preparation for field gathering amrex::Box box = pti.tilebox(); box.grow(ngEB); - const Dim3 lo = amrex::lbound(box); + const amrex::Dim3 lo = amrex::lbound(box); const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; const auto& ex_arr = Ex[pti].array(); const auto& ey_arr = Ey[pti].array(); const auto& ez_arr = Ez[pti].array(); const auto& bx_arr = Bx[pti].array(); const auto& by_arr = By[pti].array(); const auto& bz_arr = Bz[pti].array(); - const IndexType ex_type = Ex[pti].box().ixType(); - const IndexType ey_type = Ey[pti].box().ixType(); - const IndexType ez_type = Ez[pti].box().ixType(); - const IndexType bx_type = Bx[pti].box().ixType(); - const IndexType by_type = By[pti].box().ixType(); - const IndexType bz_type = Bz[pti].box().ixType(); + const amrex::IndexType ex_type = Ex[pti].box().ixType(); + const amrex::IndexType ey_type = Ey[pti].box().ixType(); + const amrex::IndexType ez_type = Ez[pti].box().ixType(); + const amrex::IndexType bx_type = Bx[pti].box().ixType(); + const amrex::IndexType by_type = By[pti].box().ixType(); + const amrex::IndexType bz_type = Bz[pti].box().ixType(); // evaluate reduce_op reduce_op.eval(pti.numParticles(), reduce_data, [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple { // get external fields - ParticleReal xp, yp, zp; + amrex::ParticleReal xp, yp, zp; GetPosition(i, xp, yp, zp); amrex::ParticleReal ex = Ex_external_particle; amrex::ParticleReal ey = Ey_external_particle; @@ -449,7 +340,7 @@ void ParticleExtrema::ComputeDiags (int step) dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); // compute chi - Real chi = 0.0_rt; + amrex::Real chi = 0.0_rt; if ( is_photon ) { chi = QedUtils::chi_photon(ux[i]*m, uy[i]*m, uz[i]*m, ex, ey, ez, bx, by, bz); @@ -461,13 +352,13 @@ void ParticleExtrema::ComputeDiags (int step) }); } auto val = reduce_data.value(); - chimin[lev] = get<0>(val); - chimax[lev] = get<1>(val); + chimin[lev] = amrex::get<0>(val); + chimax[lev] = amrex::get<1>(val); } chimin_f = *std::min_element(chimin.begin(), chimin.end()); chimax_f = *std::max_element(chimax.begin(), chimax.end()); - ParallelDescriptor::ReduceRealMin(chimin_f, ParallelDescriptor::IOProcessorNumber()); - ParallelDescriptor::ReduceRealMax(chimax_f, ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::ReduceRealMin(chimin_f, amrex::ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::ReduceRealMax(chimax_f, amrex::ParallelDescriptor::IOProcessorNumber()); } #endif diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 45d5dc1cdee..36827bd316a 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -911,33 +911,34 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, { auto const real_counter = std::min(write_real_comp.size(), real_comp_names.size()); +#if defined(WARPX_DIM_RZ) // reconstruct Cartesian positions for RZ simulations // r,z,theta -> x,y,z -#if defined(WARPX_DIM_RZ) - auto const * const r = soa.GetRealData(PIdx::x).data(); - auto const * const theta = soa.GetRealData(PIdx::theta).data(); + // If each comp is being written, create a temporary array, otherwise create an empty array. + std::shared_ptr const x( + new amrex::ParticleReal[(write_real_comp[0] ? numParticleOnTile : 0)], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + std::shared_ptr const y( + new amrex::ParticleReal[(write_real_comp[1] ? numParticleOnTile : 0)], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + const auto& tile = pti.GetParticleTile(); + const auto& ptd = tile.getConstParticleTileData(); + + for (int i = 0; i < numParticleOnTile; ++i) { + const auto& p = ptd.getSuperParticle(i); + amrex::ParticleReal xp, yp, zp; + get_particle_position(p, xp, yp, zp); + if (write_real_comp[0]) { x.get()[i] = xp; } + if (write_real_comp[1]) { y.get()[i] = yp; } + } if (write_real_comp[0]) { - std::shared_ptr const x( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (int i = 0; i < numParticleOnTile; ++i) { - x.get()[i] = r[i] * std::cos(theta[i]); - } - getComponentRecord(real_comp_names[0]).storeChunk( - x, {offset}, {numParticleOnTile64}); + getComponentRecord(real_comp_names[0]).storeChunk(x, {offset}, {numParticleOnTile64}); } if (write_real_comp[1]) { - std::shared_ptr const y( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (int i = 0; i < numParticleOnTile; ++i) { - y.get()[i] = r[i] * std::sin(theta[i]); - } - getComponentRecord(real_comp_names[1]).storeChunk( - y, {offset}, {numParticleOnTile64}); + getComponentRecord(real_comp_names[1]).storeChunk(y, {offset}, {numParticleOnTile64}); } #endif From 7269f09307ce608428bdfc406544c0df7284ab40 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 11 Jun 2024 01:03:29 +0200 Subject: [PATCH 16/87] slightly increase tolerance for embedded_circle test (#4968) --- Examples/Tests/embedded_circle/inputs_2d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Tests/embedded_circle/inputs_2d b/Examples/Tests/embedded_circle/inputs_2d index 7100f245917..7c76c1659da 100644 --- a/Examples/Tests/embedded_circle/inputs_2d +++ b/Examples/Tests/embedded_circle/inputs_2d @@ -5,7 +5,7 @@ max_step = 11 warpx.const_dt = 3.99e-13 warpx.do_electrostatic = labframe -warpx.self_fields_required_precision = 1e-06 +warpx.self_fields_required_precision = 2e-06 warpx.eb_implicit_function = -((x-0.00005)**2+(z-0.00005)**2-1e-05**2) warpx.eb_potential(x,y,z,t) = -10 warpx.self_fields_absolute_tolerance = 0.02 From a1eb908e897c5fb8cdad40fa0d041a1508c19d50 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 10 Jun 2024 17:05:06 -0700 Subject: [PATCH 17/87] Remove unneeded macros from AllocLevelData (#4979) --- Source/Parallelization/GuardCellManager.H | 2 +- Source/Parallelization/GuardCellManager.cpp | 2 +- Source/WarpX.cpp | 8 +------- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 341db01bef6..561456943f1 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -52,7 +52,7 @@ public: */ void Init( amrex::Real dt, - amrex::RealVect dx, + const amrex::Real * dx, bool do_subcycling, bool do_fdtd_nci_corr, short grid_type, diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 28157e09d8c..321be15df7e 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -33,7 +33,7 @@ using namespace amrex; void guardCellManager::Init ( const amrex::Real dt, - const amrex::RealVect dx, + const amrex::Real *dx, const bool do_subcycling, const bool do_fdtd_nci_corr, const short grid_type, diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index d3f91002ef4..ef81aef4482 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2120,13 +2120,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d { const bool aux_is_nodal = (field_gathering_algo == GatheringAlgo::MomentumConserving); -#if defined(WARPX_DIM_1D_Z) - const amrex::RealVect dx(WarpX::CellSize(lev)[2]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::RealVect dx = {WarpX::CellSize(lev)[0], WarpX::CellSize(lev)[2]}; -#elif defined(WARPX_DIM_3D) - const amrex::RealVect dx = {WarpX::CellSize(lev)[0], WarpX::CellSize(lev)[1], WarpX::CellSize(lev)[2]}; -#endif + const Real* dx = Geom(lev).CellSize(); // Initialize filter before guard cells manager // (needs info on length of filter's stencil) From a3cd47d19b16722721bb206ebdb4da52c4dca8af Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 11 Jun 2024 06:24:39 +0200 Subject: [PATCH 18/87] Release 24.06 (#4980) * AMReX: 24.06 * pyAMReX: 24.06 * WarpX: 24.06 --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 07422c36e15..3a03ea01c0c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd - + cd ../amrex && git checkout --detach 24.06 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 85b4a42b040..81c12a3df2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.05) +project(WarpX VERSION 24.06) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 471f6f2b6a3..18ca8370e80 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.05' +version = u'24.06' # The full version, including alpha/beta/rc tags. -release = u'24.05' +release = u'24.06' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index fbf3330ada0..9f9c8d7d736 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.05', + version = '24.06', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 8542e5f35d9..05ed74fe1b1 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 28b010126a1b39297d8a496ba81f171d8563953b +branch = 24.06 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index e2d69be9c60..41db9a15bdc 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 28b010126a1b39297d8a496ba81f171d8563953b +branch = 24.06 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index f80ec0d5af7..eda45dc9f77 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.05 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.06 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "28b010126a1b39297d8a496ba81f171d8563953b" +set(WarpX_amrex_branch "24.06" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index cdef8f277f6..18cab89e347 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.05 CONFIG REQUIRED) + find_package(pyAMReX 24.06 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "d4d409bd21bc4c48487883ac2331efdb1a6b3d61" +set(WarpX_pyamrex_branch "24.06" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index e9dbb0f2533..f397d31048e 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd - +cd amrex && git checkout --detach 24.06 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index 3d61cc5be36..670e376986f 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.05', + version = '24.06', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From c7b203e5670f57e6b9955db8c3f80d718f070b1c Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 11 Jun 2024 23:27:08 +0200 Subject: [PATCH 19/87] group initialization and cleanup of external libraries (#4964) --- Source/Evolve/WarpXEvolve.cpp | 3 +++ Source/Initialization/CMakeLists.txt | 1 + Source/Initialization/Make.package | 1 + Source/Initialization/WarpXInit.H | 30 ++++++++++++++++++++++++++ Source/Initialization/WarpXInit.cpp | 29 +++++++++++++++++++++++++ Source/main.cpp | 32 ++++++---------------------- 6 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 Source/Initialization/WarpXInit.H create mode 100644 Source/Initialization/WarpXInit.cpp diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 4ac50483ad3..8b2cc2c33f1 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -306,6 +306,9 @@ WarpX::Evolve (int numsteps) multi_diags->FilterComputePackFlushLastTimestep( istep[0] ); if (m_exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } } + + amrex::Print() << + ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("THE END"); } /* /brief Perform one PIC iteration, without subcycling diff --git a/Source/Initialization/CMakeLists.txt b/Source/Initialization/CMakeLists.txt index 8931de740ad..e5e2334fd7e 100644 --- a/Source/Initialization/CMakeLists.txt +++ b/Source/Initialization/CMakeLists.txt @@ -11,6 +11,7 @@ foreach(D IN LISTS WarpX_DIMS) TemperatureProperties.cpp VelocityProperties.cpp WarpXAMReXInit.cpp + WarpXInit.cpp WarpXInitData.cpp ) endforeach() diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index 8b4a4c1d669..831e3fc3f89 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -7,6 +7,7 @@ CEXE_sources += PlasmaInjector.cpp CEXE_sources += TemperatureProperties.cpp CEXE_sources += VelocityProperties.cpp CEXE_sources += WarpXAMReXInit.cpp +CEXE_sources += WarpXInit.cpp CEXE_sources += WarpXInitData.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization diff --git a/Source/Initialization/WarpXInit.H b/Source/Initialization/WarpXInit.H new file mode 100644 index 00000000000..ce179e2e997 --- /dev/null +++ b/Source/Initialization/WarpXInit.H @@ -0,0 +1,30 @@ +/* Copyright 2024 Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_INIT_H_ +#define WARPX_INIT_H_ + +namespace warpx::initialization +{ + /** Initializes, in the following order: + * - the MPI library through the mpi_init helper function in ablastr + * - the AMReX library + * - the FFT library through the anyfft::setup() function in ablastr + * + * @param[in] argc number of arguments from main() + * @param[in] argv argument strings from main() + */ + void initialize_external_libraries(int argc, char* argv[]); + + /** Initializes, in the following order: + * - the FFT library through the anyfft::cleanup() function in ablastr + * - the AMReX library + * - the MPI library through the mpi_finalize helper function in ablastr + */ + void finalize_external_libraries(); +} + +#endif //WARPX_INIT_H_ diff --git a/Source/Initialization/WarpXInit.cpp b/Source/Initialization/WarpXInit.cpp new file mode 100644 index 00000000000..7e00760bf30 --- /dev/null +++ b/Source/Initialization/WarpXInit.cpp @@ -0,0 +1,29 @@ +/* Copyright 2024 Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "WarpXInit.H" + +#include "Initialization/WarpXAMReXInit.H" + +#include + +#include +#include + +void warpx::initialization::initialize_external_libraries(int argc, char* argv[]) +{ + ablastr::parallelization::mpi_init(argc, argv); + warpx::initialization::amrex_init(argc, argv); + ablastr::math::anyfft::setup(); +} + +void warpx::initialization::finalize_external_libraries() +{ + ablastr::math::anyfft::cleanup(); + amrex::Finalize(); + ablastr::parallelization::mpi_finalize(); +} diff --git a/Source/main.cpp b/Source/main.cpp index 2a1b828c64f..9273cd3928b 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -8,25 +8,16 @@ */ #include "WarpX.H" -#include "Initialization/WarpXAMReXInit.H" +#include "Initialization/WarpXInit.H" #include "Utils/WarpXProfilerWrapper.H" -#include -#include #include -#include #include - int main(int argc, char* argv[]) { - ablastr::parallelization::mpi_init(argc, argv); - - warpx::initialization::amrex_init(argc, argv); - - ablastr::math::anyfft::setup(); - + warpx::initialization::initialize_external_libraries(argc, argv); { WARPX_PROFILE_VAR("main()", pmain); @@ -34,29 +25,18 @@ int main(int argc, char* argv[]) timer.record_start_time(); auto& warpx = WarpX::GetInstance(); - warpx.InitData(); - warpx.Evolve(); - - amrex::Print() << - ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("THE END"); + const auto is_warpx_verbose = warpx.Verbose(); + WarpX::Finalize(); timer.record_stop_time(); - if (warpx.Verbose()) - { + if (is_warpx_verbose){ amrex::Print() << "Total Time : " << timer.get_global_duration() << '\n'; } WARPX_PROFILE_VAR_STOP(pmain); - - WarpX::Finalize(); } - - ablastr::math::anyfft::cleanup(); - - amrex::Finalize(); - - ablastr::parallelization::mpi_finalize (); + warpx::initialization::finalize_external_libraries(); } From 2602b540efcd15f3ffebc3017312e98f46cdc7af Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 12 Jun 2024 00:20:20 +0200 Subject: [PATCH 20/87] Fix FieldProbe Check: Particle Shape (#4983) The constructor of FieldProbe might be called earlier than the WarpX class parameter init. That could lead to relying on an uninitialized particle shape static. Use the parser instead, similar to our general efforts to reduce the static members stored in the WarpX class. --- Source/Diagnostics/ReducedDiags/FieldProbe.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 7364d5989f1..1fc3b957ec9 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -154,7 +154,12 @@ FieldProbe::FieldProbe (const std::string& rd_name) ablastr::warn_manager::WarnPriority::low); } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(interp_order <= WarpX::nox , + // ensure assumption holds: we read the fields in the interpolation kernel as they are, + // without further communication of guard/ghost/halo regions + int particle_shape; + const ParmParse pp_algo("algo"); + utils::parser::getWithParser(pp_algo, "particle_shape", particle_shape); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(interp_order <= particle_shape , "Field probe interp_order should be less than or equal to algo.particle_shape"); if (ParallelDescriptor::IOProcessor()) { From 3c4e523e255dfa20663e45b416f828924366f2a1 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 13 Jun 2024 23:16:14 +0200 Subject: [PATCH 21/87] CMake: heFFTe Support (#4986) * CMake: heFFTe * Doc: Perlmutter heFFTe * Update Spack Dev Envs * Finalize After Testing Co-authored-by: Alfred Mishi <140518333+Haavaan@users.noreply.github.com> --------- Co-authored-by: Alfred Mishi <140518333+Haavaan@users.noreply.github.com> --- CMakeLists.txt | 37 ++++++++++++++++++ Docs/source/install/cmake.rst | 2 + Docs/source/install/dependencies.rst | 7 ++-- Docs/source/install/hpc/perlmutter.rst | 8 ++-- .../machines/desktop/spack-macos-openmp.yaml | 1 + Tools/machines/desktop/spack-ubuntu-cuda.yaml | 1 + .../machines/desktop/spack-ubuntu-openmp.yaml | 1 + Tools/machines/desktop/spack-ubuntu-rocm.yaml | 1 + .../install_cpu_dependencies.sh | 39 +++++++++++++++++++ .../install_gpu_dependencies.sh | 38 ++++++++++++++++++ .../perlmutter_cpu_warpx.profile.example | 2 + .../perlmutter_gpu_warpx.profile.example | 2 + cmake/WarpXFunctions.cmake | 1 + setup.py | 2 + 14 files changed, 135 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 81c12a3df2f..5e2d1ebba9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,6 +79,7 @@ option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) option(WarpX_FFT "FFT-based solvers" OFF) +option(WarpX_HEFFTE "Multi-node FFT-based solvers" OFF) option(WarpX_PYTHON "Python bindings" OFF) option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF) option(WarpX_QED "QED support (requires PICSAR)" ON) @@ -136,6 +137,10 @@ mark_as_advanced(WarpX_MPI_THREAD_MULTIPLE) option(WarpX_amrex_internal "Download & build AMReX" ON) +if(WarpX_HEFFTE AND NOT WarpX_MPI) + message(FATAL_ERROR "WarpX_HEFFTE (${WarpX_HEFFTE}) can only be used if WarpX_MPI is ON.") +endif() + # change the default build type to Release (or RelWithDebInfo) instead of Debug set_default_build_type("Release") @@ -174,6 +179,10 @@ option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT}) if(WarpX_FFT) set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE) endif() +option(ABLASTR_HEFFTE "compile AnyFFT wrappers" ${WarpX_HEFFTE}) +if(WarpX_HEFFTE) + set(ABLASTR_HEFFTE ON CACHE STRING "Multi-Node FFT-based solvers" FORCE) +endif() # this defined the variable BUILD_TESTING which is ON by default #include(CTest) @@ -215,6 +224,23 @@ if(WarpX_FFT) endif() endif() +# multi-node FFT +if(WarpX_HEFFTE) + if(WarpX_COMPUTE STREQUAL CUDA) + set(_heFFTe_COMPS CUDA) + elseif(WarpX_COMPUTE STREQUAL HIP) + set(_heFFTe_COMPS ROCM) + elseif(WarpX_COMPUTE STREQUAL SYCL) + set(_heFFTe_COMPS ONEAPI) + else() # NOACC, OMP + set(_heFFTe_COMPS FFTW) # or MKL + endif() + # note: we could also enforce GPUAWARE for CUDA and HIP, which can still be + # disabled at runtime + + find_package(Heffte REQUIRED COMPONENTS ${_heFFTe_COMPS}) +endif() + # Python if(WarpX_PYTHON) find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) @@ -455,6 +481,10 @@ foreach(D IN LISTS WarpX_DIMS) endif() endif() + if(ABLASTR_HEFFTE) + target_link_libraries(ablastr_${SD} PUBLIC Heffte::Heffte) + endif() + if(WarpX_PYTHON) target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras) if(WarpX_PYTHON_IPO) @@ -539,6 +569,13 @@ foreach(D IN LISTS WarpX_DIMS) target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_FFT) endif() + if(WarpX_HEFFTE) + target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_HEFFTE) + endif() + if(ABLASTR_HEFFTE) + target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_HEFFTE) + endif() + if(WarpX_PYTHON AND pyWarpX_VERSION_INFO) # for module __version__ target_compile_definitions(pyWarpX_${SD} PRIVATE diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index b9fd0b2be45..fde927c10e1 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -97,6 +97,7 @@ CMake Option Default & Values Descr ``WarpX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WarpX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set ``WarpX_FFT`` ON/**OFF** FFT-based solvers +``WarpX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WarpX_PYTHON`` ON/**OFF** Python bindings ``WarpX_QED`` **ON**/OFF QED support (requires PICSAR) ``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost) @@ -271,6 +272,7 @@ Environment Variable Default & Values Descr ``WARPX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WARPX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set ``WARPX_FFT`` ON/**OFF** FFT-based solvers +``WARPX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WARPX_QED`` **ON**/OFF PICSAR QED (requires PICSAR) ``WARPX_QED_TABLE_GEN`` ON/**OFF** QED table generation (requires PICSAR and Boost) ``BUILD_PARALLEL`` ``2`` Number of threads to use for parallel builds diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index ce9f9dca520..3bab32b7502 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -23,12 +23,13 @@ Optional dependencies include: - for on-node accelerated compute *one of either*: - `OpenMP 3.1+ `__: for threaded CPU execution or - - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `_) or + - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `__) or - `ROCm 5.2+ (5.5+ recommended) `__: for AMD GPU support -- `FFTW3 `_: for spectral solver (PSATD) support when running on CPU or SYCL +- `FFTW3 `__: for spectral solver (PSATD or IGF) support when running on CPU or SYCL - also needs the ``pkg-config`` tool on Unix -- `BLAS++ `_ and `LAPACK++ `_: for spectral solver (PSATD) support in RZ geometry +- `heFFTe 2.4.0+ `__ and `LAPACK++ `__: for spectral solver (PSATD) support in RZ geometry - `Boost 1.66.0+ `__: for QED lookup tables generation support - `openPMD-api 0.15.1+ `__: we automatically download and compile a copy of openPMD-api for openPMD I/O support diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst index 9612b64476d..dc5a985e99f 100644 --- a/Docs/source/install/hpc/perlmutter.rst +++ b/Docs/source/install/hpc/perlmutter.rst @@ -153,7 +153,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu - cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. @@ -164,7 +164,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu_py - cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu_py -j 16 --target pip_install .. tab-item:: CPU Nodes @@ -174,7 +174,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_cpu - cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_cpu/bin/``. @@ -184,7 +184,7 @@ Use the following :ref:`cmake commands ` to compile the applicat rm -rf build_pm_cpu_py - cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu_py -j 16 --target pip_install Now, you can :ref:`submit Perlmutter compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Tools/machines/desktop/spack-macos-openmp.yaml b/Tools/machines/desktop/spack-macos-openmp.yaml index 820cf7069fd..3ea78625b78 100644 --- a/Tools/machines/desktop/spack-macos-openmp.yaml +++ b/Tools/machines/desktop/spack-macos-openmp.yaml @@ -23,6 +23,7 @@ spack: - conduit ~fortran - fftw - hdf5 ~fortran + - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - llvm-openmp diff --git a/Tools/machines/desktop/spack-ubuntu-cuda.yaml b/Tools/machines/desktop/spack-ubuntu-cuda.yaml index 08d0c95ee4b..19b9ae12e24 100644 --- a/Tools/machines/desktop/spack-ubuntu-cuda.yaml +++ b/Tools/machines/desktop/spack-ubuntu-cuda.yaml @@ -25,6 +25,7 @@ spack: - cuda - fftw - hdf5 + - heffte - lapackpp - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-openmp.yaml b/Tools/machines/desktop/spack-ubuntu-openmp.yaml index b658f1e009d..1eb7d4074a7 100644 --- a/Tools/machines/desktop/spack-ubuntu-openmp.yaml +++ b/Tools/machines/desktop/spack-ubuntu-openmp.yaml @@ -22,6 +22,7 @@ spack: - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - fftw - hdf5 + - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-rocm.yaml b/Tools/machines/desktop/spack-ubuntu-rocm.yaml index 45c9b0f776e..7eee1baa13c 100644 --- a/Tools/machines/desktop/spack-ubuntu-rocm.yaml +++ b/Tools/machines/desktop/spack-ubuntu-rocm.yaml @@ -21,6 +21,7 @@ spack: - cmake - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - hdf5 + - heffte - hip - lapackpp - llvm-amdgpu diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index f65e43891d0..a5a7c28b85e 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -109,6 +109,45 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-cpu-build +# heFFTe +if [ -d $HOME/src/heffte ] +then + cd $HOME/src/heffte + git fetch --prune + git checkout v2.4.0 + git pull + cd - +else + git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte +fi +rm -rf ${HOME}/src/heffte-pm-cpu-build +cmake \ + -S ${HOME}/src/heffte \ + -B ${build_dir}/heffte-pm-cpu-build \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ + -DHeffte_DISABLE_GPU_AWARE_MPI=ON \ + -DHeffte_ENABLE_AVX=ON \ + -DHeffte_ENABLE_AVX512=OFF \ + -DHeffte_ENABLE_FFTW=ON \ + -DHeffte_ENABLE_CUDA=OFF \ + -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF \ + -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_SEQUENTIAL_TESTING=OFF \ + -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_TRACING=OFF \ + -DHeffte_ENABLE_PYTHON=OFF \ + -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_SWIG=OFF \ + -DHeffte_ENABLE_MAGMA=OFF +cmake --build ${build_dir}/heffte-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/heffte-pm-cpu-build + # Python ###################################################################### # diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index 9ac5800c6ce..125c63104be 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -109,6 +109,44 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-gpu-build +# heFFTe +if [ -d $HOME/src/heffte ] +then + cd $HOME/src/heffte + git fetch --prune + git checkout v2.4.0 + cd - +else + git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte +fi +rm -rf ${HOME}/src/heffte-pm-gpu-build +cmake \ + -S ${HOME}/src/heffte \ + -B ${build_dir}/heffte-pm-gpu-build \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ + -DHeffte_DISABLE_GPU_AWARE_MPI=OFF \ + -DHeffte_ENABLE_AVX=OFF \ + -DHeffte_ENABLE_AVX512=OFF \ + -DHeffte_ENABLE_FFTW=OFF \ + -DHeffte_ENABLE_CUDA=ON \ + -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF \ + -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_SEQUENTIAL_TESTING=OFF \ + -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_TRACING=OFF \ + -DHeffte_ENABLE_PYTHON=OFF \ + -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_SWIG=OFF \ + -DHeffte_ENABLE_MAGMA=OFF +cmake --build ${build_dir}/heffte-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/heffte-pm-gpu-build + # Python ###################################################################### # diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 1b0ac3182d5..3fea9b3aa39 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -19,11 +19,13 @@ export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 759df0b923a..9e1465d6c02 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -23,11 +23,13 @@ export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.2 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index a68ebca6c60..c12cedca3b2 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -453,6 +453,7 @@ function(warpx_print_summary) message(" PARTICLE PRECISION: ${WarpX_PARTICLE_PRECISION}") message(" PRECISION: ${WarpX_PRECISION}") message(" FFT Solvers: ${WarpX_FFT}") + message(" heFFTe: ${WarpX_HEFFTE}") message(" PYTHON: ${WarpX_PYTHON}") if(WarpX_PYTHON) message(" PYTHON IPO: ${WarpX_PYTHON_IPO}") diff --git a/setup.py b/setup.py index 670e376986f..6ad6fd81960 100644 --- a/setup.py +++ b/setup.py @@ -97,6 +97,7 @@ def build_extension(self, ext): '-DWarpX_PRECISION=' + WARPX_PRECISION, '-DWarpX_PARTICLE_PRECISION=' + WARPX_PARTICLE_PRECISION, '-DWarpX_FFT:BOOL=' + WARPX_FFT, + '-DWarpX_HEFFTE:BOOL=' + WARPX_HEFFTE, '-DWarpX_PYTHON:BOOL=ON', '-DWarpX_PYTHON_IPO:BOOL=' + WARPX_PYTHON_IPO, '-DWarpX_QED:BOOL=' + WARPX_QED, @@ -206,6 +207,7 @@ def build_extension(self, ext): WARPX_PRECISION = env.pop('WARPX_PRECISION', 'DOUBLE') WARPX_PARTICLE_PRECISION = env.pop('WARPX_PARTICLE_PRECISION', WARPX_PRECISION) WARPX_FFT = env.pop('WARPX_FFT', 'OFF') +WARPX_HEFFTE = env.pop('WARPX_HEFFTE', 'OFF') WARPX_QED = env.pop('WARPX_QED', 'ON') WARPX_QED_TABLE_GEN = env.pop('WARPX_QED_TABLE_GEN', 'OFF') WARPX_DIMS = env.pop('WARPX_DIMS', '1;2;RZ;3') From 828332089e58ef7686a8a8753062eb505514cd7e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Jun 2024 00:25:51 -0700 Subject: [PATCH 22/87] Resetting collisionXYY Temperature (#4999) Resetting after a recent change. Might need more investigation. --- Regression/Checksum/benchmarks_json/collisionXYZ.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 6e4b9abf965..c87bbd98e42 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,7 +6,7 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 362230.52300397365, - "T_ion": 338312.83502136066 + "T_electron": 358778.6506903592, + "T_ion": 341562.3085776466 } } From 41fc45bfb83d27e68964e83916fe154d05921f10 Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 18 Jun 2024 11:21:48 -0700 Subject: [PATCH 23/87] Cleanup deposition and gather routines (#4978) * Initial clean up of current deposition * Fix of current deposition * Further cleanup of current deposition * Cleanup SharedDepositionUtils * Clean up in charge deposition * Clean up field gather plus other fixes * Change LowerCorner to return XDim3, add InvCellSize * Change variable name in Lower and UpperCorner to avoid confusion * Add const * Update RepellingParticles CI test * Update Python_ionization CI test * Update ionization_boost and ionization_lab CI tests * Update PEC_particle CI test * Update BTD_rz CI test --- .../repelling_particles/analysis_repelling.py | 10 +- .../Checksum/benchmarks_json/BTD_rz.json | 12 +- .../benchmarks_json/PEC_particle.json | 40 +- .../benchmarks_json/Python_ionization.json | 30 +- .../benchmarks_json/RepellingParticles.json | 44 +- .../benchmarks_json/ionization_boost.json | 30 +- .../benchmarks_json/ionization_lab.json | 32 +- .../LatticeElementFinder.cpp | 2 +- .../WarpXFieldBoundaries.cpp | 4 +- .../ReducedDiags/ColliderRelevant.cpp | 8 +- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 4 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 9 +- .../ReducedDiags/ParticleExtrema.cpp | 9 +- .../SpectralSolver/SpectralFieldDataRZ.cpp | 4 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 16 +- .../Particles/Deposition/ChargeDeposition.H | 115 +--- .../Particles/Deposition/CurrentDeposition.H | 548 +++++------------- .../Deposition/SharedDepositionUtils.H | 32 +- .../Particles/ElementaryProcess/Ionization.H | 6 +- .../ElementaryProcess/Ionization.cpp | 5 +- .../ElementaryProcess/QEDPairGeneration.H | 6 +- .../ElementaryProcess/QEDPairGeneration.cpp | 5 +- .../ElementaryProcess/QEDPhotonEmission.H | 6 +- .../ElementaryProcess/QEDPhotonEmission.cpp | 7 +- Source/Particles/Gather/FieldGather.H | 205 +++---- Source/Particles/PhotonParticleContainer.cpp | 11 +- .../Particles/PhysicalParticleContainer.cpp | 33 +- .../RigidInjectedParticleContainer.cpp | 8 +- Source/Particles/WarpXParticleContainer.cpp | 81 +-- Source/Utils/WarpXUtil.cpp | 4 +- Source/WarpX.H | 5 +- Source/WarpX.cpp | 27 +- Source/ablastr/particles/DepositCharge.H | 14 +- 33 files changed, 496 insertions(+), 876 deletions(-) diff --git a/Examples/Tests/repelling_particles/analysis_repelling.py b/Examples/Tests/repelling_particles/analysis_repelling.py index ebff010fc40..bda3d74d274 100755 --- a/Examples/Tests/repelling_particles/analysis_repelling.py +++ b/Examples/Tests/repelling_particles/analysis_repelling.py @@ -36,7 +36,7 @@ # Check plotfile name specified in command line last_filename = sys.argv[1] -filename_radical = re.findall('(.*?)\d+/*$', last_filename)[0] +filename_radical = re.findall(r'(.*?)\d+/*$', last_filename)[0] # Loop through files, and extract the position and velocity of both particles x1 = [] @@ -48,10 +48,10 @@ ds = yt.load(filename) ad = ds.all_data() - x1.append( float(ad[('electron1','particle_position_x')]) ) - x2.append( float(ad[('electron2','particle_position_x')]) ) - beta1.append( float(ad[('electron1','particle_momentum_x')])/(m_e*c) ) - beta2.append( float(ad[('electron2','particle_momentum_x')])/(m_e*c) ) + x1.append( float(ad[('electron1','particle_position_x')][0]) ) + x2.append( float(ad[('electron2','particle_position_x')][0]) ) + beta1.append( float(ad[('electron1','particle_momentum_x')][0])/(m_e*c) ) + beta2.append( float(ad[('electron2','particle_momentum_x')][0])/(m_e*c) ) # Convert to numpy array x1 = np.array(x1) diff --git a/Regression/Checksum/benchmarks_json/BTD_rz.json b/Regression/Checksum/benchmarks_json/BTD_rz.json index 01e4c687292..cf83ce01346 100644 --- a/Regression/Checksum/benchmarks_json/BTD_rz.json +++ b/Regression/Checksum/benchmarks_json/BTD_rz.json @@ -1,13 +1,13 @@ { "lev=0": { - "Br": 1.8705552264208163e-08, - "Bt": 2380179.5677922587, - "Bz": 2.4079077116116535e-08, + "Br": 1.8705569155952808e-08, + "Bt": 2380179.567792259, + "Bz": 2.4079063431898616e-08, "Er": 497571119514841.25, - "Et": 7.048225464720808, - "Ez": 137058870936728.28, + "Et": 7.048223219552, + "Ez": 137058870936728.23, "jr": 0.0, "jt": 0.0, "jz": 0.0 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/PEC_particle.json b/Regression/Checksum/benchmarks_json/PEC_particle.json index 53253cebb0a..0c502a6d615 100644 --- a/Regression/Checksum/benchmarks_json/PEC_particle.json +++ b/Regression/Checksum/benchmarks_json/PEC_particle.json @@ -1,31 +1,31 @@ { "lev=0": { - "Bx": 6.52080713617892e-07, - "By": 5.323555640562078e-18, - "Bz": 1.1834601056649145e-06, - "Ex": 300.06725745121753, - "Ey": 320.82687281173185, - "Ez": 185.62208934593673, - "jx": 1.857782359527958e-06, - "jy": 257766.55567863808, + "Bx": 6.5208071361789e-07, + "By": 5.323579900863346e-18, + "Bz": 1.1834601056649094e-06, + "Ex": 300.0672574512165, + "Ey": 320.82687281173, + "Ez": 185.62208934593605, + "jx": 1.8577823595279576e-06, + "jy": 257766.5556786359, "jz": 0.0 }, + "electron": { + "particle_momentum_x": 1.1142179380730402e-32, + "particle_momentum_y": 4.5160903214571184e-36, + "particle_momentum_z": 5.735409609139057e-50, + "particle_position_x": 3.199800000000006e-05, + "particle_position_y": 6.267072536608953e-23, + "particle_position_z": 3.3938629027782103e-37, + "particle_weight": 1.0 + }, "proton": { - "particle_momentum_x": 1.0975605406855061e-33, + "particle_momentum_x": 1.097560540685506e-33, "particle_momentum_y": 1.002878875615427e-18, - "particle_momentum_z": 5.084424741580567e-51, + "particle_momentum_z": 8.484417024161229e-51, "particle_position_x": 3.1998e-05, "particle_position_y": 6.572670690061996e-06, - "particle_position_z": 5.681444715216788e-38, - "particle_weight": 1.0 - }, - "electron": { - "particle_momentum_x": 1.1142179380730476e-32, - "particle_momentum_y": 4.516090321457367e-36, - "particle_momentum_z": 2.2354046360787e-50, - "particle_position_x": 3.199800000000006e-05, - "particle_position_y": 6.267072536609136e-23, - "particle_position_z": 1.4145775327446577e-37, + "particle_position_z": 4.372139867361774e-39, "particle_weight": 1.0 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ionization.json b/Regression/Checksum/benchmarks_json/Python_ionization.json index a5e65fcf765..7918a30244d 100644 --- a/Regression/Checksum/benchmarks_json/Python_ionization.json +++ b/Regression/Checksum/benchmarks_json/Python_ionization.json @@ -3,30 +3,30 @@ "Bx": 0.0, "By": 26296568.434868, "Bz": 0.0, - "Ex": 7878103122971888.0, + "Ex": 7878103122971890.0, "Ey": 0.0, - "Ez": 3027.995293554466, - "jx": 1.2111358330750162e+16, + "Ez": 3027.738911163427, + "jx": 1.2111358330750164e+16, "jy": 0.0, - "jz": 1.3483401471475687e-07 + "jz": 1.3523766702993914e-07 + }, + "electrons": { + "particle_momentum_x": 4.4237309006289776e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.643146520891287e-18, + "particle_orig_z": 0.4306994012391905, + "particle_position_x": 0.11013574385450274, + "particle_position_y": 0.6415447480129455, + "particle_weight": 3.4456249999999997e-10 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.76132401934254e-18, + "particle_momentum_x": 1.7613240193425407e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.644887053263054e-23, + "particle_momentum_z": 3.6448870533515125e-23, "particle_orig_z": 0.128, "particle_position_x": 0.03200001189420337, "particle_position_y": 0.1280000046901387, "particle_weight": 9.999999999999999e-11 - }, - "electrons": { - "particle_momentum_x": 4.4206237143449475e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.6361297302081026e-18, - "particle_orig_z": 0.4305565137391907, - "particle_position_x": 0.11009154442846772, - "particle_position_y": 0.6414658436421568, - "particle_weight": 3.4450781249999996e-10 } } diff --git a/Regression/Checksum/benchmarks_json/RepellingParticles.json b/Regression/Checksum/benchmarks_json/RepellingParticles.json index 57b36cdc90c..0331363053a 100644 --- a/Regression/Checksum/benchmarks_json/RepellingParticles.json +++ b/Regression/Checksum/benchmarks_json/RepellingParticles.json @@ -1,32 +1,32 @@ { + "lev=0": { + "Bx": 0.0, + "By": 10603.681421961119, + "Bz": 0.0, + "Ex": 12084698236273.04, + "Ey": 0.0, + "Ez": 16282812598779.527, + "F": 21124.39775810062, + "divE": 1.495787831639977e+18, + "jx": 496020344341010.9, + "jy": 0.0, + "jz": 10.641859867595102, + "rho": 6408706.535999998 + }, "electron1": { - "particle_momentum_x": 7.303686586478214e-23, + "particle_momentum_x": 7.303686586478215e-23, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.556862195127266e-36, - "particle_position_x": 1.293487226551532e-05, - "particle_position_y": 1.777889117487881e-19, + "particle_momentum_z": 1.552083282915395e-36, + "particle_position_x": 1.293487226551533e-05, + "particle_position_y": 1.7783750855614515e-19, "particle_weight": 5000000000000.0 }, "electron2": { - "particle_momentum_x": 7.303686586477884e-23, + "particle_momentum_x": 7.303686586477887e-23, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.567350238825104e-36, + "particle_momentum_z": 1.5731022908002534e-36, "particle_position_x": 1.293487226551495e-05, - "particle_position_y": 1.797289839952283e-19, + "particle_position_y": 1.7989574553330234e-19, "particle_weight": 5000000000000.0 - }, - "lev=0": { - "Bx": 0.0, - "By": 10603.68142196111, - "Bz": 0.0, - "Ex": 12084698236273.04, - "Ey": 0.0, - "Ez": 16282812598779.53, - "F": 21124.39775810061, - "divE": 1.495787831639981e+18, - "jx": 496020344341011.4, - "jy": 0.0, - "jz": 10.93513947024536, - "rho": 6408706.535999998 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/ionization_boost.json b/Regression/Checksum/benchmarks_json/ionization_boost.json index 35b2db84a1a..dbde29d19f3 100644 --- a/Regression/Checksum/benchmarks_json/ionization_boost.json +++ b/Regression/Checksum/benchmarks_json/ionization_boost.json @@ -1,30 +1,30 @@ { "lev=0": { "Bx": 0.0, - "By": 18263123.342891, + "By": 18263123.342890993, "Bz": 0.0, - "Ex": 5472992180428804.0, + "Ex": 5472992180428803.0, "Ey": 0.0, - "Ez": 922.5589707939612, - "jx": 12440856508004.96, + "Ez": 922.2798874201462, + "jx": 12440856508004.957, "jy": 0.0, - "jz": 78616.0000011086 - }, - "electrons": { - "particle_momentum_x": 2.1386770170623736e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7287241262743654e-17, - "particle_position_x": 0.11064981928849067, - "particle_position_y": 1.7121826057017473, - "particle_weight": 3.0755014319061045e-09 + "jz": 78616.00000110897 }, "ions": { "particle_ionizationLevel": 52741.0, - "particle_momentum_x": 3.630619728387593e-18, + "particle_momentum_x": 3.630619728387586e-18, "particle_momentum_y": 0.0, "particle_momentum_z": 1.0432995297946715e-13, "particle_position_x": 0.021439968272741083, "particle_position_y": 0.4742770090248555, "particle_weight": 5.000948082142308e-10 + }, + "electrons": { + "particle_momentum_x": 2.138486151076505e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7285619821244245e-17, + "particle_position_x": 0.11066682695880013, + "particle_position_y": 1.712069774427624, + "particle_weight": 3.0755014319061037e-09 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/ionization_lab.json b/Regression/Checksum/benchmarks_json/ionization_lab.json index 82984141b59..99d93020532 100644 --- a/Regression/Checksum/benchmarks_json/ionization_lab.json +++ b/Regression/Checksum/benchmarks_json/ionization_lab.json @@ -3,29 +3,29 @@ "Bx": 0.0, "By": 26296568.434868, "Bz": 0.0, - "Ex": 7878103122971888.0, + "Ex": 7878103122971890.0, "Ey": 0.0, - "Ez": 3027.995293554467, - "jx": 1.2111358330750162e+16, + "Ez": 3027.7389111634266, + "jx": 1.2111358330750164e+16, "jy": 0.0, - "jz": 1.3575651149270143e-07 + "jz": 1.355962687103163e-07 + }, + "electrons": { + "particle_momentum_x": 4.4195594812814575e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.6520195422151263e-18, + "particle_orig_z": 0.43004993872230357, + "particle_position_x": 0.11017033205040265, + "particle_position_y": 0.6412498318709239, + "particle_weight": 3.442265625e-10 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.7613240052056494e-18, + "particle_momentum_x": 1.761324005205651e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.6186967375178554e-23, + "particle_momentum_z": 3.618696737610014e-23, "particle_position_x": 0.0320000118071032, "particle_position_y": 0.12800000462171646, "particle_weight": 9.999999999999999e-11 - }, - "electrons": { - "particle_momentum_x": 4.428135211584547e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.6627518442038486e-18, - "particle_orig_z": 0.43043207622230534, - "particle_position_x": 0.11023346490802505, - "particle_position_y": 0.6417814148540429, - "particle_weight": 3.44453125e-10 } -} \ No newline at end of file +} diff --git a/Source/AcceleratorLattice/LatticeElementFinder.cpp b/Source/AcceleratorLattice/LatticeElementFinder.cpp index ba2e51715fb..ec784049760 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.cpp +++ b/Source/AcceleratorLattice/LatticeElementFinder.cpp @@ -60,7 +60,7 @@ LatticeElementFinder::UpdateIndices (int const lev, amrex::MFIter const& a_mfi, // Note that the current box is used since the box may have been updated since // the initialization in InitElementFinder. const amrex::Box box = a_mfi.tilebox(); - m_zmin = WarpX::LowerCorner(box, lev, 0._rt)[2]; + m_zmin = WarpX::LowerCorner(box, lev, 0._rt).z; m_time = warpx.gett_new(lev); if (accelerator_lattice.h_quad.nelements > 0) { diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index a5e0004325a..2b063b99a15 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -201,8 +201,8 @@ WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex // Lower corner of tile box physical domain // Note that this is done before the tilebox.grow so that // these do not include the guard cells. - const std::array& xyzmin = LowerCorner(tilebox, lev, 0._rt); - const amrex::Real rmin = xyzmin[0]; + const amrex::XDim3 xyzmin = LowerCorner(tilebox, lev, 0._rt); + const amrex::Real rmin = xyzmin.x; // Skip blocks that don't touch the axis if (rmin > 0._rt) { continue; } diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 434ab1d7949..1b0c74cf7fa 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -441,8 +441,7 @@ void ColliderRelevant::ComputeDiags (int step) const int lev = 0; // define variables in preparation for field gathering - const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); @@ -478,8 +477,7 @@ void ColliderRelevant::ComputeDiags (int step) amrex::Box box = pti.tilebox(); box.grow(ngEB); const amrex::Dim3 lo = amrex::lbound(box); - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const amrex::Array4 & ex_arr = Ex[pti].array(); const amrex::Array4 & ey_arr = Ey[pti].array(); const amrex::Array4 & ez_arr = Ez[pti].array(); @@ -515,7 +513,7 @@ void ColliderRelevant::ComputeDiags (int step) ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); // compute chi amrex::Real chi = 0.0_rt; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index 40ef1a088e6..9c0fac101a2 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -178,10 +178,10 @@ FieldEnergy::ComputeNorm2RZ(const amrex::MultiFab& field, const int lev) amrex::Box tb = convert(tilebox, field.ixType().toIntVect()); // Lower corner of tile box physical domain - const std::array& xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); const Dim3 lo = lbound(tilebox); const Dim3 hi = ubound(tilebox); - const Real rmin = xyzmin[0] + (tb.ixType().nodeCentered(0) ? 0._rt : 0.5_rt*dr); + const Real rmin = xyzmin.x + (tb.ixType().nodeCentered(0) ? 0._rt : 0.5_rt*dr); const int irmin = lo.x; const int irmax = hi.x; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 1fc3b957ec9..1113075f4cd 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -489,11 +489,8 @@ void FieldProbe::ComputeDiags (int step) auto * const AMREX_RESTRICT idcpu = pti.GetStructOfArrays().GetIdCPUData().data(); - const auto &xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const std::array &dx = WarpX::CellSize(lev); - - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); + const amrex::XDim3 dinv = WarpX::InvCellSize(lev); const Dim3 lo = lbound(box); // Temporarily defining modes and interp outside ParallelFor to avoid GPU compilation errors. @@ -514,7 +511,7 @@ void FieldProbe::ComputeDiags (int step) doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, arrEx, arrEy, arrEz, arrBx, arrBy, arrBz, Extype, Eytype, Eztype, Bxtype, Bytype, Bztype, - dx_arr, xyzmin_arr, lo, temp_modes, + dinv, xyzmin, lo, temp_modes, temp_interp_order, false); //Calculate the Poynting Vector S diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 9adfd3f238c..811fee7a9de 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -264,8 +264,8 @@ void ParticleExtrema::ComputeDiags (int step) for (int lev = 0; lev <= level_number; ++lev) { // define variables in preparation for field gathering - const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); @@ -300,8 +300,7 @@ void ParticleExtrema::ComputeDiags (int step) amrex::Box box = pti.tilebox(); box.grow(ngEB); const amrex::Dim3 lo = amrex::lbound(box); - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const auto& ex_arr = Ex[pti].array(); const auto& ey_arr = Ey[pti].array(); const auto& ez_arr = Ez[pti].array(); @@ -337,7 +336,7 @@ void ParticleExtrema::ComputeDiags (int step) ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); // compute chi amrex::Real chi = 0.0_rt; diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp index b26248fb2d0..d295d61b43a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp @@ -199,8 +199,8 @@ SpectralFieldDataRZ::SpectralFieldDataRZ (const int lev, #endif // Create the Hankel transformer for each box. - std::array xmax = WarpX::UpperCorner(mfi.tilebox(), lev, 0._rt); - multi_spectral_hankel_transformer[mfi] = SpectralHankelTransformer(grid_size[0], n_rz_azimuthal_modes, xmax[0]); + amrex::XDim3 const xyzmax = WarpX::UpperCorner(mfi.tilebox(), lev, 0._rt); + multi_spectral_hankel_transformer[mfi] = SpectralHankelTransformer(grid_size[0], n_rz_azimuthal_modes, xyzmax.x); } } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 6631c1b88ee..602a2666b27 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -1245,11 +1245,11 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu // Lower corner of tile box physical domain // Note that this is done before the tilebox.grow so that // these do not include the guard cells. - const std::array& xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); - const Real rmin = xyzmin[0]; - const Real rminr = xyzmin[0] + (tbr.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); - const Real rmint = xyzmin[0] + (tbt.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); - const Real rminz = xyzmin[0] + (tbz.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); + const Real rmin = xyzmin.x; + const Real rminr = xyzmin.x + (tbr.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const Real rmint = xyzmin.x + (tbt.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const Real rminz = xyzmin.x + (tbz.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); const Dim3 lo = lbound(tilebox); const int irmin = lo.x; @@ -1416,10 +1416,10 @@ WarpX::ApplyInverseVolumeScalingToChargeDensity (MultiFab* Rho, int lev) // Lower corner of tile box physical domain // Note that this is done before the tilebox.grow so that // these do not include the guard cells. - const std::array& xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); const Dim3 lo = lbound(tilebox); - const Real rmin = xyzmin[0]; - const Real rminr = xyzmin[0] + (tb.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const Real rmin = xyzmin.x; + const Real rminr = xyzmin.x + (tb.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); const int irmin = lo.x; const int ishift = (rminr > rmin ? 1 : 0); diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index 2912b769c57..d3d36dbfbda 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -28,8 +28,8 @@ ion_lev is a null pointer. * \param rho_fab FArrayBox of charge density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. - * \param dx 3D cell size - * \param xyzmin Physical lower bounds of domain. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. @@ -40,37 +40,19 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, const int* ion_lev, amrex::FArrayBox& rho_fab, long np_to_deposit, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]] int n_rz_azimuthal_modes) { using namespace amrex; // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; amrex::Array4 const& rho_arr = rho_fab.array(); amrex::IntVect const rho_type = rho_fab.box().type(); @@ -98,19 +80,12 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, // Get particle position in grid coordinates #if defined(WARPX_DIM_RZ) const amrex::Real rp = std::sqrt(xp*xp + yp*yp); - amrex::Real costheta; - amrex::Real sintheta; - if (rp > 0.) { - costheta = xp/rp; - sintheta = yp/rp; - } else { - costheta = 1._rt; - sintheta = 0._rt; - } + const amrex::Real costheta = (rp > 0._rt ? xp/rp : 1._rt); + const amrex::Real sintheta = (rp > 0._rt ? yp/rp : 0._rt); const Complex xy0 = Complex{costheta, sintheta}; - const amrex::Real x = (rp - xmin)*dxi; + const amrex::Real x = (rp - xyzmin.x)*dinv.x; #else - const amrex::Real x = (xp - xmin)*dxi; + const amrex::Real x = (xp - xyzmin.x)*dinv.x; #endif // Compute shape factor along x @@ -125,7 +100,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, #endif //defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) #if defined(WARPX_DIM_3D) // y direction - const amrex::Real y = (yp - ymin)*dyi; + const amrex::Real y = (yp - xyzmin.y)*dinv.y; amrex::Real sy[depos_order + 1] = {0._rt}; int j = 0; if (rho_type[1] == NODE) { @@ -135,7 +110,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, } #endif // z direction - const amrex::Real z = (zp - zmin)*dzi; + const amrex::Real z = (zp - xyzmin.z)*dinv.z; amrex::Real sz[depos_order + 1] = {0._rt}; int k = 0; if (rho_type[WARPX_ZINDEX] == NODE) { @@ -151,8 +126,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, &rho_arr(lo.x+k+iz, 0, 0, 0), sz[iz]*wq); } -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ amrex::Gpu::Atomic::AddNoRet( @@ -182,10 +156,6 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, #endif } ); - -#ifndef WARPX_DIM_RZ - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif } /* \brief Perform charge deposition on a tile using shared memory @@ -198,8 +168,8 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, * \param rho_fab FArrayBox of charge density, either full array or tile. * \param ix_type * \param np_to_deposit Number of particles for which charge is deposited. - * \param dx 3D cell size - * \param xyzmin Physical lower bounds of domain. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. @@ -216,11 +186,11 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio amrex::FArrayBox& rho_fab, const amrex::IntVect& ix_type, const long np_to_deposit, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, + [[maybe_unused]]const int n_rz_azimuthal_modes, const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, @@ -239,26 +209,8 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; amrex::Array4 const& rho_arr = rho_fab.array(); auto rho_box = rho_fab.box(); @@ -269,7 +221,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio // Loop over particles and deposit into rho_fab #if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) - const auto dxiarr = geom.InvCellSizeArray(); const auto plo = geom.ProbLoArray(); const auto domain = geom.Domain(); @@ -316,15 +267,14 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio ParticleReal xp, yp, zp; GetPosition(permutation[bin_start], xp, yp, zp); #if defined(WARPX_DIM_3D) - IntVect iv = IntVect(int(amrex::Math::floor((xp-plo[0])*dxiarr[0])), - int(amrex::Math::floor((yp-plo[1])*dxiarr[1])), - int(amrex::Math::floor((zp-plo[2])*dxiarr[2]))); + IntVect iv = IntVect(int(amrex::Math::floor((xp-plo[0])*dinv.x)), + int(amrex::Math::floor((yp-plo[1])*dinv.y)), + int(amrex::Math::floor((zp-plo[2])*dinv.z))); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - IntVect iv = IntVect( - int(amrex::Math::floor((xp-plo[0])*dxiarr[0])), - int(amrex::Math::floor((zp-plo[1])*dxiarr[1]))); + IntVect iv = IntVect(int(amrex::Math::floor((xp-plo[0])*dinv.x)), + int(amrex::Math::floor((zp-plo[1])*dinv.z))); #elif defined(WARPX_DIM_1D_Z) - IntVect iv = IntVect(int(amrex::Math::floor((zp-plo[0])*dxiarr[0]))); + IntVect iv = IntVect(int(amrex::Math::floor((zp-plo[0])*dinv.z))); #endif iv += domain.smallEnd(); getTileIndex(iv, box, true, bin_size, buffer_box); @@ -381,9 +331,9 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio sintheta = 0._rt; } const Complex xy0 = Complex{costheta, sintheta}; - const amrex::Real x = (rp - xmin)*dxi; + const amrex::Real x = (rp - xyzmin.x)*dinv.x; #else - const amrex::Real x = (xp - xmin)*dxi; + const amrex::Real x = (xp - xyzmin.x)*dinv.x; #endif // Compute shape factor along x @@ -398,7 +348,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif //defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) #if defined(WARPX_DIM_3D) // y direction - const amrex::Real y = (yp - ymin)*dyi; + const amrex::Real y = (yp - xyzmin.y)*dinv.y; amrex::Real sy[depos_order + 1] = {0._rt}; int j = 0; if (rho_type[1] == NODE) { @@ -408,7 +358,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio } #endif // z direction - const amrex::Real z = (zp - zmin)*dzi; + const amrex::Real z = (zp - xyzmin.z)*dinv.z; amrex::Real sz[depos_order + 1] = {0._rt}; int k = 0; if (rho_type[WARPX_ZINDEX] == NODE) { @@ -424,8 +374,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio &buf(lo.x+k+iz, 0, 0, 0), sz[iz]*wq); } -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ amrex::Gpu::Atomic::AddNoRet( @@ -462,10 +411,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif // defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) } ); - -#ifndef WARPX_DIM_RZ - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif } #endif // WARPX_CHARGEDEPOSITION_H_ diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 3dce7d28534..cb56c559bc0 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -24,6 +24,7 @@ #include #include #include +#include #include /** @@ -38,17 +39,17 @@ * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dzi, dxi, dyi The inverse cell sizes - * \param zmin, xmin, ymin The lower bounds of the domain + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param invvol The inverse volume of a grid cell * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. */ template AMREX_GPU_HOST_DEVICE AMREX_INLINE -void doDepositionShapeNKernel(const amrex::ParticleReal xp, - const amrex::ParticleReal yp, - const amrex::ParticleReal zp, +void doDepositionShapeNKernel([[maybe_unused]] const amrex::ParticleReal xp, + [[maybe_unused]] const amrex::ParticleReal yp, + [[maybe_unused]] const amrex::ParticleReal zp, const amrex::ParticleReal wq, const amrex::ParticleReal vx, const amrex::ParticleReal vy, @@ -60,28 +61,14 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, amrex::IntVect const& jy_type, amrex::IntVect const& jz_type, const amrex::Real relative_time, - AMREX_D_DECL(const amrex::Real dzi, - const amrex::Real dxi, - const amrex::Real dyi), - AMREX_D_DECL(const amrex::Real zmin, - const amrex::Real xmin, - const amrex::Real ymin), + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Real invvol, const amrex::Dim3 lo, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif -#if defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(xp, yp); -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(yp); -#endif - constexpr int zdir = WARPX_ZINDEX; constexpr int NODE = amrex::IndexType::NODE; constexpr int CELL = amrex::IndexType::CELL; @@ -92,15 +79,8 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, const amrex::Real xpmid = xp + relative_time*vx; const amrex::Real ypmid = yp + relative_time*vy; const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); - amrex::Real costheta; - amrex::Real sintheta; - if (rpmid > 0._rt) { - costheta = xpmid/rpmid; - sintheta = ypmid/rpmid; - } else { - costheta = 1._rt; - sintheta = 0._rt; - } + const amrex::Real costheta = (rpmid > 0._rt ? xpmid/rpmid : 1._rt); + const amrex::Real sintheta = (rpmid > 0._rt ? ypmid/rpmid : 0._rt); const Complex xy0 = Complex{costheta, sintheta}; const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); @@ -115,12 +95,13 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, #if (AMREX_SPACEDIM >= 2) // x direction // Get particle position after 1/2 push back in position -#if defined(WARPX_DIM_RZ) // Keep these double to avoid bug in single precision - const double xmid = (rpmid - xmin)*dxi; +#if defined(WARPX_DIM_RZ) + const double xmid = (rpmid - xyzmin.x)*dinv.x; #else - const double xmid = ((xp - xmin) + relative_time*vx)*dxi; + const double xmid = ((xp - xyzmin.x) + relative_time*vx)*dinv.x; #endif + // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current // sx_j[xyz] shape factor along x for the centering of each current // There are only two possible centerings, node or cell centered, so at most only two shape factor @@ -155,7 +136,7 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, #if defined(WARPX_DIM_3D) // y direction // Keep these double to avoid bug in single precision - const double ymid = ((yp - ymin) + relative_time*vy)*dyi; + const double ymid = ((yp - xyzmin.y) + relative_time*vy)*dinv.y; double sy_node[depos_order + 1] = {0.}; double sy_cell[depos_order + 1] = {0.}; int k_node = 0; @@ -182,7 +163,8 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, // z direction // Keep these double to avoid bug in single precision - const double zmid = ((zp - zmin) + relative_time*vz)*dzi; + constexpr int zdir = WARPX_ZINDEX; + const double zmid = ((zp - xyzmin.z) + relative_time*vz)*dinv.z; double sz_node[depos_order + 1] = {0.}; double sz_cell[depos_order + 1] = {0.}; int l_node = 0; @@ -282,7 +264,7 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -300,41 +282,19 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jz_fab, long np_to_deposit, amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]]int n_rz_azimuthal_modes) { using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; const amrex::Real clightsq = 1.0_rt/PhysConst::c/PhysConst::c; @@ -367,9 +327,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, jx_type, jy_type, jz_type, - relative_time, - AMREX_D_DECL(dzi, dxi, dyi), - AMREX_D_DECL(zmin, xmin, ymin), + relative_time, dinv, xyzmin, invvol, lo, n_rz_azimuthal_modes); } @@ -391,7 +349,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, ion_lev is a null pointer. * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -411,40 +369,19 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, amrex::FArrayBox& jy_fab, amrex::FArrayBox& jz_fab, const long np_to_deposit, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes) + [[maybe_unused]]const int n_rz_azimuthal_modes) { using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; amrex::Array4 const& jx_arr = jx_fab.array(); amrex::Array4 const& jy_arr = jy_fab.array(); @@ -483,9 +420,7 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, const amrex::Real relative_time = 0._rt; doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, jx_type, jy_type, jz_type, - relative_time, - AMREX_D_DECL(dzi, dxi, dyi), - AMREX_D_DECL(zmin, xmin, ymin), + relative_time, dinv, xyzmin, invvol, lo, n_rz_azimuthal_modes); } @@ -509,7 +444,7 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -527,8 +462,8 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jz_fab, long np_to_deposit, const amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, int n_rz_azimuthal_modes, @@ -634,7 +569,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, { const unsigned int ip = permutation[ip_orig]; depositComponent(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_buff, jx_type, - relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, + relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, ip, zdir, NODE, CELL, 0); } @@ -649,7 +584,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, { const unsigned int ip = permutation[ip_orig]; depositComponent(GetPosition, wp, uxp, uyp, uzp, ion_lev, jy_buff, jy_type, - relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, + relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, ip, zdir, NODE, CELL, 1); } @@ -664,7 +599,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, { const unsigned int ip = permutation[ip_orig]; depositComponent(GetPosition, wp, uxp, uyp, uzp, ion_lev, jz_buff, jz_type, - relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, + relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, ip, zdir, NODE, CELL, 2); } @@ -675,7 +610,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, // Note, you should never reach this part of the code. This funcion cannot be called unless // using HIP/CUDA, and those things are checked prior //don't use any args - ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, a_bins, box, geom, a_tbox_max_size); + ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, a_bins, box, geom, a_tbox_max_size); WARPX_ABORT_WITH_MESSAGE("Shared memory only implemented for HIP/CUDA"); #endif } @@ -698,7 +633,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -717,53 +652,28 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, long np_to_deposit, amrex::Real dt, amrex::Real relative_time, - const std::array& dx, - std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]]int n_rz_azimuthal_modes) { using namespace amrex; using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - ignore_unused(n_rz_azimuthal_modes); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; +#if !defined(WARPX_DIM_3D) + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; #endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - Real const invdtdx = 1.0_rt / (dt*dx[1]*dx[2]); - Real const invdtdy = 1.0_rt / (dt*dx[0]*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]*dx[1]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - Real const invdtdx = 1.0_rt / (dt*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[0]*dx[2]); -#elif defined(WARPX_DIM_1D_Z) - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[2]); -#endif + amrex::XDim3 const invdtd = amrex::XDim3{(1.0_rt/dt)*dinv.y*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.y}; -#if defined(WARPX_DIM_RZ) - Complex const I = Complex{0._rt, 1._rt}; -#endif + Real constexpr clightsq = 1.0_rt / ( PhysConst::c * PhysConst::c ); - Real const clightsq = 1.0_rt / ( PhysConst::c * PhysConst::c ); #if !defined(WARPX_DIM_1D_Z) Real constexpr one_third = 1.0_rt / 3.0_rt; Real constexpr one_sixth = 1.0_rt / 6.0_rt; @@ -778,7 +688,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, + uyp[ip]*uyp[ip]*clightsq + uzp[ip]*uzp[ip]*clightsq); - // wqx, wqy wqz are particle current in each direction Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; @@ -787,14 +696,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, ParticleReal xp, yp, zp; GetPosition(ip, xp, yp, zp); -#if !defined(WARPX_DIM_1D_Z) - Real const wqx = wq*invdtdx; -#endif -#if defined(WARPX_DIM_3D) - Real const wqy = wq*invdtdy; -#endif - Real const wqz = wq*invdtdz; - // computes current and old position in grid units #if defined(WARPX_DIM_RZ) Real const xp_new = xp + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv; @@ -806,51 +707,33 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); Real const rp_mid = std::sqrt(xp_mid*xp_mid + yp_mid*yp_mid); Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); - Real costheta_new, sintheta_new; - if (rp_new > 0._rt) { - costheta_new = xp_new/rp_new; - sintheta_new = yp_new/rp_new; - } else { - costheta_new = 1._rt; - sintheta_new = 0._rt; - } - amrex::Real costheta_mid, sintheta_mid; - if (rp_mid > 0._rt) { - costheta_mid = xp_mid/rp_mid; - sintheta_mid = yp_mid/rp_mid; - } else { - costheta_mid = 1._rt; - sintheta_mid = 0._rt; - } - amrex::Real costheta_old, sintheta_old; - if (rp_old > 0._rt) { - costheta_old = xp_old/rp_old; - sintheta_old = yp_old/rp_old; - } else { - costheta_old = 1._rt; - sintheta_old = 0._rt; - } + const amrex::Real costheta_mid = (rp_mid > 0._rt ? xp_mid/rp_mid : 1._rt); + const amrex::Real sintheta_mid = (rp_mid > 0._rt ? yp_mid/rp_mid : 0._rt); + const amrex::Real costheta_new = (rp_new > 0._rt ? xp_new/rp_new : 1._rt); + const amrex::Real sintheta_new = (rp_new > 0._rt ? yp_new/rp_new : 0._rt); + const amrex::Real costheta_old = (rp_old > 0._rt ? xp_old/rp_old : 1._rt); + const amrex::Real sintheta_old = (rp_old > 0._rt ? yp_old/rp_old : 0._rt); const Complex xy_new0 = Complex{costheta_new, sintheta_new}; const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; const Complex xy_old0 = Complex{costheta_old, sintheta_old}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; #else #if !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp - xmin + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv)*dxi; - double const x_old = x_new - dt*dxi*uxp[ip]*gaminv; + double const x_new = (xp - xyzmin.x + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv)*dinv.x; + double const x_old = x_new - dt*dinv.x*uxp[ip]*gaminv; #endif #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp - ymin + (relative_time + 0.5_rt*dt)*uyp[ip]*gaminv)*dyi; - double const y_old = y_new - dt*dyi*uyp[ip]*gaminv; + double const y_new = (yp - xyzmin.y + (relative_time + 0.5_rt*dt)*uyp[ip]*gaminv)*dinv.y; + double const y_old = y_new - dt*dinv.y*uyp[ip]*gaminv; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp - zmin + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv)*dzi; - double const z_old = z_new - dt*dzi*uzp[ip]*gaminv; + double const z_new = (zp - xyzmin.z + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv)*dinv.z; + double const z_old = z_new - dt*dinv.z*uzp[ip]*gaminv; #if defined(WARPX_DIM_RZ) Real const vy = (-uxp[ip]*sintheta_mid + uyp[ip]*costheta_mid)*gaminv; @@ -861,6 +744,12 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, Real const vy = uyp[ip]*gaminv; #endif + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + const Compute_shape_factor< depos_order > compute_shape_factor; + const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + // Shape factor arrays // Note that there are extra values above and below // to possibly hold the factor for the old particle @@ -869,30 +758,17 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #if !defined(WARPX_DIM_1D_Z) double sx_new[depos_order + 3] = {0.}; double sx_old[depos_order + 3] = {0.}; -#endif -#if defined(WARPX_DIM_3D) - // Keep these double to avoid bug in single precision - double sy_new[depos_order + 3] = {0.}; - double sy_old[depos_order + 3] = {0.}; -#endif - // Keep these double to avoid bug in single precision - double sz_new[depos_order + 3] = {0.}; - double sz_old[depos_order + 3] = {0.}; - - // --- Compute shape factors - // Compute shape factors for position as they are now and at old positions - // [ijk]_new: leftmost grid point that the particle touches - const Compute_shape_factor< depos_order > compute_shape_factor; - const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; - -#if !defined(WARPX_DIM_1D_Z) const int i_new = compute_shape_factor(sx_new+1, x_new); const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); #endif #if defined(WARPX_DIM_3D) + double sy_new[depos_order + 3] = {0.}; + double sy_old[depos_order + 3] = {0.}; const int j_new = compute_shape_factor(sy_new+1, y_new); const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); #endif + double sz_new[depos_order + 3] = {0.}; + double sz_old[depos_order + 3] = {0.}; const int k_new = compute_shape_factor(sz_new+1, z_new); const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); @@ -917,7 +793,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int j=djl; j<=depos_order+2-dju; j++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*( + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*( one_third*(sy_new[j]*sz_new[k] + sy_old[j]*sz_old[k]) +one_sixth*(sy_new[j]*sz_old[k] + sy_old[j]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdxi); @@ -928,7 +804,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdyj = 0._rt; for (int j=djl; j<=depos_order+1-dju; j++) { - sdyj += wqy*(sy_old[j] - sy_new[j])*( + sdyj += wq*invdtd.y*(sy_old[j] - sy_new[j])*( one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdyj); @@ -939,7 +815,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*( + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*( one_third*(sx_new[i]*sy_new[j] + sx_old[i]*sy_old[j]) +one_sixth*(sx_new[i]*sy_old[j] + sx_old[i]*sy_new[j])); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdzk); @@ -952,7 +828,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int k=dkl; k<=depos_order+2-dku; k++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdxi); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -973,6 +849,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdyj); #if defined(WARPX_DIM_RZ) + Complex const I = Complex{0._rt, 1._rt}; Complex xy_new = xy_new0; Complex xy_mid = xy_mid0; Complex xy_old = xy_old0; @@ -980,7 +857,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { // The factor 2 comes from the normalization of the modes // The minus sign comes from the different convention with respect to Davidson et al. - const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(amrex::Real)imode + const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xyzmin.x*dinv.x)*wq*invdtd.x/(amrex::Real)imode *(Complex(sx_new[i]*sz_new[k], 0._rt)*(xy_new - xy_mid) + Complex(sx_old[i]*sz_old[k], 0._rt)*(xy_mid - xy_old)); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); @@ -995,7 +872,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int i=dil; i<=depos_order+2-diu; i++) { Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdzk); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -1021,7 +898,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, } amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+k_new-1+k, 0, 0, 0), sdzk); } #endif @@ -1047,16 +924,16 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. */ template -void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, - const amrex::ParticleReal * const yp_n, - const amrex::ParticleReal * const zp_n, +void doChargeConservingDepositionShapeNImplicit ([[maybe_unused]]const amrex::ParticleReal * const xp_n, + [[maybe_unused]]const amrex::ParticleReal * const yp_n, + [[maybe_unused]]const amrex::ParticleReal * const zp_n, const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, [[maybe_unused]]const amrex::ParticleReal * const uxp_n, @@ -1071,49 +948,25 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex; -#if !defined(WARPX_DIM_RZ) - ignore_unused(n_rz_azimuthal_modes); -#endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - Real const invdtdx = 1.0_rt / (dt*dx[1]*dx[2]); - Real const invdtdy = 1.0_rt / (dt*dx[0]*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]*dx[1]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - Real const invdtdx = 1.0_rt / (dt*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[0]*dx[2]); -#elif defined(WARPX_DIM_1D_Z) - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[2]); +#if !defined(WARPX_DIM_3D) + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; #endif -#if defined(WARPX_DIM_RZ) - Complex const I = Complex{0._rt, 1._rt}; -#endif + amrex::XDim3 const invdtd = amrex::XDim3{(1.0_rt/dt)*dinv.y*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.y}; #if !defined(WARPX_DIM_1D_Z) Real constexpr one_third = 1.0_rt / 3.0_rt; @@ -1137,7 +990,6 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); #endif - // wqx, wqy wqz are particle current in each direction Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; @@ -1148,24 +1000,12 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #if !defined(WARPX_DIM_1D_Z) ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; -#else - ignore_unused(xp_n); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; -#else - ignore_unused(yp_n); #endif ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; -#if !defined(WARPX_DIM_1D_Z) - amrex::Real const wqx = wq*invdtdx; -#endif -#if defined(WARPX_DIM_3D) - amrex::Real const wqy = wq*invdtdy; -#endif - amrex::Real const wqz = wq*invdtdz; - // computes current and old position in grid units #if defined(WARPX_DIM_RZ) amrex::Real const xp_new = xp_np1; @@ -1177,51 +1017,33 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; - amrex::Real costheta_new, sintheta_new; - if (rp_new > 0._rt) { - costheta_new = xp_new/rp_new; - sintheta_new = yp_new/rp_new; - } else { - costheta_new = 1._rt; - sintheta_new = 0._rt; - } - amrex::Real costheta_mid, sintheta_mid; - if (rp_mid > 0._rt) { - costheta_mid = xp_mid/rp_mid; - sintheta_mid = yp_mid/rp_mid; - } else { - costheta_mid = 1._rt; - sintheta_mid = 0._rt; - } - amrex::Real costheta_old, sintheta_old; - if (rp_old > 0._rt) { - costheta_old = xp_old/rp_old; - sintheta_old = yp_old/rp_old; - } else { - costheta_old = 1._rt; - sintheta_old = 0._rt; - } + const amrex::Real costheta_mid = (rp_mid > 0._rt ? xp_mid/rp_mid : 1._rt); + const amrex::Real sintheta_mid = (rp_mid > 0._rt ? yp_mid/rp_mid : 0._rt); + const amrex::Real costheta_new = (rp_new > 0._rt ? xp_new/rp_new : 1._rt); + const amrex::Real sintheta_new = (rp_new > 0._rt ? yp_new/rp_new : 0._rt); + const amrex::Real costheta_old = (rp_old > 0._rt ? xp_old/rp_old : 1._rt); + const amrex::Real sintheta_old = (rp_old > 0._rt ? yp_old/rp_old : 0._rt); const Complex xy_new0 = Complex{costheta_new, sintheta_new}; const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; const Complex xy_old0 = Complex{costheta_old, sintheta_old}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; #else #if !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n[ip] - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n[ip] - xyzmin.x)*dinv.x; #endif #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n[ip] - ymin)*dyi; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n[ip] - xyzmin.y)*dinv.y; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n[ip] - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n[ip] - xyzmin.z)*dinv.z; #if defined(WARPX_DIM_RZ) amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; @@ -1232,6 +1054,12 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con amrex::Real const vy = uyp_nph[ip]*gaminv; #endif + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + const Compute_shape_factor< depos_order > compute_shape_factor; + const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + // Shape factor arrays // Note that there are extra values above and below // to possibly hold the factor for the old particle @@ -1240,30 +1068,17 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #if !defined(WARPX_DIM_1D_Z) double sx_new[depos_order + 3] = {0.}; double sx_old[depos_order + 3] = {0.}; -#endif -#if defined(WARPX_DIM_3D) - // Keep these double to avoid bug in single precision - double sy_new[depos_order + 3] = {0.}; - double sy_old[depos_order + 3] = {0.}; -#endif - // Keep these double to avoid bug in single precision - double sz_new[depos_order + 3] = {0.}; - double sz_old[depos_order + 3] = {0.}; - - // --- Compute shape factors - // Compute shape factors for position as they are now and at old positions - // [ijk]_new: leftmost grid point that the particle touches - const Compute_shape_factor< depos_order > compute_shape_factor; - const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; - -#if !defined(WARPX_DIM_1D_Z) const int i_new = compute_shape_factor(sx_new+1, x_new); const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); #endif #if defined(WARPX_DIM_3D) + double sy_new[depos_order + 3] = {0.}; + double sy_old[depos_order + 3] = {0.}; const int j_new = compute_shape_factor(sy_new+1, y_new); const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); #endif + double sz_new[depos_order + 3] = {0.}; + double sz_old[depos_order + 3] = {0.}; const int k_new = compute_shape_factor(sz_new+1, z_new); const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); @@ -1288,7 +1103,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int j=djl; j<=depos_order+2-dju; j++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*( + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*( one_third*(sy_new[j]*sz_new[k] + sy_old[j]*sz_old[k]) +one_sixth*(sy_new[j]*sz_old[k] + sy_old[j]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdxi); @@ -1299,7 +1114,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdyj = 0._rt; for (int j=djl; j<=depos_order+1-dju; j++) { - sdyj += wqy*(sy_old[j] - sy_new[j])*( + sdyj += wq*invdtd.y*(sy_old[j] - sy_new[j])*( one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdyj); @@ -1310,7 +1125,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*( + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*( one_third*(sx_new[i]*sy_new[j] + sx_old[i]*sy_old[j]) +one_sixth*(sx_new[i]*sy_old[j] + sx_old[i]*sy_new[j])); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdzk); @@ -1323,7 +1138,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int k=dkl; k<=depos_order+2-dku; k++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdxi); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -1344,6 +1159,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdyj); #if defined(WARPX_DIM_RZ) + Complex const I = Complex{0._rt, 1._rt}; Complex xy_new = xy_new0; Complex xy_mid = xy_mid0; Complex xy_old = xy_old0; @@ -1351,7 +1167,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { // The factor 2 comes from the normalization of the modes // The minus sign comes from the different convention with respect to Davidson et al. - const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(amrex::Real)imode + const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xyzmin.x*dinv.x)*wq*invdtd.x/(amrex::Real)imode *(Complex(sx_new[i]*sz_new[k], 0._rt)*(xy_new - xy_mid) + Complex(sx_old[i]*sz_old[k], 0._rt)*(xy_mid - xy_old)); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); @@ -1366,7 +1182,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int i=dil; i<=depos_order+2-diu; i++) { Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdzk); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -1392,7 +1208,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con } amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+k_new-1+k, 0, 0, 0), sdzk); } #endif @@ -1420,16 +1236,16 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. */ template -void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, - const amrex::ParticleReal * const yp_n, - const amrex::ParticleReal * const zp_n, +void doVillasenorDepositionShapeNImplicit ([[maybe_unused]]const amrex::ParticleReal * const xp_n, + [[maybe_unused]]const amrex::ParticleReal * const yp_n, + [[maybe_unused]]const amrex::ParticleReal * const zp_n, const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, [[maybe_unused]]const amrex::ParticleReal * const uxp_n, @@ -1444,42 +1260,21 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex; -#if !defined(WARPX_DIM_RZ) - ignore_unused(n_rz_azimuthal_modes); -#endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - Real const invvol = 1.0_rt / (dx[0]*dx[1]*dx[2]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - Real const invvol = 1.0_rt / (dx[0]*dx[2]); -#elif defined(WARPX_DIM_1D_Z) - Real const invvol = 1.0_rt / (dx[2]); -#endif + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; -#if !defined(WARPX_DIM_1D_Z) +#if (AMREX_SPACEDIM > 1) Real constexpr one_third = 1.0_rt / 3.0_rt; Real constexpr one_sixth = 1.0_rt / 6.0_rt; #endif @@ -1502,7 +1297,6 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); #endif - // wqx, wqy wqz are particle current in each direction Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; @@ -1513,13 +1307,9 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ #if !defined(WARPX_DIM_1D_Z) ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; -#else - ignore_unused(xp_n); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; -#else - ignore_unused(yp_n); #endif ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; @@ -1545,14 +1335,14 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; amrex::Real const vx = (rp_new - rp_old)/dt; amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; #elif defined(WARPX_DIM_XZ) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n[ip] - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n[ip] - xyzmin.x)*dinv.x; amrex::Real const vx = (xp_np1 - xp_n[ip])/dt; amrex::Real const vy = uyp_nph[ip]*gaminv; #elif defined(WARPX_DIM_1D_Z) @@ -1560,17 +1350,17 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ amrex::Real const vy = uyp_nph[ip]*gaminv; #elif defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n[ip] - xmin)*dxi; - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n[ip] - ymin)*dyi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n[ip] - xyzmin.x)*dinv.x; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n[ip] - xyzmin.y)*dinv.y; amrex::Real const vx = (xp_np1 - xp_n[ip])/dt; amrex::Real const vy = (yp_np1 - yp_n[ip])/dt; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n[ip] - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n[ip] - xyzmin.z)*dinv.z; amrex::Real const vz = (zp_np1 - zp_n[ip])/dt; // Define velocity kernals to deposit @@ -2057,7 +1847,7 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param[in] dx 3D cell size + * \param[in] dinv 3D cell size inverse * \param[in] xyzmin 3D lower bounds of physical domain * \param[in] lo Dimension-agnostic lower bounds of index domain * \param[in] q Species charge @@ -2076,57 +1866,37 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, long np_to_deposit, amrex::Real dt, amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]]int n_rz_azimuthal_modes) { using namespace amrex::literals; #if defined(WARPX_DIM_RZ) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q); WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in RZ geometry"); #endif #if defined(WARPX_DIM_1D_Z) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); - WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in cartesian 1D geometry"); + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q); + WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in 1D geometry"); #endif #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) - amrex::ignore_unused(n_rz_azimuthal_modes); // If ion_lev is a null pointer, then do_ionization=0, else do_ionization=1 const bool do_ionization = ion_lev; - // Inverse cell volume in each direction - const amrex::Real dxi = 1._rt / dx[0]; - const amrex::Real dzi = 1._rt / dx[2]; -#if defined(WARPX_DIM_3D) - const amrex::Real dyi = 1._rt / dx[1]; -#endif - // Inverse of time step const amrex::Real invdt = 1._rt / dt; - // Total inverse cell volume -#if defined(WARPX_DIM_XZ) - const amrex::Real invvol = dxi * dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real invvol = dxi * dyi * dzi; -#endif - - // Lower bound of physical domain in each direction - const amrex::Real xmin = xyzmin[0]; - const amrex::Real zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; // Allocate temporary arrays #if defined(WARPX_DIM_3D) @@ -2172,23 +1942,18 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, yp += relative_time * vy; zp += relative_time * vz; - // Particle current densities -#if defined(WARPX_DIM_XZ) - const amrex::Real wqy = wq * vy * invvol; -#endif - // Current and old particle positions in grid units // Keep these double to avoid bug in single precision. - double const x_new = (xp - xmin + 0.5_rt*dt*vx) * dxi; - double const x_old = (xp - xmin - 0.5_rt*dt*vx) * dxi; + double const x_new = (xp - xyzmin.x + 0.5_rt*dt*vx) * dinv.x; + double const x_old = (xp - xyzmin.x - 0.5_rt*dt*vx) * dinv.x; #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision. - double const y_new = (yp - ymin + 0.5_rt*dt*vy) * dyi; - double const y_old = (yp - ymin - 0.5_rt*dt*vy) * dyi; + double const y_new = (yp - xyzmin.y + 0.5_rt*dt*vy) * dinv.y; + double const y_old = (yp - xyzmin.y - 0.5_rt*dt*vy) * dinv.y; #endif // Keep these double to avoid bug in single precision. - double const z_new = (zp - zmin + 0.5_rt*dt*vz) * dzi; - double const z_old = (zp - zmin - 0.5_rt*dt*vz) * dzi; + double const z_new = (zp - xyzmin.z + 0.5_rt*dt*vz) * dinv.z; + double const z_old = (zp - xyzmin.z - 0.5_rt*dt*vz) * dinv.z; // Shape factor arrays for current and old positions (nodal) // Keep these double to avoid bug in single precision. @@ -2235,6 +2000,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, // Deposit current into Dx_arr, Dy_arr and Dz_arr #if defined(WARPX_DIM_XZ) + const amrex::Real wqy = wq * vy * invvol; for (int k=0; k<=depos_order; k++) { for (int i=0; i<=depos_order; i++) { diff --git a/Source/Particles/Deposition/SharedDepositionUtils.H b/Source/Particles/Deposition/SharedDepositionUtils.H index 1e2294be3a2..e6deb51b5df 100644 --- a/Source/Particles/Deposition/SharedDepositionUtils.H +++ b/Source/Particles/Deposition/SharedDepositionUtils.H @@ -74,8 +74,8 @@ void depositComponent (const GetParticlePosition& GetPosition, amrex::Array4 const& j_buff, amrex::IntVect const j_type, const amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 dinv, + const amrex::XDim3 xyzmin, const amrex::Dim3 lo, const amrex::Real q, const int n_rz_azimuthal_modes, @@ -91,26 +91,8 @@ void depositComponent (const GetParticlePosition& GetPosition, // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; const amrex::Real clightsq = 1.0_rt/PhysConst::c/PhysConst::c; @@ -171,9 +153,9 @@ void depositComponent (const GetParticlePosition& GetPosition, // Get particle position after 1/2 push back in position #if defined(WARPX_DIM_RZ) // Keep these double to avoid bug in single precision - const double xmid = (rpmid - xmin)*dxi; + const double xmid = (rpmid - xyzmin.x)*dinv.x; #else - const double xmid = ((xp - xmin) + relative_time*vx)*dxi; + const double xmid = ((xp - xyzmin.x) + relative_time*vx)*dinv.x; #endif // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current // sx_j[xyz] shape factor along x for the centering of each current @@ -203,7 +185,7 @@ void depositComponent (const GetParticlePosition& GetPosition, #if defined(WARPX_DIM_3D) // y direction // Keep these double to avoid bug in single precision - const double ymid = ((yp - ymin) + relative_time*vy)*dyi; + const double ymid = ((yp - xyzmin.y) + relative_time*vy)*dinv.y; double sy_node[depos_order + 1] = {0.}; double sy_cell[depos_order + 1] = {0.}; int k_node = 0; @@ -224,7 +206,7 @@ void depositComponent (const GetParticlePosition& GetPosition, // z direction // Keep these double to avoid bug in single precision - const double zmid = ((zp - zmin) + relative_time*vz)*dzi; + const double zmid = ((zp - xyzmin.z) + relative_time*vz)*dinv.z; double sz_node[depos_order + 1] = {0.}; double sz_cell[depos_order + 1] = {0.}; int l_node = 0; diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index cee7ec07eb5..6f98c18959a 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -62,8 +62,8 @@ struct IonizationFilterFunc amrex::IndexType m_by_type; amrex::IndexType m_bz_type; - amrex::GpuArray m_dx_arr; - amrex::GpuArray m_xyzmin_arr; + amrex::XDim3 m_dinv; + amrex::XDim3 m_xyzmin; bool m_galerkin_interpolation; int m_nox; @@ -117,7 +117,7 @@ struct IonizationFilterFunc doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, - m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_dinv, m_xyzmin, m_lo, m_n_rz_azimuthal_modes, m_nox, m_galerkin_interpolation); m_get_externalEB(i, ex, ey, ez, bx, by, bz); diff --git a/Source/Particles/ElementaryProcess/Ionization.cpp b/Source/Particles/ElementaryProcess/Ionization.cpp index b7b91e4d4e3..0568e302eec 100644 --- a/Source/Particles/ElementaryProcess/Ionization.cpp +++ b/Source/Particles/ElementaryProcess/Ionization.cpp @@ -76,11 +76,10 @@ IonizationFilterFunc::IonizationFilterFunc (const WarpXParIter& a_pti, int lev, box.grow(ngEB); const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - m_dx_arr = {dx[0], dx[1], dx[2]}; + m_dinv = amrex::XDim3{1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + m_xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_lo = amrex::lbound(box); } diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index d550c5f03ee..180fdf0fb35 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -144,7 +144,7 @@ public: doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, - m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_dinv, m_xyzmin, m_lo, m_n_rz_azimuthal_modes, m_nox, m_galerkin_interpolation); //Despite the names of the variables, positrons and electrons @@ -198,8 +198,8 @@ private: amrex::IndexType m_by_type; amrex::IndexType m_bz_type; - amrex::GpuArray m_dx_arr; - amrex::GpuArray m_xyzmin_arr; + amrex::XDim3 m_dinv; + amrex::XDim3 m_xyzmin; bool m_galerkin_interpolation; int m_nox; diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp b/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp index 2b380d454f4..82546e43ef4 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp @@ -64,11 +64,10 @@ PairGenerationTransformFunc (BreitWheelerGeneratePairs const generate_functor, box.grow(ngEB); const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - m_dx_arr = {dx[0], dx[1], dx[2]}; + m_dinv = amrex::XDim3{1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + m_xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_lo = amrex::lbound(box); } diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index dd5f57a8f24..514526374bd 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -156,7 +156,7 @@ public: doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, - m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_dinv, m_xyzmin, m_lo, m_n_rz_azimuthal_modes, m_nox, m_galerkin_interpolation); auto& ux = src.m_rdata[PIdx::ux][i_src]; @@ -209,8 +209,8 @@ private: amrex::IndexType m_by_type; amrex::IndexType m_bz_type; - amrex::GpuArray m_dx_arr; - amrex::GpuArray m_xyzmin_arr; + amrex::XDim3 m_dinv; + amrex::XDim3 m_xyzmin; bool m_galerkin_interpolation; int m_nox; diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp b/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp index 077a4659ce5..688c3c0184d 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp @@ -67,13 +67,10 @@ PhotonEmissionTransformFunc (QuantumSynchrotronGetOpticalDepth opt_depth_functor box.grow(ngEB); const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - m_dx_arr = {dx[0], dx[1], dx[2]}; + m_dinv = amrex::XDim3{1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - - + m_xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_lo = amrex::lbound(box); } diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index b5bd4376ba1..4b4590b8642 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -28,16 +28,16 @@ * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. * \param ex_type,ey_type,ez_type IndexType of the electric field * \param bx_type,by_type,bz_type IndexType of the magnetic field - * \param dx 3D cell spacing - * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void doGatherShapeN (const amrex::ParticleReal xp, - const amrex::ParticleReal yp, - const amrex::ParticleReal zp, +void doGatherShapeN ([[maybe_unused]] const amrex::ParticleReal xp, + [[maybe_unused]] const amrex::ParticleReal yp, + [[maybe_unused]] const amrex::ParticleReal zp, amrex::ParticleReal& Exp, amrex::ParticleReal& Eyp, amrex::ParticleReal& Ezp, @@ -56,41 +56,13 @@ void doGatherShapeN (const amrex::ParticleReal xp, const amrex::IndexType bx_type, const amrex::IndexType by_type, const amrex::IndexType bz_type, - const amrex::GpuArray& dx, - const amrex::GpuArray& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex; -#if defined(WARPX_DIM_XZ) - amrex::ignore_unused(yp); -#endif - -#if defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(xp,yp); -#endif - -#ifndef WARPX_DIM_RZ - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif - -#if (AMREX_SPACEDIM >= 2) - const amrex::Real dxi = 1.0_rt/dx[0]; -#endif - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_3D) - const amrex::Real dyi = 1.0_rt/dx[1]; -#endif - -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; - constexpr int zdir = WARPX_ZINDEX; constexpr int NODE = amrex::IndexType::NODE; constexpr int CELL = amrex::IndexType::CELL; @@ -105,9 +77,9 @@ void doGatherShapeN (const amrex::ParticleReal xp, // Get particle position #ifdef WARPX_DIM_RZ const amrex::Real rp = std::sqrt(xp*xp + yp*yp); - const amrex::Real x = (rp - xmin)*dxi; + const amrex::Real x = (rp - xyzmin.x)*dinv.x; #else - const amrex::Real x = (xp-xmin)*dxi; + const amrex::Real x = (xp-xyzmin.x)*dinv.x; #endif // j_[eb][xyz] leftmost grid point in x that the particle touches for the centering of each current @@ -151,7 +123,7 @@ void doGatherShapeN (const amrex::ParticleReal xp, #if defined(WARPX_DIM_3D) // y direction - const amrex::Real y = (yp-ymin)*dyi; + const amrex::Real y = (yp-xyzmin.y)*dinv.y; amrex::Real sy_node[depos_order + 1]; amrex::Real sy_cell[depos_order + 1]; amrex::Real sy_node_v[depos_order + 1 - galerkin_interpolation]; @@ -187,7 +159,7 @@ void doGatherShapeN (const amrex::ParticleReal xp, #endif // z direction - const amrex::Real z = (zp-zmin)*dzi; + const amrex::Real z = (zp-xyzmin.z)*dinv.z; amrex::Real sz_node[depos_order + 1]; amrex::Real sz_cell[depos_order + 1]; amrex::Real sz_node_v[depos_order + 1 - galerkin_interpolation]; @@ -464,10 +436,10 @@ void doGatherShapeN (const amrex::ParticleReal xp, * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. * \param Ex_type,Ey_type,Ez_type IndexType of the electric field * \param Bx_type,By_type,Bz_type IndexType of the magnetic field - * \param dx 3D cell spacing - * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. - * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -496,8 +468,8 @@ void doGatherShapeNEsirkepovStencilImplicit ( [[maybe_unused]] const amrex::IndexType Bx_type, [[maybe_unused]] const amrex::IndexType By_type, [[maybe_unused]] const amrex::IndexType Bz_type, - const amrex::GpuArray& dx, - const amrex::GpuArray& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes) { @@ -506,19 +478,6 @@ void doGatherShapeNEsirkepovStencilImplicit ( ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; - #if !defined(WARPX_DIM_1D_Z) Real constexpr one_third = 1.0_rt / 3.0_rt; Real constexpr one_sixth = 1.0_rt / 6.0_rt; @@ -553,23 +512,23 @@ void doGatherShapeNEsirkepovStencilImplicit ( } const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; #else #if !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n - xyzmin.x)*dinv.x; #endif #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n - ymin)*dyi; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n - xyzmin.y)*dinv.y; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n - xyzmin.z)*dinv.z; // Shape factor arrays // Note that there are extra values above and below @@ -894,8 +853,8 @@ void doGatherShapeNEsirkepovStencilImplicit ( * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. * \param Ex_type,Ey_type,Ez_type IndexType of the electric field * \param Bx_type,By_type,Bz_type IndexType of the magnetic field - * \param dx 3D cell spacing - * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ @@ -926,8 +885,8 @@ void doGatherPicnicShapeN ( [[maybe_unused]] const amrex::IndexType Bx_type, [[maybe_unused]] const amrex::IndexType By_type, [[maybe_unused]] const amrex::IndexType Bz_type, - const amrex::GpuArray& dx, - const amrex::GpuArray& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes) { @@ -936,19 +895,6 @@ void doGatherPicnicShapeN ( ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; - #if !defined(WARPX_DIM_1D_Z) const ParticleReal xp_np1 = 2._prt*xp_nph - xp_n; #endif @@ -983,25 +929,25 @@ void doGatherPicnicShapeN ( } const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; - double const x_bar = (rp_mid - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; + double const x_bar = (rp_mid - xyzmin.x)*dinv.x; #elif !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n - xmin)*dxi; - double const x_bar = (xp_nph - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n - xyzmin.x)*dinv.x; + double const x_bar = (xp_nph - xyzmin.x)*dinv.x; #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n - ymin)*dyi; - double const y_bar = (yp_nph - ymin)*dyi; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n - xyzmin.y)*dinv.y; + double const y_bar = (yp_nph - xyzmin.y)*dinv.y; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n - zmin)*dzi; - double const z_bar = (zp_nph - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n - xyzmin.z)*dinv.z; + double const z_bar = (zp_nph - xyzmin.z)*dinv.z; // 1) Determine the number of segments. // 2) Loop over segments and gather electric field. @@ -1567,8 +1513,8 @@ void doGatherPicnicShapeN ( * \param exfab,eyfab,ezfab Array4 of the electric field, either full array or tile. * \param bxfab,byfab,bzfab Array4 of the magnetic field, either full array or tile. * \param np_to_gather Number of particles for which field is gathered. - * \param dx 3D cell size - * \param xyzmin Physical lower bounds of domain. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ @@ -1585,15 +1531,12 @@ void doGatherShapeN(const GetParticlePosition& getPosition, amrex::FArrayBox const * const byfab, amrex::FArrayBox const * const bzfab, const long np_to_gather, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const int n_rz_azimuthal_modes) { - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -1622,7 +1565,7 @@ void doGatherShapeN(const GetParticlePosition& getPosition, xp, yp, zp, Exp[ip], Eyp[ip], Ezp[ip], Bxp[ip], Byp[ip], Bzp[ip], ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } ); } @@ -1637,8 +1580,8 @@ void doGatherShapeN(const GetParticlePosition& getPosition, * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. * \param ex_type,ey_type,ez_type IndexType of the electric field * \param bx_type,by_type,bz_type IndexType of the magnetic field - * \param dx_arr 3D cell spacing - * \param xyzmin_arr Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry * \param nox order of the particle shape function @@ -1666,8 +1609,8 @@ void doGatherShapeN (const amrex::ParticleReal xp, const amrex::IndexType bx_type, const amrex::IndexType by_type, const amrex::IndexType bz_type, - const amrex::GpuArray& dx_arr, - const amrex::GpuArray& xyzmin_arr, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes, const int nox, @@ -1678,44 +1621,44 @@ void doGatherShapeN (const amrex::ParticleReal xp, doGatherShapeN<1,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeN<2,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeN<3,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeN<4,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } else { if (nox == 1) { doGatherShapeN<1,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeN<2,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeN<3,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeN<4,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } } @@ -1732,8 +1675,8 @@ void doGatherShapeN (const amrex::ParticleReal xp, * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. * \param ex_type,ey_type,ez_type IndexType of the electric field * \param bx_type,by_type,bz_type IndexType of the magnetic field - * \param dx_arr 3D cell spacing - * \param xyzmin_arr Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry * \param nox order of the particle shape function @@ -1766,8 +1709,8 @@ void doGatherShapeNImplicit ( const amrex::IndexType bx_type, const amrex::IndexType by_type, const amrex::IndexType bz_type, - const amrex::GpuArray& dx_arr, - const amrex::GpuArray& xyzmin_arr, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes, const int nox, @@ -1779,25 +1722,25 @@ void doGatherShapeNImplicit ( Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeNEsirkepovStencilImplicit<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeNEsirkepovStencilImplicit<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeNEsirkepovStencilImplicit<4>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } else if (depos_type==3) { // CurrentDepositionAlgo::Villasenor @@ -1806,25 +1749,25 @@ void doGatherShapeNImplicit ( Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherPicnicShapeN<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherPicnicShapeN<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherPicnicShapeN<4>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } else if (depos_type==1) { // CurrentDepositionAlgo::Direct @@ -1832,22 +1775,22 @@ void doGatherShapeNImplicit ( doGatherShapeN<1,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeN<2,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeN<3,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeN<4,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } } diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index aa9f04be224..1f15d5210f5 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -93,8 +93,8 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, int lev, int gather_lev, amrex::Real dt, ScaleFields /*scaleFields*/, DtType a_dt_type) { - // Get cell size on gather_lev - const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + // Get inverse cell size on gather_lev + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(gather_lev,0)); // Get box from which field is gathered. // If not gathering from the finest level, the box is coarsened. @@ -142,7 +142,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); const Dim3 lo = lbound(box); @@ -150,9 +150,6 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -201,7 +198,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, doGatherShapeN(x, y, z, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e2be4f948ca..89ae435882b 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2558,7 +2558,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, if (do_not_push) { return; } - const std::array& dx = WarpX::CellSize(std::max(lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev,0)); #ifdef AMREX_USE_OMP #pragma omp parallel @@ -2590,7 +2590,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const Dim3 lo = lbound(box); @@ -2598,9 +2598,6 @@ PhysicalParticleContainer::PushP (int lev, Real dt, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab.array(); amrex::Array4 const& ey_arr = eyfab.array(); amrex::Array4 const& ez_arr = ezfab.array(); @@ -2657,7 +2654,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); } @@ -2755,7 +2752,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, if (np_to_push == 0) { return; } // Get cell size on gather_lev - const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(gather_lev,0)); // Get box from which field is gathered. // If not gathering from the finest level, the box is coarsened. @@ -2783,7 +2780,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); const Dim3 lo = lbound(box); @@ -2791,9 +2788,6 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -2908,7 +2902,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); } @@ -3008,7 +3002,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, if (np_to_push == 0) { return; } // Get cell size on gather_lev - const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(gather_lev,0)); // Get box from which field is gathered. // If not gathering from the finest level, the box is coarsened. @@ -3035,7 +3029,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); const Dim3 lo = lbound(box); @@ -3043,9 +3037,6 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -3157,9 +3148,9 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, amrex::ParticleReal dxp, dxp_save; amrex::ParticleReal dyp, dyp_save; amrex::ParticleReal dzp, dzp_save; - auto idxg2 = static_cast(1._rt/(dx[0]*dx[0])); - auto idyg2 = static_cast(1._rt/(dx[1]*dx[1])); - auto idzg2 = static_cast(1._rt/(dx[2]*dx[2])); + auto idxg2 = static_cast(dinv.x*dinv.x); + auto idyg2 = static_cast(dinv.y*dinv.y); + auto idzg2 = static_cast(dinv.z*dinv.z); amrex::ParticleReal step_norm = 1._prt; for (int iter=0; iter& dx = WarpX::CellSize(std::max(lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev,0)); #ifdef AMREX_USE_OMP #pragma omp parallel @@ -369,7 +369,6 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const Dim3 lo = lbound(box); @@ -377,8 +376,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); amrex::Array4 const& ex_arr = exfab.array(); amrex::Array4 const& ey_arr = eyfab.array(); @@ -445,7 +443,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 4800d9e209f..bdce18b7b2b 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -396,7 +396,8 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, amrex::numParticlesOutOfRange(pti, range) == 0, "Particles shape does not fit within tile (CPU) or guard cells (GPU) used for current deposition"); - const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(depos_lev,0)); + const amrex::ParticleReal q = this->charge; WARPX_PROFILE_VAR_NS("WarpXParticleContainer::DepositCurrent::Sorting", blp_sort); @@ -467,7 +468,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, // Note that this includes guard cells since it is after tilebox.ngrow const Dim3 lo = lbound(tilebox); // Take into account Galilean shift - const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, 0.5_rt*dt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, depos_lev, 0.5_rt*dt); if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { @@ -544,28 +545,28 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doDepositionSharedShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 2){ doDepositionSharedShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 3){ doDepositionSharedShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 4){ doDepositionSharedShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } @@ -580,25 +581,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doEsirkepovDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { @@ -625,7 +626,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doChargeConservingDepositionShapeNImplicit<2>( @@ -633,7 +634,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doChargeConservingDepositionShapeNImplicit<3>( @@ -641,7 +642,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doChargeConservingDepositionShapeNImplicit<4>( @@ -649,7 +650,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } @@ -678,7 +679,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVillasenorDepositionShapeNImplicit<2>( @@ -686,7 +687,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVillasenorDepositionShapeNImplicit<3>( @@ -694,7 +695,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVillasenorDepositionShapeNImplicit<4>( @@ -702,7 +703,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } @@ -717,25 +718,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doVayDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVayDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVayDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVayDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else { // Direct deposition @@ -744,25 +745,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { @@ -775,7 +776,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeNImplicit<2>( @@ -783,7 +784,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeNImplicit<3>( @@ -791,7 +792,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeNImplicit<4>( @@ -799,7 +800,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } @@ -931,7 +932,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, "Particles shape does not fit within tile (CPU) or guard cells (GPU) used for charge deposition"); amrex::ignore_unused(range); // In case the assertion isn't compiled - const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); const Real q = this->charge; WARPX_PROFILE_VAR_NS("WarpXParticleContainer::DepositCharge::Sorting", blp_sort); @@ -981,12 +981,13 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // Take into account Galilean shift const Real dt = warpx.getdt(lev); const amrex::Real time_shift_delta = (icomp == 0 ? 0.0_rt : dt); - const std::array& xyzmin = WarpX::LowerCorner( - tilebox, depos_lev, time_shift_delta); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); // Indices of the lower bound const Dim3 lo = lbound(tilebox); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(depos_lev,0)); + // HACK - sort particles by bin here. WARPX_PROFILE_VAR_START(blp_sort); amrex::DenseBins bins; @@ -1088,25 +1089,25 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, if (WarpX::nox == 1){ doChargeDepositionSharedShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 2){ doChargeDepositionSharedShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 3){ doChargeDepositionSharedShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 4){ doChargeDepositionSharedShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); @@ -1125,7 +1126,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // note: this is smaller than rho->nGrowVect() for PSATD const amrex::IntVect& ng_rho = warpx.get_ng_depos_rho(); - const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); amrex::IntVect ref_ratio; if (lev == depos_lev) { ref_ratio = IntVect(AMREX_D_DECL(1, 1, 1 )); @@ -1150,7 +1150,8 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // Take into account Galilean shift const amrex::Real dt = warpx.getdt(lev); const amrex::Real time_shift_delta = (icomp == 0 ? 0.0_rt : dt); - const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(depos_lev,0)); AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noy); AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noz); @@ -1158,7 +1159,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, ablastr::particles::deposit_charge( pti, wp, this->charge, ion_lev, rho, local_rho[thread_num], - WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, + WarpX::noz, dinv, xyzmin, WarpX::n_rz_azimuthal_modes, ng_rho, depos_lev, ref_ratio, offset, np_to_deposit, icomp, nc); diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index d2691685c53..2ef4ee55d6e 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -232,8 +232,8 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) for(amrex::MFIter mfi(mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.tilebox(); // Get box lower and upper physical z bound, and dz - const amrex::Real zmin_box = WarpX::LowerCorner(bx, lev, 0._rt)[2]; - const amrex::Real zmax_box = WarpX::UpperCorner(bx, lev, 0._rt)[2]; + const amrex::Real zmin_box = WarpX::LowerCorner(bx, lev, 0._rt).z; + const amrex::Real zmax_box = WarpX::UpperCorner(bx, lev, 0._rt).z; const amrex::Real dz = WarpX::CellSize(lev)[2]; // Get box lower index in the z direction #if defined(WARPX_DIM_3D) diff --git a/Source/WarpX.H b/Source/WarpX.H index a0a1379d9e2..768f2d486dc 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -895,6 +895,7 @@ public: amrex::Vector& output_geom ) const; static std::array CellSize (int lev); + static amrex::XDim3 InvCellSize (int lev); static amrex::RealBox getRealBox(const amrex::Box& bx, int lev); /** @@ -905,7 +906,7 @@ public: * (when v_galilean is not zero) * \return An array of the position coordinates */ - static std::array LowerCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); + static amrex::XDim3 LowerCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); /** * \brief Return the upper corner of the box in real units. * \param bx The box @@ -914,7 +915,7 @@ public: * (when v_galilean is not zero) * \return An array of the position coordinates */ - static std::array UpperCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); + static amrex::XDim3 UpperCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); static amrex::IntVect RefRatio (int lev); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index ef81aef4482..d1f9eb16f47 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2846,6 +2846,13 @@ WarpX::CellSize (int lev) #endif } +amrex::XDim3 +WarpX::InvCellSize (int lev) +{ + std::array dx = WarpX::CellSize(lev); + return {1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; +} + amrex::RealBox WarpX::getRealBox(const Box& bx, int lev) { @@ -2854,13 +2861,13 @@ WarpX::getRealBox(const Box& bx, int lev) return( grid_box ); } -std::array +amrex::XDim3 WarpX::LowerCorner(const Box& bx, const int lev, const amrex::Real time_shift_delta) { auto & warpx = GetInstance(); const RealBox grid_box = getRealBox( bx, lev ); - const Real* xyzmin = grid_box.lo(); + const Real* grid_min = grid_box.lo(); const amrex::Real cur_time = warpx.gett_new(lev); const amrex::Real time_shift = (cur_time + time_shift_delta - warpx.time_of_last_gal_shift); @@ -2869,23 +2876,23 @@ WarpX::LowerCorner(const Box& bx, const int lev, const amrex::Real time_shift_de warpx.m_v_galilean[2]*time_shift }; #if defined(WARPX_DIM_3D) - return { xyzmin[0] + galilean_shift[0], xyzmin[1] + galilean_shift[1], xyzmin[2] + galilean_shift[2] }; + return { grid_min[0] + galilean_shift[0], grid_min[1] + galilean_shift[1], grid_min[2] + galilean_shift[2] }; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - return { xyzmin[0] + galilean_shift[0], std::numeric_limits::lowest(), xyzmin[1] + galilean_shift[2] }; + return { grid_min[0] + galilean_shift[0], std::numeric_limits::lowest(), grid_min[1] + galilean_shift[2] }; #elif defined(WARPX_DIM_1D_Z) - return { std::numeric_limits::lowest(), std::numeric_limits::lowest(), xyzmin[0] + galilean_shift[2] }; + return { std::numeric_limits::lowest(), std::numeric_limits::lowest(), grid_min[0] + galilean_shift[2] }; #endif } -std::array +amrex::XDim3 WarpX::UpperCorner(const Box& bx, const int lev, const amrex::Real time_shift_delta) { auto & warpx = GetInstance(); const RealBox grid_box = getRealBox( bx, lev ); - const Real* xyzmax = grid_box.hi(); + const Real* grid_max = grid_box.hi(); const amrex::Real cur_time = warpx.gett_new(lev); const amrex::Real time_shift = (cur_time + time_shift_delta - warpx.time_of_last_gal_shift); @@ -2894,13 +2901,13 @@ WarpX::UpperCorner(const Box& bx, const int lev, const amrex::Real time_shift_de warpx.m_v_galilean[2]*time_shift }; #if defined(WARPX_DIM_3D) - return { xyzmax[0] + galilean_shift[0], xyzmax[1] + galilean_shift[1], xyzmax[2] + galilean_shift[2] }; + return { grid_max[0] + galilean_shift[0], grid_max[1] + galilean_shift[1], grid_max[2] + galilean_shift[2] }; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - return { xyzmax[0] + galilean_shift[0], std::numeric_limits::max(), xyzmax[1] + galilean_shift[1] }; + return { grid_max[0] + galilean_shift[0], std::numeric_limits::max(), grid_max[1] + galilean_shift[1] }; #elif defined(WARPX_DIM_1D_Z) - return { std::numeric_limits::max(), std::numeric_limits::max(), xyzmax[0] + galilean_shift[0] }; + return { std::numeric_limits::max(), std::numeric_limits::max(), grid_max[0] + galilean_shift[0] }; #endif } diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index ff3741a7a43..75e3bca170d 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -34,7 +34,7 @@ namespace ablastr::particles * \param rho MultiFab of the charge density * \param local_rho temporary FArrayBox for deposition with OpenMP * \param particle_shape shape factor in each direction - * \param dx cell spacing at level lev + * \param dinv cell spacing inverses at level lev * \param xyzmin lo corner of the current tile in physical coordinates. * \param n_rz_azimuthal_modes number of azimuthal modes in use, irrelevant outside RZ geometry (default: 0) * \param num_rho_deposition_guards number of ghost cells to use for rho (default: rho.nGrowVect()) @@ -54,8 +54,8 @@ deposit_charge (typename T_PC::ParIterType& pti, amrex::MultiFab* rho, amrex::FArrayBox& local_rho, int const particle_shape, - std::array const & dx, - std::array const & xyzmin, + const amrex::XDim3 dinv, + const amrex::XDim3 xyzmin, int const n_rz_azimuthal_modes = 0, std::optional num_rho_deposition_guards = std::nullopt, std::optional depos_lev = std::nullopt, @@ -175,19 +175,19 @@ deposit_charge (typename T_PC::ParIterType& pti, if (nox == 1){ doChargeDepositionShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } else if (nox == 2){ doChargeDepositionShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } else if (nox == 3){ doChargeDepositionShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } else if (nox == 4){ doChargeDepositionShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd); From 8a94987a33c329431c9db848e7dcf35ce2b7a224 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 19 Jun 2024 00:20:20 +0200 Subject: [PATCH 24/87] Fix typo (#5001) --- Docs/source/usage/parameters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index a3ca703d15e..ae1353159dd 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -88,7 +88,7 @@ Overall simulation parameters * ``theta_implicit_em``: Use a fully implicit electromagnetic solver with a time-biasing parameter theta bound between 0.5 and 1.0. Exact energy conservation is achieved using theta = 0.5. Maximal damping of high-k modes is obtained using theta = 1.0. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed (PS) JNFK. The algorithm itself is numerical stable for large time steps. That is, it does not require time steps that resolve the plasma period or the CFL condition for light waves. However, the practicality of using a large time step depends on the nonlinear solver. Note that the Picard solver is for demonstration only. It is inefficient and will most like not converge when - :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relavent physics. + :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relevant physics. The Picard method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. The PS-JFNK method is described in `Angus et al., An implicit particle code with exact energy and charge conservation for electromagnetic studies of dense plasmas `__ . (The version implemented in WarpX is an updated version that includes the relativistic gamma factor for the particles.) Also see `Chen et al., An energy- and charge-conserving, implicit, electrostatic particle-in-cell algorithm. `__ . Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. From 34794926c373add1f8839efb1ce5919b6641874f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Jun 2024 17:15:38 -0700 Subject: [PATCH 25/87] Doc: Update WarpX Ack Text (#5004) Generalize text post ECP. (Various funding sources from many people.) --- Docs/source/acknowledge_us.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/acknowledge_us.rst b/Docs/source/acknowledge_us.rst index 4c7a2d8137c..bffbb80a797 100644 --- a/Docs/source/acknowledge_us.rst +++ b/Docs/source/acknowledge_us.rst @@ -23,14 +23,14 @@ Please add the following sentence to your publications, it helps contributors ke **Plain text:** - This research used the open-source particle-in-cell code WarpX https://github.com/ECP-WarpX/WarpX, primarily funded by the US DOE Exascale Computing Project. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors. + This research used the open-source particle-in-cell code WarpX https://github.com/ECP-WarpX/WarpX. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors. **LaTeX:** .. code-block:: latex \usepackage{hyperref} - This research used the open-source particle-in-cell code WarpX \url{https://github.com/ECP-WarpX/WarpX}, primarily funded by the US DOE Exascale Computing Project. + This research used the open-source particle-in-cell code WarpX \url{https://github.com/ECP-WarpX/WarpX}. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors. From bdda0bd0c68cb4687c85acc76e4aa2921dcc8b1a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Jun 2024 18:41:11 -0700 Subject: [PATCH 26/87] Doc: Resonant excitation of plasma waves in a plasma channel (#4998) * Doc: Resonant excitation of plasma waves in a plasma channel WarpX used in the paper by A. J. Ross et al., PRR 6 (2024) * Nguyen, B et a. IPAC24 * Sort by Pub Year * Update PASC24 in other sections --- Docs/source/acknowledge_us.rst | 4 ++-- Docs/source/highlights.rst | 31 +++++++++++++++++++++---------- Docs/source/refs.bib | 25 ++++++++++++++----------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/Docs/source/acknowledge_us.rst b/Docs/source/acknowledge_us.rst index bffbb80a797..b01ebf00a39 100644 --- a/Docs/source/acknowledge_us.rst +++ b/Docs/source/acknowledge_us.rst @@ -55,8 +55,8 @@ If your project uses a specific algorithm or component, please consider citing t - Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint __` + Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024. + `DOI:10.1145/3659914.3659937 `__ - Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index b243f62cd97..7baec74d606 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -14,7 +14,17 @@ Plasma-Based Acceleration Scientific works in laser-plasma and beam-plasma acceleration. -#. Peng, H. and Huang, T. W. and Jiang, K. and Li, R. and Wu, C. N. and Yu, M. Y. and Riconda, C. and Weber, S. and Zhou, C. T. and Ruan, S. C. +#. Ross AJ, Chappell J, van de Wetering JJ, Cowley J, Archer E, Bourgeois N, Corner L, Emerson DR, Feder L, Gu XJ, Jakobsson O, Jones H, Picksley A, Reid L, Wang W, Walczak R, Hooker SM. + **Resonant excitation of plasma waves in a plasma channel**. + Phys. Rev. Research **6**, L022001, 2024 + `DOI:10.1103/PhysRevResearch.6.L022001 `__ + +#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024. + `DOI:10.1145/3659914.3659937 `__ + +#. Peng H, Huang TW, Jiang K, Li R, Wu CN, Yu MY, Riconda C, Weber S, Zhou CT, Ruan SC. **Coherent Subcycle Optical Shock from a Superluminal Plasma Wake**. Phys. Rev. Lett. **131**, 145003, 2023 `DOI:10.1103/PhysRevLett.131.145003 `__ @@ -24,11 +34,6 @@ Scientific works in laser-plasma and beam-plasma acceleration. Phys. Rev. Research **5**, 033112, 2023 `DOI:10.1103/PhysRevResearch.5.033112 `__ -#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. - **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint `__ - #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. @@ -103,14 +108,20 @@ Scientific works in particle and beam modeling. #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint `__ + Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024. + `DOI:10.1145/3659914.3659937 `__ + +#. Nguyen B, Formenti A, Lehe R, Vay J-L, Gessner S, and Fedeli L. + **Comparison of WarpX and GUINEA-PIG for electron positron collisions**. + 15th International Particle Accelerator Conference (IPAC'24), WEPC84, 2024. + `preprint `__, + `DOI:10.18429/JACoW-IPAC2024-WEPC84 `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. - 14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023. + 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. `preprint `__, - `DOI:10.18429/JACoW-IPAC-23-WEPA101 `__ + `DOI:10.18429/JACoW-IPAC2023-WEPA101 `__ #. Tan W H, Piot P, Myers A, Zhang W, Rheaume T, Jambunathan R, Huebl A, Lehe R, Vay J-L. **Simulation studies of drive-beam instability in a dielectric wakefield accelerator**. diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 84b34b17351..130e0ce4da7 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -205,17 +205,20 @@ @article{Roedel2010 year = {2010} } -@misc{SandbergPASC24, -address = {Zuerich, Switzerland}, -author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Andrew Myers and Ji Qiang and Jean-Luc Vay and Axel Huebl}, -booktitle = {Proc. of PASC24}, -note = {accepted}, -series = {PASC'24 - Platform for Advanced Scientific Computing}, -title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, -venue = {Zuerich, Switzerland}, -year = {2024}, -doi = {10.48550/arXiv.2402.17248}, -url = {https://arxiv.org/abs/2402.17248} +@inproceedings{SandbergPASC24, +author = {Sandberg, Ryan and Lehe, Remi and Mitchell, Chad and Garten, Marco and Myers, Andrew and Qiang, Ji and Vay, Jean-Luc and Huebl, Axel}, +title = {{Synthesizing Particle-In-Cell Simulations through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, +series = {PASC '24}, +booktitle = {Proceedings of the Platform for Advanced Scientific Computing Conference}, +location = {Zuerich, Switzerland}, +publisher = {Association for Computing Machinery}, +address = {New York, NY, USA}, +articleno = {23}, +numpages = {11}, +isbn = {9798400706394}, +doi = {10.1145/3659914.3659937}, +note = {{PASC24 Best Paper Award}}, +year = {2024} } @inproceedings{SandbergIPAC23, From 8fe9c24e332acd6af8d2f68f572af66f6666636b Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:57:19 -0700 Subject: [PATCH 27/87] Warning instead of error for Background MCC (#4991) * add warning instead of error * Apply suggestions from code review Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --------- Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- .../BackgroundMCC/BackgroundMCCCollision.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 1eb89ff4c0f..80ce13744fd 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -234,9 +234,12 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult // dt has to be small enough that a linear expansion of the collision // probability is sufficiently accurately, otherwise the MCC results // will be very heavily affected by small changes in the timestep - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(coll_n < 0.1_prt, - "dt is too large to ensure accurate MCC results" - ); + if (coll_n > 0.1_prt) { + ablastr::warn_manager::WMRecordWarning("BackgroundMCC Collisions", + "dt is too large to ensure accurate MCC results , coll_n: " + + std::to_string(coll_n) + " is > 0.1 and collision probability is = " + + std::to_string(m_total_collision_prob) + "\n"); + } if (ionization_flag) { // calculate maximum collision frequency for ionization @@ -246,9 +249,12 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult auto coll_n_ioniz = m_nu_max_ioniz * dt; m_total_collision_prob_ioniz = 1.0_prt - std::exp(-coll_n_ioniz); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(coll_n_ioniz < 0.1_prt, - "dt is too large to ensure accurate MCC results" - ); + if (coll_n_ioniz > 0.1_prt) { + ablastr::warn_manager::WMRecordWarning("BackgroundMCC Collisions", + "dt is too large to ensure accurate MCC ionization , coll_n_ionization: " + + std::to_string(coll_n_ioniz) + " is > 0.1 and ionization probability is = " + + std::to_string(m_total_collision_prob_ioniz) + "\n"); + } // if an ionization process is included the secondary species mass // is taken as the background mass From 6dd5e9261a15860ad46ff3e6dba4340e9fae9e2a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Jun 2024 21:14:46 -0700 Subject: [PATCH 28/87] AMReX/pyAMReX/PICSAR: Weekly Update (#4995) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 3a03ea01c0c..71775879896 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.06 && cd - + cd ../amrex && git checkout --detach 1f038e767011e20100af6ab0db02c69cf7ebe55c && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 05ed74fe1b1..f1fb5f4b0db 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.06 +branch = 1f038e767011e20100af6ab0db02c69cf7ebe55c [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 41db9a15bdc..f00fb083cbc 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.06 +branch = 1f038e767011e20100af6ab0db02c69cf7ebe55c [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index eda45dc9f77..30fe67bc018 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.06" +set(WarpX_amrex_branch "1f038e767011e20100af6ab0db02c69cf7ebe55c" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 18cab89e347..1016158fc4a 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.06" +set(WarpX_pyamrex_branch "25e097f4ca10e0c5ad506041e56b7b2181d2f34a" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index f397d31048e..d590a710f4a 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.06 && cd - +cd amrex && git checkout --detach 1f038e767011e20100af6ab0db02c69cf7ebe55c && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 1077dbc00f4ca6be589dee8b13ba8bd41940fe01 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Jun 2024 23:57:54 -0700 Subject: [PATCH 29/87] Fix: Perlmutter heFFTe CPU (#5006) --- Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index a5a7c28b85e..e9ac393cd32 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -115,7 +115,6 @@ then cd $HOME/src/heffte git fetch --prune git checkout v2.4.0 - git pull cd - else git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte From 03dc9b76cfe17e7cbd1d66983ccfad41e0e8c810 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Thu, 20 Jun 2024 13:49:06 -0700 Subject: [PATCH 30/87] Fix ccache for macos (#5007) Need to ignore time macros. --- .github/workflows/macos.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 21d5ae04faf..00ac8f06b5d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -44,7 +44,7 @@ jobs: - name: CCache Cache uses: actions/cache@v4 with: - path: /Users/runner/Library/Caches/ccache + path: ~/Library/Caches/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | ccache-${{ github.workflow }}-${{ github.job }}-git- @@ -53,7 +53,7 @@ jobs: export CCACHE_COMPRESS=1 export CCACHE_COMPRESSLEVEL=10 export CCACHE_MAXSIZE=100M - export CCACHE_DEPEND=1 + export CCACHE_SLOPPINESS=time_macros ccache -z source py-venv/bin/activate @@ -76,6 +76,7 @@ jobs: cmake --build build_sp -j 3 cmake --build build_sp --target pip_install + du -hs ~/Library/Caches/ccache ccache -s - name: run pywarpx From 3e5112f13f5b6edf50f11cdb8dc46a599fdd7d1d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 24 Jun 2024 10:41:40 -0700 Subject: [PATCH 31/87] BLAS++: v2024.05.31+ (#5012) Update our HankelTransform logic calling into BLAS++ by using the latest APIs for `blas::Queue`. Pass the active AMReX GPU stream. --- .../SpectralSolver/SpectralHankelTransform/HankelTransform.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp index 9f4cac71736..7fd419b54f8 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp @@ -36,7 +36,8 @@ HankelTransform::HankelTransform (int const hankel_order, // SYCL note: we need to double check AMReX device ID conventions and // BLAS++ device ID conventions are the same int const device_id = amrex::Gpu::Device::deviceId(); - m_queue = std::make_unique( device_id, 0 ); + blas::Queue::stream_t stream_id = amrex::Gpu::gpuStream(); + m_queue = std::make_unique( device_id, stream_id ); #endif amrex::Vector alphas; From 64ab258e93198b87114fb9c92d435809f79608a9 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 24 Jun 2024 16:05:44 -0700 Subject: [PATCH 32/87] Doc: Latest Commit of BLAS++/LAPACK++ (#5013) For BLAS++ and LAPACK++, switch the documented HPC dependency install logic from using the latest commit on `master` to the latest release `2024.05.31`. Using tags instead of living branches increases robustness and reproducibility with regards to future changes. --- .../adastra-cines/adastra_warpx.profile.example | 8 ++++---- .../machines/adastra-cines/install_dependencies.sh | 14 ++++++-------- .../cori-nersc/haswell_warpx.profile.example | 4 ++-- .../machines/cori-nersc/knl_warpx.profile.example | 4 ++-- .../crusher-olcf/crusher_warpx.profile.example | 8 ++++---- .../machines/crusher-olcf/install_dependencies.sh | 14 ++++++-------- .../frontier-olcf/frontier_warpx.profile.example | 8 ++++---- .../machines/frontier-olcf/install_dependencies.sh | 14 ++++++-------- .../greatlakes_v100_warpx.profile.example | 8 ++++---- .../greatlakes-umich/install_v100_dependencies.sh | 14 ++++++-------- .../hpc3-uci/hpc3_gpu_warpx.profile.example | 8 ++++---- .../machines/hpc3-uci/install_gpu_dependencies.sh | 14 ++++++-------- .../lassen-llnl/install_v100_dependencies.sh | 14 ++++++-------- .../lassen-llnl/lassen_v100_warpx.profile.example | 8 ++++---- .../lassen_v100_warpx_toss3.profile.example | 8 ++++---- .../lawrencium_warpx.profile.example | 4 ++-- .../leonardo-cineca/install_gpu_dependencies.sh | 14 ++++++-------- .../leonardo_gpu_warpx.profile.example | 8 ++++---- Tools/machines/lumi-csc/install_dependencies.sh | 14 ++++++-------- Tools/machines/lumi-csc/lumi_warpx.profile.example | 8 ++++---- .../perlmutter-nersc/install_cpu_dependencies.sh | 14 ++++++-------- .../perlmutter-nersc/install_gpu_dependencies.sh | 14 ++++++-------- .../perlmutter_cpu_warpx.profile.example | 8 ++++---- .../perlmutter_gpu_warpx.profile.example | 8 ++++---- .../polaris-alcf/install_gpu_dependencies.sh | 14 ++++++-------- .../polaris-alcf/polaris_gpu_warpx.profile.example | 8 ++++---- Tools/machines/quartz-llnl/install_dependencies.sh | 14 ++++++-------- .../quartz-llnl/quartz_warpx.profile.example | 8 ++++---- .../summit-olcf/install_gpu_dependencies.sh | 14 ++++++-------- .../summit-olcf/summit_warpx.profile.example | 8 ++++---- 30 files changed, 140 insertions(+), 166 deletions(-) diff --git a/Tools/machines/adastra-cines/adastra_warpx.profile.example b/Tools/machines/adastra-cines/adastra_warpx.profile.example index 0d55e869d6a..3cba4346421 100644 --- a/Tools/machines/adastra-cines/adastra_warpx.profile.example +++ b/Tools/machines/adastra-cines/adastra_warpx.profile.example @@ -11,10 +11,10 @@ module load CCE-GPU-3.0.0 module load amd-mixed/5.2.3 # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.83.0-mpi-python3 diff --git a/Tools/machines/adastra-cines/install_dependencies.sh b/Tools/machines/adastra-cines/install_dependencies.sh index b48bf144c2a..242f5fc664c 100755 --- a/Tools/machines/adastra-cines/install_dependencies.sh +++ b/Tools/machines/adastra-cines/install_dependencies.sh @@ -38,14 +38,13 @@ if [ -d $SHAREDHOMEDIR/src/blaspp ] then cd $SHAREDHOMEDIR/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $SHAREDHOMEDIR/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $SHAREDHOMEDIR/src/blaspp fi rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -CXX=$(which CC) cmake -S $SHAREDHOMEDIR/src/blaspp -B $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $SHAREDHOMEDIR/src/blaspp -B $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build --target install --parallel 16 rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build @@ -54,14 +53,13 @@ if [ -d $SHAREDHOMEDIR/src/lapackpp ] then cd $SHAREDHOMEDIR/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $SHAREDHOMEDIR/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $SHAREDHOMEDIR/src/lapackpp fi rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $SHAREDHOMEDIR/src/lapackpp -B $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $SHAREDHOMEDIR/src/lapackpp -B $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build --target install --parallel 16 rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build diff --git a/Tools/machines/cori-nersc/haswell_warpx.profile.example b/Tools/machines/cori-nersc/haswell_warpx.profile.example index b0ac225cc72..f636b1ab555 100644 --- a/Tools/machines/cori-nersc/haswell_warpx.profile.example +++ b/Tools/machines/cori-nersc/haswell_warpx.profile.example @@ -8,8 +8,8 @@ module load cray-python/3.9.7.1 export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH export CMAKE_PREFIX_PATH=$HOME/sw/haswell/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/haswell/adios2-2.7.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/haswell/blaspp-master-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/haswell/lapackpp-master-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/haswell/blaspp-2024.05.31-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/haswell/lapackpp-2024.05.31-install:$CMAKE_PREFIX_PATH if [ -d "$HOME/sw/haswell/venvs/haswell_warpx" ] then diff --git a/Tools/machines/cori-nersc/knl_warpx.profile.example b/Tools/machines/cori-nersc/knl_warpx.profile.example index 093c7f62783..fdd7e14d594 100644 --- a/Tools/machines/cori-nersc/knl_warpx.profile.example +++ b/Tools/machines/cori-nersc/knl_warpx.profile.example @@ -9,8 +9,8 @@ module load cray-python/3.9.7.1 export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH export CMAKE_PREFIX_PATH=$HOME/sw/knl/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/knl/adios2-2.7.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/knl/blaspp-master-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/knl/lapackpp-master-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/knl/blaspp-2024.05.31-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/knl/lapackpp-2024.05.31-install:$CMAKE_PREFIX_PATH if [ -d "$HOME/sw/knl/venvs/knl_warpx" ] then diff --git a/Tools/machines/crusher-olcf/crusher_warpx.profile.example b/Tools/machines/crusher-olcf/crusher_warpx.profile.example index ffc70a128e2..7b05546b3f3 100644 --- a/Tools/machines/crusher-olcf/crusher_warpx.profile.example +++ b/Tools/machines/crusher-olcf/crusher_warpx.profile.example @@ -21,10 +21,10 @@ module load ninja module load nano # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.79.0-cxx17 diff --git a/Tools/machines/crusher-olcf/install_dependencies.sh b/Tools/machines/crusher-olcf/install_dependencies.sh index ab5deb6b0a6..6e9f97eddae 100755 --- a/Tools/machines/crusher-olcf/install_dependencies.sh +++ b/Tools/machines/crusher-olcf/install_dependencies.sh @@ -49,14 +49,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-crusher-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-crusher-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-crusher-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-crusher-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-crusher-gpu-build @@ -65,14 +64,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-crusher-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-crusher-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-crusher-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-crusher-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-crusher-gpu-build diff --git a/Tools/machines/frontier-olcf/frontier_warpx.profile.example b/Tools/machines/frontier-olcf/frontier_warpx.profile.example index 2f86d047d00..50e12e99a38 100644 --- a/Tools/machines/frontier-olcf/frontier_warpx.profile.example +++ b/Tools/machines/frontier-olcf/frontier_warpx.profile.example @@ -20,10 +20,10 @@ module load ninja module load nano # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.79.0-cxx17 diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 98c30cfca8f..503ab4525e9 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -49,14 +49,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-frontier-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-frontier-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-frontier-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-frontier-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-frontier-gpu-build @@ -65,14 +64,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-frontier-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-frontier-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-frontier-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-frontier-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-frontier-gpu-build diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example index 98476eecfab..c08255e7962 100644 --- a/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example +++ b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example @@ -22,13 +22,13 @@ module load phdf5/1.12.1 SW_DIR="${HOME}/sw/greatlakes/v100" export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc2-2.14.4:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.0:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${SW_DIR}/c-blosc2-2.14.4/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.0/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=${SW_DIR}/adios2-2.10.0/bin:${PATH} diff --git a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh index 9361fd6c64b..30faec52421 100755 --- a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh +++ b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh @@ -80,14 +80,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-v100-build -cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-v100-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-v100-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-v100-build --target install --parallel 8 rm -rf ${build_dir}/blaspp-v100-build @@ -96,14 +95,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-v100-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-v100-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-v100-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-v100-build --target install --parallel 8 rm -rf ${build_dir}/lapackpp-v100-build diff --git a/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example b/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example index 9f41cc7e337..017613f9d60 100644 --- a/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example +++ b/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example @@ -19,13 +19,13 @@ module load OpenBLAS/0.3.21 module load hdf5/1.13.1/gcc.11.2.0-openmpi.4.1.2 export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh index 51d4bcfe2c5..b585c2702b6 100755 --- a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh +++ b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh @@ -79,14 +79,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/blaspp-pm-gpu-build @@ -95,14 +94,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/lapackpp-pm-gpu-build diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh index fe8285b7501..6368010ab7b 100755 --- a/Tools/machines/lassen-llnl/install_v100_dependencies.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies.sh @@ -82,13 +82,12 @@ if [ -d ${SRC_DIR}/blaspp ] then cd ${SRC_DIR}/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi -cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 # LAPACK++ (for PSATD+RZ) @@ -96,13 +95,12 @@ if [ -d ${SRC_DIR}/lapackpp ] then cd ${SRC_DIR}/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example index 4d4efbf917d..80755ece308 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example @@ -22,10 +22,10 @@ export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} export PATH=${SW_DIR}/adios2-2.8.3/bin:$PATH # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings module load python/3.8.2 diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example index abad909943c..6a7067f596e 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example @@ -22,10 +22,10 @@ export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings module load python/3.8.2 diff --git a/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example b/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example index 472d0785bb2..8db2b44b8a7 100644 --- a/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example +++ b/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example @@ -18,8 +18,8 @@ module load lapack/3.8.0-gcc export CMAKE_PREFIX_PATH=$HOME/sw/v100/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/v100/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/v100/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/v100/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/v100/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/v100/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export PATH=$HOME/sw/v100/adios2-2.8.3/bin:$PATH diff --git a/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh b/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh index bbaf0ab8464..4d89e30cd29 100644 --- a/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh +++ b/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh @@ -48,14 +48,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-gpu-build -CXX=$(which g++) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which g++) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-gpu-build @@ -65,14 +64,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-gpu-build diff --git a/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example b/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example index cffe565f9d7..dd8e79ffb37 100644 --- a/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example +++ b/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example @@ -16,13 +16,13 @@ module load boost/1.80.0--openmpi--4.1.4--gcc--11.3.0 module load openblas/0.3.21--gcc--11.3.0 export CMAKE_PREFIX_PATH=/leonardo/prod/spack/03/install/0.19/linux-rhel8-icelake/gcc-11.3.0/c-blosc-1.21.1-aifmix6v5lwxgt7rigwoebalrgbcnv26:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/adios2-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=/leonardo/prod/spack/03/install/0.19/linux-rhel8-icelake/gcc-11.3.0/c-blosc-1.21.1-aifmix6v5lwxgt7rigwoebalrgbcnv26/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$HOME/sw/adios2-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$HOME/sw/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$HOME/sw/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=$HOME/sw/adios2-master/bin:$PATH diff --git a/Tools/machines/lumi-csc/install_dependencies.sh b/Tools/machines/lumi-csc/install_dependencies.sh index 814e134bad4..2fd31b79bce 100755 --- a/Tools/machines/lumi-csc/install_dependencies.sh +++ b/Tools/machines/lumi-csc/install_dependencies.sh @@ -43,11 +43,10 @@ if [ -d ${SRC_DIR}/blaspp ] then cd ${SRC_DIR}/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi rm -rf ${build_dir}/blaspp-lumi-gpu-build CXX=$(which CC) \ @@ -56,7 +55,7 @@ cmake -S ${SRC_DIR}/blaspp \ -Duse_openmp=OFF \ -Dgpu_backend=hip \ -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-lumi-gpu-build --target install --parallel 16 rm -rf ${build_dir}/blaspp-lumi-gpu-build @@ -65,11 +64,10 @@ if [ -d ${SRC_DIR}/lapackpp ] then cd ${SRC_DIR}/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi rm -rf ${build_dir}/lapackpp-lumi-gpu-build CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" \ @@ -78,7 +76,7 @@ cmake -S ${SRC_DIR}/lapackpp \ -DCMAKE_CXX_STANDARD=17 \ -Dbuild_tests=OFF \ -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-lumi-gpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-lumi-gpu-build diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example index 33aff1946c2..13fb6b1d81e 100644 --- a/Tools/machines/lumi-csc/lumi_warpx.profile.example +++ b/Tools/machines/lumi-csc/lumi_warpx.profile.example @@ -11,10 +11,10 @@ module load nano # optional: for PSATD in RZ geometry support SW_DIR="${HOME}/sw/lumi/gpu" -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load Boost/1.82.0-cpeCray-23.09 diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index e9ac393cd32..437300b8303 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -82,14 +82,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-cpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-pm-cpu-build --target install --parallel 16 rm -rf ${build_dir}/blaspp-pm-cpu-build @@ -98,14 +97,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-cpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-cpu-build diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index 125c63104be..ac95ad9f3a0 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -82,14 +82,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-pm-gpu-build --target install --parallel 16 rm -rf ${build_dir}/blaspp-pm-gpu-build @@ -98,14 +97,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-gpu-build diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 3fea9b3aa39..94d598abf5b 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -17,14 +17,14 @@ export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/ module load cray-hdf5-parallel/1.12.2.9 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 9e1465d6c02..da1d55964d1 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -21,14 +21,14 @@ export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/ module load cray-hdf5-parallel/1.12.2.9 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/polaris-alcf/install_gpu_dependencies.sh b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh index dbc66b2ffff..18f94e6fd3b 100755 --- a/Tools/machines/polaris-alcf/install_gpu_dependencies.sh +++ b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh @@ -65,14 +65,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-pm-gpu-build @@ -81,14 +80,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-pm-gpu-build diff --git a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example index d7635188141..333434c1b97 100644 --- a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example +++ b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example @@ -19,13 +19,13 @@ module load cmake/3.23.2 module load cray-hdf5-parallel/1.12.2.3 export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/quartz-llnl/install_dependencies.sh b/Tools/machines/quartz-llnl/install_dependencies.sh index 2920b00663b..cfb01769384 100755 --- a/Tools/machines/quartz-llnl/install_dependencies.sh +++ b/Tools/machines/quartz-llnl/install_dependencies.sh @@ -67,13 +67,12 @@ if [ -d ${HOME}/src/blaspp ] then cd ${HOME}/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${HOME}/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${HOME}/src/blaspp fi -cmake -S ${HOME}/src/blaspp -B ${build_dir}/blaspp-quartz-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S ${HOME}/src/blaspp -B ${build_dir}/blaspp-quartz-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-quartz-build --target install --parallel 6 # LAPACK++ (for PSATD+RZ) @@ -81,13 +80,12 @@ if [ -d ${HOME}/src/lapackpp ] then cd ${HOME}/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${HOME}/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${HOME}/src/lapackpp fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${HOME}/src/lapackpp -B ${build_dir}/lapackpp-quartz-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${HOME}/src/lapackpp -B ${build_dir}/lapackpp-quartz-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-quartz-build --target install --parallel 6 diff --git a/Tools/machines/quartz-llnl/quartz_warpx.profile.example b/Tools/machines/quartz-llnl/quartz_warpx.profile.example index a3646bfd557..f296a0738ff 100644 --- a/Tools/machines/quartz-llnl/quartz_warpx.profile.example +++ b/Tools/machines/quartz-llnl/quartz_warpx.profile.example @@ -21,10 +21,10 @@ export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings module load python/3.9.12 diff --git a/Tools/machines/summit-olcf/install_gpu_dependencies.sh b/Tools/machines/summit-olcf/install_gpu_dependencies.sh index 433b7ab0580..042e34538d0 100755 --- a/Tools/machines/summit-olcf/install_gpu_dependencies.sh +++ b/Tools/machines/summit-olcf/install_gpu_dependencies.sh @@ -63,13 +63,12 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi -cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-summit-build -Duse_openmp=ON -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-summit-build -Duse_openmp=ON -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-summit-build --target install --parallel 10 # LAPACK++ (for PSATD+RZ) @@ -77,13 +76,12 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi -cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-summit-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-summit-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-summit-build --target install --parallel 10 # remove build temporary directory diff --git a/Tools/machines/summit-olcf/summit_warpx.profile.example b/Tools/machines/summit-olcf/summit_warpx.profile.example index e41521b5815..d70ee9eb42a 100644 --- a/Tools/machines/summit-olcf/summit_warpx.profile.example +++ b/Tools/machines/summit-olcf/summit_warpx.profile.example @@ -18,10 +18,10 @@ module load ccache module load ninja # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.76.0 From 1565175d304917101be165877c26c887251e1b81 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 24 Jun 2024 18:26:06 -0700 Subject: [PATCH 33/87] AMReX/pyAMReX/PICSAR: Weekly Update (#5014) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 71775879896..7e1cf7ae63f 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 1f038e767011e20100af6ab0db02c69cf7ebe55c && cd - + cd ../amrex && git checkout --detach 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index f1fb5f4b0db..e4e5d7b8678 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 1f038e767011e20100af6ab0db02c69cf7ebe55c +branch = 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index f00fb083cbc..a18f39b89c1 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 1f038e767011e20100af6ab0db02c69cf7ebe55c +branch = 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 30fe67bc018..632b9edd4f4 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "1f038e767011e20100af6ab0db02c69cf7ebe55c" +set(WarpX_amrex_branch "259db7cfb99e7d1d2ab4bec9b1587fdf624a138a" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 1016158fc4a..90aee6cb930 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "25e097f4ca10e0c5ad506041e56b7b2181d2f34a" +set(WarpX_pyamrex_branch "fd7126cb907be7e3c8570930b79209db5cd7c492" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index d590a710f4a..6ac22466ef0 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 1f038e767011e20100af6ab0db02c69cf7ebe55c && cd - +cd amrex && git checkout --detach 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 77c04659700bad186aacd326469d3cb8cf7541e9 Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Fri, 28 Jun 2024 23:42:40 -0700 Subject: [PATCH 34/87] obvious bug fix in ElasticCollisionPerez (#5021) * obvious bug fix in ElasticCollisionPerez * updating benchmark json files for collisionISO and collisionXYZ --- .../Checksum/benchmarks_json/collisionISO.json | 14 +++++++------- .../Checksum/benchmarks_json/collisionXYZ.json | 4 ++-- .../Coulomb/ElasticCollisionPerez.H | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json index 96a2f5cbdac..5146837be6f 100644 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ b/Regression/Checksum/benchmarks_json/collisionISO.json @@ -11,12 +11,12 @@ "jz": 0.0 }, "electron": { - "particle_momentum_x": 3.5856774369040155e-19, - "particle_momentum_y": 3.5832349891760786e-19, - "particle_momentum_z": 3.5757013442445486e-19, - "particle_position_x": 1.0241562532152744, - "particle_position_y": 1.0238555904782611, - "particle_position_z": 1.0240053504978093, + "particle_momentum_x": 3.579130200773753e-19, + "particle_momentum_y": 3.5788119408700804e-19, + "particle_momentum_z": 3.584163522201744e-19, + "particle_position_x": 1.024188253213835, + "particle_position_y": 1.0238795904737117, + "particle_position_z": 1.02399735048655, "particle_weight": 714240000000.0 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index c87bbd98e42..984dd1d2761 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,7 +6,7 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 358778.6506903592, - "T_ion": 341562.3085776466 + "T_electron": 351375.5409141184, + "T_ion": 349005.3985888339 } } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index e407ae1603b..af4e6ba38c9 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -58,7 +58,7 @@ void ElasticCollisionPerez ( const T_index NI1 = I1e - I1s; const T_index NI2 = I2e - I2s; const T_index max_N = amrex::max(NI1,NI2); - const T_index min_N = amrex::max(NI1,NI2); + const T_index min_N = amrex::min(NI1,NI2); T_PR * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; T_PR * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; From dcdf4f7d31313850925650a1878fe965ba09611f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 2 Jul 2024 18:59:33 +0200 Subject: [PATCH 35/87] Document how to define EB with STL file (#5023) --- Docs/source/usage/parameters.rst | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index ae1353159dd..afa570cca02 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -581,11 +581,26 @@ Additional PML parameters Embedded Boundary Conditions ---------------------------- -* ``warpx.eb_implicit_function`` (`string`) - A function of `x`, `y`, `z` that defines the surface of the embedded - boundary. That surface lies where the function value is 0 ; - the physics simulation area is where the function value is negative ; - the interior of the embeddded boundary is where the function value is positive. +In WarpX, the embedded boundary can be defined in either of two ways: + + - **From an analytical function:** + In that case, you will need to set the following parameter in the input file. + + * ``warpx.eb_implicit_function`` (`string`) + A function of `x`, `y`, `z` that defines the surface of the embedded + boundary. That surface lies where the function value is 0 ; + the physics simulation area is where the function value is negative ; + the interior of the embeddded boundary is where the function value is positive. + + - **From an STL file:** + In that case, you will need to set the following parameters in the input file. + + * ``eb2.stl_file`` (`string`) + The path to an STL file. In addition, you also need to set ``eb2.geom_type = stl``, + in order for the file to be read by WarpX. + +Whether the embedded boundary is defined with an analytical function or an STL file, you can +additionally define the electric potential at the embedded boundary with an analytical function: * ``warpx.eb_potential(x,y,z,t)`` (`string`) Gives the value of the electric potential at the surface of the embedded boundary, From 2d845dcf967ae0a593b88bb4d12911d402725981 Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 2 Jul 2024 09:59:52 -0700 Subject: [PATCH 36/87] Add random seed to Examples/Tests/collision/inputs_3d (#5022) * Add random seed to Examples/Tests/collision/inputs_3d * Update benchmark values * Update benchmark again --- Examples/Tests/collision/inputs_3d | 1 + Regression/Checksum/benchmarks_json/collisionXYZ.json | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Examples/Tests/collision/inputs_3d b/Examples/Tests/collision/inputs_3d index ed413ba2776..11d6b150dd2 100644 --- a/Examples/Tests/collision/inputs_3d +++ b/Examples/Tests/collision/inputs_3d @@ -22,6 +22,7 @@ boundary.field_hi = periodic periodic periodic warpx.serialize_initial_conditions = 1 warpx.verbose = 1 warpx.const_dt = 1.224744871e-07 +warpx.random_seed = 2034958209 # Do not evolve the E and B fields algo.maxwell_solver = none diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 984dd1d2761..7436302a99e 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,7 +6,7 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 351375.5409141184, - "T_ion": 349005.3985888339 + "T_electron": 351570.64632548566, + "T_ion": 350085.3998917431 } } From ca34cbc10e70ed8f25cfe2f0f4566fd45f7e06d7 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 3 Jul 2024 15:50:37 +0200 Subject: [PATCH 37/87] Simplify VelocityProperties.cpp (#5016) * simplify VelocityProperties.cpp * fix issues --- Source/Initialization/VelocityProperties.cpp | 23 ++++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Source/Initialization/VelocityProperties.cpp b/Source/Initialization/VelocityProperties.cpp index 14b48c79524..b6986970e31 100644 --- a/Source/Initialization/VelocityProperties.cpp +++ b/Source/Initialization/VelocityProperties.cpp @@ -18,26 +18,25 @@ VelocityProperties::VelocityProperties (const amrex::ParmParse& pp, std::string std::string vel_dir_s = "x"; utils::parser::query(pp, source_name, "bulk_vel_dir", vel_dir_s); - if(vel_dir_s[0] == '-'){ - m_sign_dir = -1; - } - else { - m_sign_dir = 1; + + if(vel_dir_s.empty()){ + WARPX_ABORT_WITH_MESSAGE("'.bulk_vel_dir input ' can't be empty."); } - if ((vel_dir_s == "x" || vel_dir_s[1] == 'x') || - (vel_dir_s == "X" || vel_dir_s[1] == 'X')){ + m_sign_dir = (vel_dir_s[0] == '-') ? -1 : 1; + + const auto dir = std::tolower(vel_dir_s.back()); + + if (dir == 'x'){ m_dir = 0; } - else if ((vel_dir_s == "y" || vel_dir_s[1] == 'y') || - (vel_dir_s == "Y" || vel_dir_s[1] == 'Y')){ + else if (dir == 'y'){ m_dir = 1; } - else if ((vel_dir_s == "z" || vel_dir_s[1] == 'z') || - (vel_dir_s == "Z" || vel_dir_s[1] == 'Z')) { + else if (dir == 'z'){ m_dir = 2; } - else { + else{ WARPX_ABORT_WITH_MESSAGE( "Cannot interpret .bulk_vel_dir input '" + vel_dir_s + "'. Please enter +/- x, y, or z with no whitespace between the sign and"+ From fbe9d6aa9036fa214d90e578f39e8c5cab4522fa Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 3 Jul 2024 18:47:22 +0200 Subject: [PATCH 38/87] Fix gatherParticlesFromEmbeddedBoundaries (#5011) * touch particle tiles before OMP loop * touch tiles before OMP loop --- Source/Particles/ParticleBoundaryBuffer.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 784c27816da..25068b2e65c 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -390,7 +390,15 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC buffer[i].AddRealComp("ny", false); buffer[i].AddRealComp("nz", false); } + auto& species_buffer = buffer[i]; + for (int lev = 0; lev < pc.numLevels(); ++lev){ + for(PIter pti(pc, lev); pti.isValid(); ++pti){ + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } + } + for (int lev = 0; lev < pc.numLevels(); ++lev) { const auto& plevel = pc.GetParticles(lev); @@ -474,7 +482,15 @@ void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries ( buffer[i].AddRealComp("nz", false); } + auto& species_buffer = buffer[i]; + for (int lev = 0; lev < pc.numLevels(); ++lev){ + for(PIter pti(pc, lev); pti.isValid(); ++pti){ + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } + } + for (int lev = 0; lev < pc.numLevels(); ++lev) { const auto& plevel = pc.GetParticles(lev); From a1980cf989c94354e2194d9b6d832c83093c7ff0 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 3 Jul 2024 23:03:07 +0200 Subject: [PATCH 39/87] Update default behavior for gathering with direct deposition (#5024) * Update default behavior for gathering with direct deposition * Modify condition under which to use galerkin interpolation * Update condition * Update benchmarks --------- Co-authored-by: Edoardo Zoni --- Docs/source/usage/parameters.rst | 2 +- .../Langmuir_multi_2d_MR_psatd.json | 44 +++++------ .../Langmuir_multi_2d_psatd.json | 28 +++---- .../Langmuir_multi_2d_psatd_multiJ.json | 32 ++++---- .../Langmuir_multi_psatd_div_cleaning.json | 46 ++++++------ .../Langmuir_multi_psatd_multiJ.json | 42 +++++------ .../benchmarks_json/Python_Langmuir.json | 16 ++-- .../benchmarks_json/Python_Langmuir_2d.json | 14 ++-- .../benchmarks_json/Python_gaussian_beam.json | 48 ++++++------ .../benchmarks_json/restart_psatd.json | 74 +++++++++---------- .../restart_psatd_time_avg.json | 66 ++++++++--------- Source/WarpX.cpp | 12 ++- 12 files changed, 216 insertions(+), 208 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index afa570cca02..4dc888df3d0 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2446,7 +2446,7 @@ Grid types (collocated, staggered, hybrid) For example, :math:`E_z` is gathered using ``algo.particle_shape`` along :math:`(x,y)` and ``algo.particle_shape - 1`` along :math:`z`. See equations (21)-(23) of :cite:t:`param-Godfrey2013` and associated references for details. - Default: ``interpolation.galerkin_scheme = 0`` with collocated grids and/or momentum-conserving field gathering, ``interpolation.galerkin_scheme = 1`` otherwise. + Default: ``interpolation.galerkin_scheme = 0`` with collocated grids, or momentum-conserving field gathering, or when ``algo.current_deposition = direct`` ; ``interpolation.galerkin_scheme = 1`` otherwise. .. warning:: diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json index 378b7691898..07c990b3997 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json @@ -1,40 +1,40 @@ { - "electrons": { - "particle_momentum_x": 4.2387000553111037e-20, - "particle_momentum_y": 0.0, - "particle_momentum_z": 4.238700039223283e-20, - "particle_position_x": 0.6553599659707846, - "particle_position_y": 0.6553599659699976, - "particle_weight": 3200000000000000.5 - }, "lev=0": { "Bx": 0.0, - "By": 46.86760020196344, + "By": 46.65494014556924, "Bz": 0.0, - "Ex": 7589330186947.288, + "Ex": 7596470155558.35, "Ey": 0.0, - "Ez": 7589330198843.742, - "jx": 7283102663077210.0, + "Ez": 7596470167219.471, + "jx": 7277974695617658.0, "jy": 0.0, - "jz": 7283102630279985.0 + "jz": 7277974664068728.0 }, "lev=1": { "Bx": 0.0, - "By": 369.2209081024592, + "By": 371.2497742366917, "Bz": 0.0, - "Ex": 7593801203430.788, + "Ex": 7596619292118.433, "Ey": 0.0, - "Ez": 7593800988021.711, - "jx": 6597471633118175.0, + "Ez": 7596619076137.7295, + "jx": 6592788899529852.0, "jy": 0.0, - "jz": 6597471447306984.0 + "jz": 6592788734672843.0 + }, + "electrons": { + "particle_momentum_x": 4.2314133085678375e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 4.2314132920088513e-20, + "particle_position_x": 0.6553599899254421, + "particle_position_y": 0.6553599899246253, + "particle_weight": 3200000000000000.5 }, "positrons": { - "particle_momentum_x": 4.238670783273887e-20, + "particle_momentum_x": 4.2314444192249906e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 4.238670762294589e-20, - "particle_position_x": 0.6553596917214832, - "particle_position_y": 0.6553596917221793, + "particle_momentum_z": 4.231444401040395e-20, + "particle_position_x": 0.655359770013864, + "particle_position_y": 0.6553597700144888, "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json index 85a09296e38..c59a09cc616 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json @@ -1,27 +1,27 @@ { - "electrons": { - "particle_momentum_x": 5.663705661977914e-20, - "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705661977915e-20, - "particle_position_x": 0.65536, - "particle_position_y": 0.6553599999999999, - "particle_weight": 3200000000000000.5 - }, "lev=0": { - "Ex": 3771082811318.298, + "Ex": 3751590727260.198, "Ey": 0.0, - "Ez": 3771082811318.301, - "jx": 1.009365330460902e+16, + "Ez": 3751590727260.1885, + "jx": 1.0100623838390416e+16, "jy": 0.0, - "jz": 1.009365330460902e+16, + "jz": 1.0100623838390416e+16, "part_per_cell": 131072.0 }, "positrons": { - "particle_momentum_x": 5.663705661977916e-20, + "particle_momentum_x": 5.668407775012392e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705661977916e-20, + "particle_momentum_z": 5.668407775012392e-20, "particle_position_x": 0.65536, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 + }, + "electrons": { + "particle_momentum_x": 5.668407775012391e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.668407775012391e-20, + "particle_position_x": 0.65536, + "particle_position_y": 0.6553600000000002, + "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json index c342d9dcaeb..ddf519913bd 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json @@ -1,29 +1,29 @@ { - "electrons": { - "particle_momentum_x": 5.663705657675969e-20, - "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705657675969e-20, - "particle_position_x": 0.65536, - "particle_position_y": 0.65536, - "particle_weight": 3200000000000000.5 - }, "lev=0": { "Bx": 0.0, - "By": 3.4892704618136277, + "By": 5.710236645367475, "Bz": 0.0, - "Ex": 3771082786646.7104, + "Ex": 3751590785149.1333, "Ey": 0.0, - "Ez": 3771082786646.702, - "jx": 1.0093631772735916e+16, + "Ez": 3751590785149.124, + "jx": 1.0100602372860838e+16, "jy": 0.0, - "jz": 1.0093631772735912e+16 + "jz": 1.0100602372860836e+16 }, "positrons": { - "particle_momentum_x": 5.663705657675971e-20, + "particle_momentum_x": 5.668407759026453e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705657675969e-20, - "particle_position_x": 0.65536, + "particle_momentum_z": 5.668407759026454e-20, + "particle_position_x": 0.6553599999999999, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 + }, + "electrons": { + "particle_momentum_x": 5.668407759026452e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.668407759026452e-20, + "particle_position_x": 0.65536, + "particle_position_y": 0.6553600000000002, + "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json index c9339aeb1aa..7098b83dbac 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json @@ -1,30 +1,30 @@ { - "electrons": { - "particle_momentum_x": 9.520485222844096e-20, - "particle_position_x": 2.6214400000233367, - "particle_position_y": 2.6214400000233367, - "particle_position_z": 2.6214400000000007, - "particle_weight": 128000000000.00002 - }, "lev=0": { - "Bx": 78.90261993791012, - "By": 78.90261993794903, - "Bz": 78.9041393998435, - "Ex": 84580428018688.97, - "Ey": 84580428018688.73, - "Ez": 84580441923246.2, - "F": 685.6927382678912, - "divE": 7.94184911020468e+19, - "jx": 6.007457695701311e+16, - "jy": 6.007457695701312e+16, - "jz": 6.007453636158103e+16, + "Bx": 86.03228226677659, + "By": 86.03228226679013, + "Bz": 86.033798606458, + "Ex": 84659309211473.42, + "Ey": 84659309211473.42, + "Ez": 84659323064827.52, + "F": 700.2020613746595, + "divE": 7.94984970149747e+19, + "jx": 5.999100369888243e+16, + "jy": 5.999100369888247e+16, + "jz": 5.999096310379822e+16, "part_per_cell": 524288.0, - "rho": 705198439.2596256 + "rho": 705877712.4251474 }, "positrons": { - "particle_momentum_z": 9.520479438980041e-20, - "particle_position_x": 2.6214400000233367, - "particle_position_y": 2.6214400000233358, + "particle_momentum_z": 9.508931687286131e-20, + "particle_position_x": 2.621440000034161, + "particle_position_y": 2.6214400000341618, "particle_position_z": 2.62144 + }, + "electrons": { + "particle_momentum_x": 9.508937471997448e-20, + "particle_position_x": 2.6214400000341604, + "particle_position_y": 2.621440000034161, + "particle_position_z": 2.6214399999999993, + "particle_weight": 128000000000.00002 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json index 487d4a89828..e1785dcce4d 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json @@ -1,28 +1,28 @@ { - "electrons": { - "particle_momentum_x": 9.633869745818886e-20, - "particle_position_x": 2.621440000001177, - "particle_position_y": 2.6214400000011784, - "particle_position_z": 2.6214400000000007, - "particle_weight": 128000000000.00002 - }, "lev=0": { - "Bx": 80.96035538655111, - "By": 80.96035538657691, - "Bz": 80.96271445263956, - "Ex": 84777489275096.88, - "Ey": 84777489275096.88, - "Ez": 84777485856239.4, - "jx": 6.08447015360442e+16, - "jy": 6.084470153604425e+16, - "jz": 6.084470085554113e+16, + "Bx": 88.73949682623977, + "By": 88.73949682627456, + "Bz": 88.74185838578241, + "Ex": 84851140116479.64, + "Ey": 84851140116479.62, + "Ez": 84851136696893.72, + "jx": 6.075773528102969e+16, + "jy": 6.0757735281029736e+16, + "jz": 6.075773460167053e+16, "part_per_cell": 524288.0, - "rho": 703546536.8089281 + "rho": 704298346.5173542 }, "positrons": { - "particle_momentum_z": 9.63386961193585e-20, - "particle_position_x": 2.621440000001177, - "particle_position_y": 2.621440000001179, - "particle_position_z": 2.6214400000000007 + "particle_momentum_z": 9.621748455552108e-20, + "particle_position_x": 2.621440000001309, + "particle_position_y": 2.6214400000013076, + "particle_position_z": 2.6214399999999998 + }, + "electrons": { + "particle_momentum_x": 9.621748589257923e-20, + "particle_position_x": 2.621440000001309, + "particle_position_y": 2.6214400000013076, + "particle_position_z": 2.6214399999999998, + "particle_weight": 128000000000.00002 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir.json b/Regression/Checksum/benchmarks_json/Python_Langmuir.json index 950d7f9194a..46867790cb4 100644 --- a/Regression/Checksum/benchmarks_json/Python_Langmuir.json +++ b/Regression/Checksum/benchmarks_json/Python_Langmuir.json @@ -1,13 +1,13 @@ { + "lev=0": { + "Ex": 3047094224871324.5, + "jx": 3.294250379355652e+18 + }, "electrons": { - "particle_momentum_x": 1.7507165884671642e-17, - "particle_position_x": 10.350911572498216, - "particle_position_y": 10.485760000000003, - "particle_position_z": 10.485760000000003, + "particle_momentum_x": 1.740964347107983e-17, + "particle_position_x": 10.351045542291539, + "particle_position_y": 10.485759999999997, + "particle_position_z": 10.48576, "particle_weight": 320000000000.00006 - }, - "lev=0": { - "Ex": 3091324128949712.5, - "jx": 3.3015975627803766e+18 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json b/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json index 2d1ca2f23c5..d152ff41998 100644 --- a/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json @@ -1,12 +1,12 @@ { + "lev=0": { + "Ex": 53556162492188.62, + "jx": 5.722548841990386e+16 + }, "electrons": { - "particle_momentum_x": 1.0449418710231497e-19, - "particle_position_x": 0.0831205701527011, - "particle_position_y": 0.08192, + "particle_momentum_x": 1.0581442023547547e-19, + "particle_position_x": 0.0831038802568349, + "particle_position_y": 0.08191999999999999, "particle_weight": 8000000000000001.0 - }, - "lev=0": { - "Ex": 54311177004701.516, - "jx": 5.639670983525456e+16 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json b/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json index 2129e8af565..2b4297e3627 100644 --- a/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json +++ b/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json @@ -1,32 +1,32 @@ { - "electrons": { - "particle_momentum_x": 6.808479576836945e-20, - "particle_momentum_y": 6.876934757849776e-20, - "particle_momentum_z": 6.802215755177324e-20, - "particle_position_x": 6339.279719574188, - "particle_position_y": 6400.519522256783, - "particle_position_z": 6332.552838182088, - "particle_weight": 4999999954710.04 - }, "lev=0": { - "Bx": 2.18760546171463e-05, - "By": 1.777427672926153e-05, - "Bz": 1.798431779205324e-05, - "Ex": 262735.3418998487, - "Ey": 264285.9081716973, - "Ez": 262283.0576561827, - "jx": 930.8363498458478, - "jy": 940.577756171662, - "jz": 929.9701516370898, + "Bx": 2.1885709623428054e-05, + "By": 1.7781538740987698e-05, + "Bz": 1.798374460438725e-05, + "Ex": 262814.04898035893, + "Ey": 264360.66474815976, + "Ez": 262360.8533027486, + "jx": 931.6306925490601, + "jy": 941.3279496407436, + "jz": 930.7478072804985, "part_per_cell": 65536.0 }, + "electrons": { + "particle_momentum_x": 6.823819887231586e-20, + "particle_momentum_y": 6.891558962132863e-20, + "particle_momentum_z": 6.817050175691089e-20, + "particle_position_x": 6339.113170504729, + "particle_position_y": 6400.3586668158905, + "particle_position_z": 6332.387112999683, + "particle_weight": 4999999954710.04 + }, "protons": { - "particle_momentum_x": 3.177356695273816e-21, - "particle_momentum_y": 3.165603704977051e-21, - "particle_momentum_z": 3.157786057110636e-21, - "particle_position_x": 6550.923797287247, - "particle_position_y": 6537.195882086984, - "particle_position_z": 6523.379262220709, + "particle_momentum_x": 2.9968929267374238e-21, + "particle_momentum_y": 2.984482330834465e-21, + "particle_momentum_z": 2.9781449297861096e-21, + "particle_position_x": 6550.923883255297, + "particle_position_y": 6537.195968516041, + "particle_position_z": 6523.379347813018, "particle_weight": 4999999954710.04 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd.json b/Regression/Checksum/benchmarks_json/restart_psatd.json index d22fb5b57e7..fd31e7057f4 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd.json @@ -1,15 +1,33 @@ { "lev=0": { - "Bx": 116102.13010406512, - "By": 245467.5412026496, - "Bz": 65305.35287114741, - "Ex": 118565481099326.73, - "Ey": 18640100573642.96, - "Ez": 51777969821120.38, - "jx": 2.1177246490642464e+16, - "jy": 351544815339900.3, - "jz": 4573466562652713.0, - "rho": 22294533.587530266 + "Bx": 116097.15119490701, + "By": 245135.43138545044, + "Bz": 65087.42714316293, + "Ex": 118461420042835.34, + "Ey": 18622845483678.824, + "Ez": 51793144992361.125, + "jx": 2.1162465965376624e+16, + "jy": 349487178458066.6, + "jz": 4563460427657682.0, + "rho": 22252333.818822004 + }, + "driverback": { + "particle_momentum_x": 4.813131349021332e+21, + "particle_momentum_y": 5.16548074090123e+21, + "particle_momentum_z": 3.005830430844926e+25, + "particle_position_x": 0.001649481123084974, + "particle_position_y": 0.0016172218745428432, + "particle_position_z": 0.4899808854005956, + "particle_weight": 6241509074.460762 + }, + "plasma_e": { + "particle_momentum_x": 2.084066965693142e-19, + "particle_momentum_y": 1.330336553422521e-20, + "particle_momentum_z": 2.6335518561017005e-17, + "particle_position_x": 0.4991724057788647, + "particle_position_y": 0.49919697481183745, + "particle_position_z": 0.4562433914656611, + "particle_weight": 33067341227104.625 }, "beam": { "particle_momentum_x": 4.178482505909375e-19, @@ -20,6 +38,15 @@ "particle_position_z": 1.901942694291968, "particle_weight": 3120754537.230381 }, + "plasma_p": { + "particle_momentum_x": 2.097709570232301e-19, + "particle_momentum_y": 3.402465268604418e-21, + "particle_momentum_z": 4.82960000875208e-14, + "particle_position_x": 0.49912500313484137, + "particle_position_y": 0.4991249983460674, + "particle_position_z": 0.4563845471150555, + "particle_weight": 33067341227104.625 + }, "driver": { "particle_momentum_x": 4.700436405078562e+21, "particle_momentum_y": 4.6785862113093076e+21, @@ -28,32 +55,5 @@ "particle_position_y": 0.0016212373149699414, "particle_position_z": 0.30417779498653386, "particle_weight": 6241509074.460762 - }, - "driverback": { - "particle_momentum_x": 4.813131349021332e+21, - "particle_momentum_y": 5.16548074090123e+21, - "particle_momentum_z": 3.005830430844926e+25, - "particle_position_x": 0.001649481123084974, - "particle_position_y": 0.0016172218745428432, - "particle_position_z": 0.4899808854005956, - "particle_weight": 6241509074.460762 - }, - "plasma_e": { - "particle_momentum_x": 2.0912364079892767e-19, - "particle_momentum_y": 1.3510804924903876e-20, - "particle_momentum_z": 2.6336070297356917e-17, - "particle_position_x": 0.49917333523960167, - "particle_position_y": 0.49919828074154127, - "particle_position_z": 0.45624167532314724, - "particle_weight": 33067341227104.625 - }, - "plasma_p": { - "particle_momentum_x": 2.1053025446547977e-19, - "particle_momentum_y": 3.3272838305044046e-21, - "particle_momentum_z": 4.829599953646669e-14, - "particle_position_x": 0.49912500297599893, - "particle_position_y": 0.49912499825258866, - "particle_position_z": 0.45638454709795806, - "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json index 50971f0d45e..cfd031d6e33 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json @@ -1,24 +1,15 @@ { "lev=0": { - "Bx": 115622.1878941125, - "By": 243403.66715242947, - "Bz": 63602.66529020351, - "Ex": 117118934931289.56, - "Ey": 18448377440588.27, - "Ez": 50821967818817.24, - "jx": 2.1044522674691452e+16, - "jy": 329314843111847.94, - "jz": 4524787623275627.0, - "rho": 22128883.53068064 - }, - "driverback": { - "particle_momentum_x": 4.813131349021332e+21, - "particle_momentum_y": 5.16548074090123e+21, - "particle_momentum_z": 3.005830430844926e+25, - "particle_position_x": 0.001649481123084974, - "particle_position_y": 0.0016172218745428432, - "particle_position_z": 0.4899808854005956, - "particle_weight": 6241509074.460762 + "Bx": 115610.21497814235, + "By": 243019.564489834, + "Bz": 63397.3416561916, + "Ex": 117012138466922.22, + "Ey": 18429606525820.12, + "Ez": 50835997822445.34, + "jx": 2.1028213761477244e+16, + "jy": 326844980054897.7, + "jz": 4514643615865121.0, + "rho": 22084564.546177678 }, "beam": { "particle_momentum_x": 4.178482505909375e-19, @@ -29,24 +20,33 @@ "particle_position_z": 1.901942694291968, "particle_weight": 3120754537.230381 }, - "plasma_e": { - "particle_momentum_x": 1.9905288525315664e-19, - "particle_momentum_y": 1.2685401564810352e-20, - "particle_momentum_z": 2.6334746597885943e-17, - "particle_position_x": 0.4991702885212871, - "particle_position_y": 0.4991949978513417, - "particle_position_z": 0.4562491611716817, + "plasma_p": { + "particle_momentum_x": 1.9959818202810056e-19, + "particle_momentum_y": 2.9455107653405726e-21, + "particle_momentum_z": 4.82960013792663e-14, + "particle_position_x": 0.4991250027534606, + "particle_position_y": 0.49912499804419774, + "particle_position_z": 0.4563845471451087, "particle_weight": 33067341227104.625 }, - "plasma_p": { - "particle_momentum_x": 2.002897995377179e-19, - "particle_momentum_y": 2.87423099731012e-21, - "particle_momentum_z": 4.8296000849128737e-14, - "particle_position_x": 0.49912500259846493, - "particle_position_y": 0.4991249979476612, - "particle_position_z": 0.45638454712861487, + "plasma_e": { + "particle_momentum_x": 1.9841122555955512e-19, + "particle_momentum_y": 1.2475587913571367e-20, + "particle_momentum_z": 2.6334235362474606e-17, + "particle_position_x": 0.49916947686056645, + "particle_position_y": 0.49919374997004357, + "particle_position_z": 0.4562507922838501, "particle_weight": 33067341227104.625 }, + "driverback": { + "particle_momentum_x": 4.813131349021332e+21, + "particle_momentum_y": 5.16548074090123e+21, + "particle_momentum_z": 3.005830430844926e+25, + "particle_position_x": 0.001649481123084974, + "particle_position_y": 0.0016172218745428432, + "particle_position_z": 0.4899808854005956, + "particle_weight": 6241509074.460762 + }, "driver": { "particle_momentum_x": 4.700436405078562e+21, "particle_momentum_y": 4.6785862113093076e+21, diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index d1f9eb16f47..5e3380cc1a6 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1284,8 +1284,16 @@ WarpX::ReadParameters () } } - // Use same shape factors in all directions, for gathering - if (field_gathering_algo == GatheringAlgo::MomentumConserving) { galerkin_interpolation = false; } + // Use same shape factors in all directions + // - with momentum-conserving field gathering + if (field_gathering_algo == GatheringAlgo::MomentumConserving) {galerkin_interpolation = false;} + // - with direct current deposition and the EM solver + if( electromagnetic_solver_id != ElectromagneticSolverAlgo::None && + electromagnetic_solver_id != ElectromagneticSolverAlgo::HybridPIC ) { + if (current_deposition_algo == CurrentDepositionAlgo::Direct) { + galerkin_interpolation = false; + } + } { const ParmParse pp_interpolation("interpolation"); From fd2248ace32d2022fb89d6d04bacc5410428ae50 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 4 Jul 2024 07:18:54 +0200 Subject: [PATCH 40/87] Release 24.07 (#5025) * AMReX: 24.07 * pyAMReX: 24.07 * WarpX: 24.07 --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 7e1cf7ae63f..190294bb0e1 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a && cd - + cd ../amrex && git checkout --detach 24.07 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e2d1ebba9c..23427a00ec8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.06) +project(WarpX VERSION 24.07) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 18ca8370e80..7997b1e338c 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.06' +version = u'24.07' # The full version, including alpha/beta/rc tags. -release = u'24.06' +release = u'24.07' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index 9f9c8d7d736..e4adbb6c458 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.06', + version = '24.07', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index e4e5d7b8678..0003dcff134 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a +branch = 24.07 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index a18f39b89c1..aa7c74cf21c 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a +branch = 24.07 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 632b9edd4f4..0b90b09df30 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.06 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.07 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "259db7cfb99e7d1d2ab4bec9b1587fdf624a138a" +set(WarpX_amrex_branch "24.07" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 90aee6cb930..5c17681d4b6 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.06 CONFIG REQUIRED) + find_package(pyAMReX 24.07 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "fd7126cb907be7e3c8570930b79209db5cd7c492" +set(WarpX_pyamrex_branch "24.07" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 6ac22466ef0..37264bef243 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a && cd - +cd amrex && git checkout --detach 24.07 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index 6ad6fd81960..f05d5a71899 100644 --- a/setup.py +++ b/setup.py @@ -280,7 +280,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.06', + version = '24.07', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From f9f3db5534836b5ddc53b10310007522bf84e9cd Mon Sep 17 00:00:00 2001 From: Steve Richardson Date: Fri, 5 Jul 2024 12:56:46 -0400 Subject: [PATCH 41/87] Fix typo in parameters.rst (#5030) --- Docs/source/usage/parameters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 4dc888df3d0..313c5f0d740 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -86,7 +86,7 @@ Overall simulation parameters * ``explicit``: Use an explicit solver, such as the standard FDTD or PSATD - * ``theta_implicit_em``: Use a fully implicit electromagnetic solver with a time-biasing parameter theta bound between 0.5 and 1.0. Exact energy conservation is achieved using theta = 0.5. Maximal damping of high-k modes is obtained using theta = 1.0. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed (PS) JNFK. + * ``theta_implicit_em``: Use a fully implicit electromagnetic solver with a time-biasing parameter theta bound between 0.5 and 1.0. Exact energy conservation is achieved using theta = 0.5. Maximal damping of high-k modes is obtained using theta = 1.0. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed (PS) JFNK. The algorithm itself is numerical stable for large time steps. That is, it does not require time steps that resolve the plasma period or the CFL condition for light waves. However, the practicality of using a large time step depends on the nonlinear solver. Note that the Picard solver is for demonstration only. It is inefficient and will most like not converge when :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relevant physics. The Picard method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. From bf77d6d6c5ef75e61c13c3ab9a3014cd18a04b35 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Sat, 6 Jul 2024 02:33:06 +0200 Subject: [PATCH 42/87] replace std::is_same::value with std::is_same_v (#5028) --- Source/Diagnostics/ReducedDiags/FieldReduction.H | 6 +++--- Source/Initialization/WarpXInitData.cpp | 4 ++-- .../Collision/BinaryCollision/BinaryCollision.H | 2 +- Source/Particles/LaserParticleContainer.cpp | 2 +- Source/Utils/Parser/ParserUtils.H | 12 ++++++------ Source/ablastr/fields/PoissonSolver.H | 2 +- Source/ablastr/fields/VectorPoissonSolver.H | 2 +- Tools/QedTablesUtils/Source/QedTableGenerator.cpp | 4 ++-- Tools/QedTablesUtils/Source/QedTableReader.cpp | 4 ++-- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index a15d6489e2c..f467499ad56 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -212,15 +212,15 @@ public: amrex::Real reduce_value = amrex::get<0>(reduce_data.value()); // MPI reduce - if (std::is_same::value) + if (std::is_same_v) { amrex::ParallelDescriptor::ReduceRealMax(reduce_value); } - if (std::is_same::value) + if (std::is_same_v) { amrex::ParallelDescriptor::ReduceRealMin(reduce_value); } - if (std::is_same::value) + if (std::is_same_v) { amrex::ParallelDescriptor::ReduceRealSum(reduce_value); // If reduction operation is a sum, multiply the value by the cell volume so that the diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 68b38153f8d..03393debeae 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -155,13 +155,13 @@ WarpX::PrintMainPICparameters () amrex::Print() << "-------------------------------------------------------------------------------\n"; // print warpx build information - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { amrex::Print() << "Precision: | SINGLE" << "\n"; } else { amrex::Print() << "Precision: | DOUBLE" << "\n"; } - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { amrex::Print() << "Particle precision: | SINGLE" << "\n"; } else { diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index b3d39a9c581..a386cb3012e 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -108,7 +108,7 @@ public: } m_have_product_species = !m_product_species.empty(); - if ((std::is_same::value) && (m_have_product_species)) { + if ((std::is_same_v) && (m_have_product_species)) { WARPX_ABORT_WITH_MESSAGE( "Binary collision " + collision_name + " does not produce species. Thus, `product_species` should not be specified in the input script." ); } diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 0693a13c1f9..bd266ab368a 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -719,7 +719,7 @@ LaserParticleContainer::ComputeSpacing (int lev, Real& Sx, Real& Sy) const #if !defined(WARPX_DIM_RZ) constexpr float small_float_coeff = 1.e-25f; constexpr double small_double_coeff = 1.e-50; - constexpr Real small_coeff = std::is_same::value ? + constexpr Real small_coeff = std::is_same_v ? static_cast(small_float_coeff) : static_cast(small_double_coeff); const auto eps = static_cast(dx[0]*small_coeff); diff --git a/Source/Utils/Parser/ParserUtils.H b/Source/Utils/Parser/ParserUtils.H index dbc4845d1bf..96937abdbbe 100644 --- a/Source/Utils/Parser/ParserUtils.H +++ b/Source/Utils/Parser/ParserUtils.H @@ -147,7 +147,7 @@ namespace utils::parser auto parser = makeParser(str_val, {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); @@ -174,7 +174,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -217,7 +217,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -250,7 +250,7 @@ namespace utils::parser Store_parserString(a_pp, str, str_val); auto parser = makeParser(str_val, {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -270,7 +270,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -308,7 +308,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index e5340e825c9..fed76b2987b 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -315,7 +315,7 @@ computePhi (amrex::Vector const & rho, } // Run additional operations, such as calculation of the E field for embedded boundaries - if constexpr (!std::is_same::value) { + if constexpr (!std::is_same_v) { if (post_phi_calculation.has_value()) { post_phi_calculation.value()(mlmg, lev); } diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index d49335723d8..3ef96c30c84 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -241,7 +241,7 @@ computeVectorPotential ( amrex::Vector > co curr[lev][adim]->mult(-1._rt/ablastr::constant::SI::mu0); } // Loop over adim // Run additional operations, such as calculation of the B fields for embedded boundaries - if constexpr (!std::is_same::value) { + if constexpr (!std::is_same_v) { if (post_A_calculation.has_value()) { post_A_calculation.value()(mlmg, lev); } diff --git a/Tools/QedTablesUtils/Source/QedTableGenerator.cpp b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp index dfbde1df050..1ea62b5c6ed 100644 --- a/Tools/QedTablesUtils/Source/QedTableGenerator.cpp +++ b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp @@ -106,7 +106,7 @@ template void GenerateTableBW (const ParsedArgs& args, const string& outfile_name) { cout << " Generating BW table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; if (!Contains(args, "--dndt_chi_min") || !Contains(args, "--dndt_chi_max") || !Contains(args, "--dndt_how_many") || !Contains(args, "--pair_chi_min") || @@ -173,7 +173,7 @@ template void GenerateTableQS (const ParsedArgs& args, const string& outfile_name) { cout << " Generating QS table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; if (!Contains(args, "--dndt_chi_min") || !Contains(args, "--dndt_chi_max") || !Contains(args, "--dndt_how_many") || !Contains(args, "--em_chi_min") || diff --git a/Tools/QedTablesUtils/Source/QedTableReader.cpp b/Tools/QedTablesUtils/Source/QedTableReader.cpp index ae689a87643..ba9d58775f2 100644 --- a/Tools/QedTablesUtils/Source/QedTableReader.cpp +++ b/Tools/QedTablesUtils/Source/QedTableReader.cpp @@ -117,7 +117,7 @@ template void ReadTableBW (const string& input_file, const string& outfile_name) { cout << " Reading BW table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; auto ifs = ifstream(input_file, std::ios::binary); auto raw_data = vector(istreambuf_iterator(ifs), {}); @@ -161,7 +161,7 @@ void ReadTableQS ( const string& input_file, const string& outfile_name) { cout << " Reading QS table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; auto ifs = ifstream(input_file, std::ios::binary); auto raw_data = vector(istreambuf_iterator(ifs), {}); From 0c333653ce35d25cace06754b03518e3566b0139 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 8 Jul 2024 07:35:20 -0700 Subject: [PATCH 43/87] Remove debug build type from Langmuir 2D fluid test (#5031) While working on another PR, I noticed that the CI test `Langmuir_fluid_2D` was compiled in debug mode. The test was added originally in #3991 and it might be that the debug build type was left over unintentionally. In general, I think we avoid running CI tests in debug mode, in order to keep the runtime of the tests as low as possible (the current runtime of this test in debug mode is around 80 seconds). However, we might have changed policy in the last months and I might not be up-to-date, so feel free to let me know and close this PR without merging if it is not relevant. --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index aa7c74cf21c..e33be6b42ba 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -1173,7 +1173,7 @@ inputFile = Examples/Tests/langmuir_fluids/inputs_2d runtime_params = dim = 2 addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -DCMAKE_BUILD_TYPE=Debug +cmakeSetupOpts = -DWarpX_DIMS=2 restartTest = 0 useMPI = 1 numprocs = 2 From ce9f4af505c5b02d66573c3163dfbc9d338fecc3 Mon Sep 17 00:00:00 2001 From: Olga Shapoval <30510597+oshapoval@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:59:26 -0700 Subject: [PATCH 44/87] Add ability to load external particle fields from file (#4997) * Update documentation * Allocate fields * Always parse filename * Add option "FromFile" in external particles fields * Create two separate examples * Update documentation * Allocate aux array * Update aux to take into account copy * Add external fields from file to auxiliary data * Update tests * Remove debugging Print statements * If averaged PSATD is used, we copy Efield_avg_fp to Efield_aux, otherwise copy Efield_fp. * Do not access cells that have invalid data when summing external fields * Update path of checkpoint files * Update comments * Update number of components * Fix clang tidy error * Make the code compatible with momentum-conserving gather * Add Python interface and tests * Use PICMI version that defined LoadAppliedField * Update file names * Update PICMI standard version * Allocate dedicated MultiFabs for external particle fields * Some clean-up * Performance optimization: do not use external fields in gather kernel * Allow external particle fields and external grid fields to be simultaneously defined * External fields are incompatible with the moving window * Support load balancing * Update Source/Initialization/WarpXInitData.cpp * Update Source/Initialization/WarpXInitData.cpp --------- Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 19 ++- ...s_3d.py => PICMI_inputs_3d_grid_fields.py} | 4 +- .../PICMI_inputs_3d_particle_fields.py | 140 ++++++++++++++++++ .../{inputs_rz => inputs_rz_grid_fields} | 0 .../inputs_rz_particle_fields | 65 ++++++++ Python/pywarpx/picmi.py | 7 + ...ldRZ.json => LoadExternalFieldRZGrid.json} | 0 .../LoadExternalFieldRZParticles.json | 22 +++ ...on => Python_LoadExternalGridField3D.json} | 0 .../Python_LoadExternalParticleField3D.json | 22 +++ Regression/WarpX-tests.ini | 51 ++++++- Source/Initialization/ExternalField.cpp | 1 - Source/Initialization/WarpXInitData.cpp | 50 +++++++ Source/Parallelization/WarpXComm.cpp | 44 +++++- Source/Parallelization/WarpXRegrid.cpp | 6 + Source/Particles/Gather/GetExternalFields.cpp | 12 ++ Source/WarpX.H | 2 + Source/WarpX.cpp | 92 ++++++++---- requirements.txt | 2 +- 19 files changed, 497 insertions(+), 42 deletions(-) rename Examples/Tests/LoadExternalField/{PICMI_inputs_3d.py => PICMI_inputs_3d_grid_fields.py} (96%) create mode 100644 Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py rename Examples/Tests/LoadExternalField/{inputs_rz => inputs_rz_grid_fields} (100%) create mode 100644 Examples/Tests/LoadExternalField/inputs_rz_particle_fields rename Regression/Checksum/benchmarks_json/{LoadExternalFieldRZ.json => LoadExternalFieldRZGrid.json} (100%) create mode 100644 Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json rename Regression/Checksum/benchmarks_json/{Python_LoadExternalField3D.json => Python_LoadExternalGridField3D.json} (100%) create mode 100644 Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 313c5f0d740..d6420440935 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1747,8 +1747,8 @@ are applied to the grid directly. In particular, these fields can be seen in the One can refer to input files in ``Examples/Tests/LoadExternalField`` for more information. Regarding how to prepare the openPMD data file, one can refer to the `openPMD-example-datasets `__. - Note that if both `B_ext_grid_init_style` and `E_ext_grid_init_style` are set to - `read_from_file`, the openPMD file specified by `warpx.read_fields_from_path` + Note that if both ``B_ext_grid_init_style`` and ``E_ext_grid_init_style`` are set to + ``read_from_file``, the openPMD file specified by ``warpx.read_fields_from_path`` should contain both B and E external fields data. * ``warpx.E_external_grid`` & ``warpx.B_external_grid`` (list of `3 floats`) @@ -1804,6 +1804,21 @@ are applied to the particles directly, at each timestep. As a results, these fie Note that the position is defined in Cartesian coordinates, as a function of (x,y,z), even for RZ. + * ``read_from_file``: load the external field from an openPMD file. + An additional parameter, indicating the path of an openPMD data file, ``particles.read_fields_from_path`` + must be specified, from which the external E field data can be loaded into WarpX. + One can refer to input files in ``Examples/Tests/LoadExternalField`` for more information. + Regarding how to prepare the openPMD data file, one can refer to + the `openPMD-example-datasets `__. + Note that if both ``B_ext_particle_init_style`` and ``E_ext_particle_init_style`` are set to + ``read_from_file``, the openPMD file specified by ``particles.read_fields_from_path`` + should contain both B and E external fields data. + + .. note:: + + When using ``read_from_file``, the fields loaded from the file will be interpolated + to the resolution of the grid used for the simulation. + * ``repeated_plasma_lens``: apply a series of plasma lenses. The properties of the lenses are defined in the lab frame by the input parameters: diff --git a/Examples/Tests/LoadExternalField/PICMI_inputs_3d.py b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py similarity index 96% rename from Examples/Tests/LoadExternalField/PICMI_inputs_3d.py rename to Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py index 849d4d2be08..d128d9c10e0 100644 --- a/Examples/Tests/LoadExternalField/PICMI_inputs_3d.py +++ b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py @@ -93,7 +93,7 @@ species=[ions], data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], write_dir='.', - warpx_file_prefix='Python_LoadExternalField3D_plt' + warpx_file_prefix='Python_LoadExternalGridField3D_plt' ) field_diag = picmi.FieldDiagnostic( name='diag1', @@ -101,7 +101,7 @@ period=300, data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], write_dir='.', - warpx_file_prefix='Python_LoadExternalField3D_plt' + warpx_file_prefix='Python_LoadExternalGridField3D_plt' ) ################################# diff --git a/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py new file mode 100644 index 00000000000..7bf7c5a084c --- /dev/null +++ b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# +# --- Input file for loading initial field from openPMD file. + +from pywarpx import picmi + +constants = picmi.constants + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_steps = 300 + +max_grid_size = 40 +nx = max_grid_size +ny = max_grid_size +nz = max_grid_size + +xmin = -1 +xmax = 1 +ymin = xmin +ymax = xmax +zmin = 0 +zmax = 5 + +number_per_cell = 200 + +################################# +############ NUMERICS ########### +################################# + +verbose = 1 +dt = 4.4e-7 +use_filter = 0 + +# Order of particle shape factors +particle_shape = 1 + +################################# +############ PLASMA ############# +################################# + +ion_dist = picmi.ParticleListDistribution( + x=0.0, + y=0.2, + z=2.5, + ux=9.5e-05*constants.c, + uy=0.0*constants.c, + uz=1.34e-4*constants.c, + weight=1.0 + ) + +ions = picmi.Species( + particle_type='H', + name='proton', charge='q_e',mass="m_p", + warpx_do_not_deposit=1, + initial_distribution=ion_dist + ) + +################################# +######## INITIAL FIELD ########## +################################# + +applied_field = picmi.LoadAppliedField( + read_fields_from_path="../../../../openPMD-example-datasets/example-femm-3d.h5", + load_E=False + ) + +################################# +###### GRID AND SOLVER ########## +################################# + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + warpx_max_grid_size=max_grid_size, + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=['dirichlet', 'dirichlet', 'dirichlet'], + upper_boundary_conditions=['dirichlet', 'dirichlet', 'dirichlet'], + lower_boundary_conditions_particles=['absorbing', 'absorbing', 'absorbing'], + upper_boundary_conditions_particles=['absorbing', 'absorbing', 'absorbing'] + ) +solver = picmi.ElectrostaticSolver(grid=grid) + +################################# +######### DIAGNOSTICS ########### +################################# + +particle_diag = picmi.ParticleDiagnostic( + name='diag1', + period=300, + species=[ions], + data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], + write_dir='.', + warpx_file_prefix='Python_LoadExternalParticleField3D_plt' + ) +field_diag = picmi.FieldDiagnostic( + name='diag1', + grid=grid, + period=300, + data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], + write_dir='.', + warpx_file_prefix='Python_LoadExternalParticleField3D_plt' + ) + +################################# +####### SIMULATION SETUP ######## +################################# + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=verbose, + warpx_serialize_initial_conditions=False, + warpx_grid_type='collocated', + warpx_do_dynamic_scheduling=False, + warpx_use_filter=use_filter, + time_step_size=dt, + particle_shape=particle_shape + ) + +sim.add_applied_field(applied_field) + +sim.add_species( + ions, + layout=picmi.PseudoRandomLayout( + n_macroparticles_per_cell=number_per_cell, grid=grid + ) + ) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(particle_diag) + +################################# +##### SIMULATION EXECUTION ###### +################################# + +#sim.write_input_file('PICMI_inputs_3d') +sim.step(max_steps) diff --git a/Examples/Tests/LoadExternalField/inputs_rz b/Examples/Tests/LoadExternalField/inputs_rz_grid_fields similarity index 100% rename from Examples/Tests/LoadExternalField/inputs_rz rename to Examples/Tests/LoadExternalField/inputs_rz_grid_fields diff --git a/Examples/Tests/LoadExternalField/inputs_rz_particle_fields b/Examples/Tests/LoadExternalField/inputs_rz_particle_fields new file mode 100644 index 00000000000..b76d4cb7efc --- /dev/null +++ b/Examples/Tests/LoadExternalField/inputs_rz_particle_fields @@ -0,0 +1,65 @@ +warpx.serialize_initial_conditions = 0 +warpx.do_dynamic_scheduling = 0 +particles.do_tiling = 0 + +particles.B_ext_particle_init_style = "read_from_file" +particles.read_fields_from_path = "../../../../openPMD-example-datasets/example-femm-thetaMode.h5" + +warpx.grid_type = collocated +warpx.do_electrostatic = labframe + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 300 +amr.n_cell = 40 40 +warpx.numprocs = 1 1 +amr.max_level = 0 +geometry.dims = RZ + +geometry.prob_lo = 0.0 0.0 +geometry.prob_hi = 1.0 5.0 + +################################# +###### Boundary Condition ####### +################################# +boundary.field_lo = none pec +boundary.field_hi = pec pec +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = 4.40917904849092e-7 +warpx.use_filter = 0 + +# Order of particle shape factors +algo.particle_shape = 1 + +################################# +############ PLASMA ############# +################################# +particles.species_names = proton +proton.injection_style = "SingleParticle" +proton.single_particle_pos = 0.0 0.2 2.5 +proton.single_particle_u = 9.506735958279367e-05 0.0 0.00013435537232359165 +proton.single_particle_weight = 1.0 +proton.do_not_deposit = 1 +proton.mass = m_p +proton.charge = q_e + +# Diagnostics +diagnostics.diags_names = diag1 chk +diag1.intervals = 300 +diag1.diag_type = Full + +chk.intervals = 150 +chk.diag_type = Full +chk.format = checkpoint diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 0048cb1e51b..42c0313a6e9 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1486,6 +1486,13 @@ def applied_field_initialize_inputs(self): expression = pywarpx.my_constants.mangle_expression(expression, self.mangle_dict) pywarpx.warpx.__setattr__(f'B{sdir}_external_grid_function(x,y,z)', expression) +class LoadAppliedField(picmistandard.PICMI_LoadAppliedField): + def applied_field_initialize_inputs(self): + pywarpx.particles.read_fields_from_path = self.read_fields_from_path + if self.load_E: + pywarpx.particles.E_ext_particle_init_style = 'read_from_file' + if self.load_B: + pywarpx.particles.B_ext_particle_init_style = 'read_from_file' class ConstantAppliedField(picmistandard.PICMI_ConstantAppliedField): def applied_field_initialize_inputs(self): diff --git a/Regression/Checksum/benchmarks_json/LoadExternalFieldRZ.json b/Regression/Checksum/benchmarks_json/LoadExternalFieldRZGrid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LoadExternalFieldRZ.json rename to Regression/Checksum/benchmarks_json/LoadExternalFieldRZGrid.json diff --git a/Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json b/Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json new file mode 100644 index 00000000000..dc4ffa3beb2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json @@ -0,0 +1,22 @@ +{ + "lev=0": { + "Br": 0.6380326770459374, + "Bt": 0.0, + "Bz": 4.850698699951099, + "Er": 0.0, + "Et": 0.0, + "Ez": 0.0, + "jr": 0.0, + "jt": 0.0, + "jz": 0.0 + }, + "proton": { + "particle_momentum_x": 1.2945379007125463e-23, + "particle_momentum_y": 7.901629565307941e-23, + "particle_momentum_z": 2.0004592432172922e-23, + "particle_position_x": 0.12402004585603355, + "particle_position_y": 4.363249204658946, + "particle_theta": 0.22200801011337173, + "particle_weight": 1.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LoadExternalField3D.json b/Regression/Checksum/benchmarks_json/Python_LoadExternalGridField3D.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_LoadExternalField3D.json rename to Regression/Checksum/benchmarks_json/Python_LoadExternalGridField3D.json diff --git a/Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json b/Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json new file mode 100644 index 00000000000..a34b2f136a6 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json @@ -0,0 +1,22 @@ +{ + "lev=0": { + "Bx": 29.8448958859583, + "By": 29.844895885958305, + "Bz": 197.55124310523018, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "proton": { + "particle_momentum_x": 1.2118953253556959e-23, + "particle_momentum_y": 7.822512598009088e-23, + "particle_momentum_z": 2.2761898274347412e-23, + "particle_position_x": 0.12238072371983327, + "particle_position_y": 0.009653941154598592, + "particle_position_z": 4.317544769595657, + "particle_weight": 1.0 + } +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index e33be6b42ba..3d1dbfe3b0f 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -2266,10 +2266,29 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/resampling/analysis_leveling_thinning.py -[LoadExternalFieldRZ] +[LoadExternalFieldRZGrid] buildDir = . -inputFile = Examples/Tests/LoadExternalField/inputs_rz -runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZ_chk chk.file_min_digits=5 +inputFile = Examples/Tests/LoadExternalField/inputs_rz_grid_fields +runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZGrid_chk chk.file_min_digits=5 +dim = 2 +addToCompileString = USE_RZ=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON +restartTest = 1 +restartFileNum = 150 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = proton +analysisRoutine = Examples/Tests/LoadExternalField/analysis_rz.py + +[LoadExternalFieldRZParticles] +buildDir = . +inputFile = Examples/Tests/LoadExternalField/inputs_rz_particle_fields +runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZParticles_chk chk.file_min_digits=5 dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON @@ -3448,11 +3467,31 @@ compileTest = 0 doVis = 0 analysisRoutine = Examples/analysis_default_openpmd_regression.py -[Python_LoadExternalField3D] +[Python_LoadExternalGridField3D] buildDir = . -inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d.py +inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py runtime_params = -customRunCmd = python3 PICMI_inputs_3d.py +customRunCmd = python3 PICMI_inputs_3d_grid_fields.py +dim = 3 +addToCompileString = USE_PYTHON_MAIN=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = proton +analysisRoutine = Examples/Tests/LoadExternalField/analysis_3d.py + +[Python_LoadExternalParticleField3D] +buildDir = . +inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py +runtime_params = +customRunCmd = python3 PICMI_inputs_3d_particle_fields.py dim = 3 addToCompileString = USE_PYTHON_MAIN=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON diff --git a/Source/Initialization/ExternalField.cpp b/Source/Initialization/ExternalField.cpp index e7a272e2e95..69120c0ae3c 100644 --- a/Source/Initialization/ExternalField.cpp +++ b/Source/Initialization/ExternalField.cpp @@ -172,7 +172,6 @@ ExternalFieldParams::ExternalFieldParams(const amrex::ParmParse& pp_warpx) // // External fields from file // - if (E_ext_grid_type == ExternalFieldType::read_from_file || B_ext_grid_type == ExternalFieldType::read_from_file){ const std::string read_fields_from_path="./"; diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 03393debeae..9a4c46b1c10 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1359,6 +1359,21 @@ void WarpX::CheckKnownIssues() void WarpX::LoadExternalFieldsFromFile (int const lev) { + // External fields from file are currently not compatible with the moving window + // In order to support the moving window, the MultiFab containing the external + // fields should be updated every time the window moves. + if ( (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) || + (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) || + (mypc->m_B_ext_particle_s == "read_from_file") || + (mypc->m_E_ext_particle_s == "read_from_file") ) { + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + WarpX::do_moving_window == 0, + "External fields from file are not compatible with the moving window." ); + } + + // External grid fields + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, @@ -1383,6 +1398,41 @@ WarpX::LoadExternalFieldsFromFile (int const lev) ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); +#endif + } + + // External particle fields + + if (mypc->m_B_ext_particle_s == "read_from_file") { + std::string external_fields_path; + const amrex::ParmParse pp_particles("particles"); + pp_particles.get("read_fields_from_path", external_fields_path ); +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][0].get(), "B", "r"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][1].get(), "B", "t"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][2].get(), "B", "z"); +#else + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][0].get(), "B", "x"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][1].get(), "B", "y"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][2].get(), "B", "z"); +#endif + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + std::string external_fields_path; + const amrex::ParmParse pp_particles("particles"); + pp_particles.get("read_fields_from_path", external_fields_path ); +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][0].get(), "E", "r"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][1].get(), "E", "t"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][2].get(), "E", "z"); +#else + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][0].get(), "E", "x"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][1].get(), "E", "y"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][2].get(), "E", "z"); #endif } } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index dcf5fbe5621..e7df489236e 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -18,6 +18,7 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpXComm_K.H" #include "WarpXSumGuardCells.H" +#include "Particles/MultiParticleContainer.H" #include #include @@ -59,6 +60,21 @@ WarpX::UpdateAuxilaryData () } else { UpdateAuxilaryDataStagToNodal(); } + + // When loading particle fields from file: add the external fields: + for (int lev = 0; lev <= finest_level; ++lev) { + if (mypc->m_E_ext_particle_s == "read_from_file") { + amrex::MultiFab::Add(*Efield_aux[lev][0], *E_external_particle_field[lev][0], 0, 0, E_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[lev][1], *E_external_particle_field[lev][1], 0, 0, E_external_particle_field[lev][1]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[lev][2], *E_external_particle_field[lev][2], 0, 0, E_external_particle_field[lev][2]->nComp(), guard_cells.ng_FieldGather); + } + if (mypc->m_B_ext_particle_s == "read_from_file") { + amrex::MultiFab::Add(*Bfield_aux[lev][0], *B_external_particle_field[lev][0], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[lev][1], *B_external_particle_field[lev][1], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[lev][2], *B_external_particle_field[lev][2], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + } + } + } void @@ -293,6 +309,30 @@ WarpX::UpdateAuxilaryDataStagToNodal () void WarpX::UpdateAuxilaryDataSameType () { + // Update aux field, including guard cells, up to ng_FieldGather + const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + + // Level 0: Copy from fine to aux + // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the + // same MultiFab object. MultiFab::Copy operation automatically detects this and does nothing in this case. + if (WarpX::fft_do_time_averaging) + { + MultiFab::Copy(*Efield_aux[0][0], *Efield_avg_fp[0][0], 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *Efield_avg_fp[0][1], 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *Efield_avg_fp[0][2], 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][0], *Bfield_avg_fp[0][0], 0, 0, Bfield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][1], *Bfield_avg_fp[0][1], 0, 0, Bfield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][2], *Bfield_avg_fp[0][2], 0, 0, Bfield_aux[0][2]->nComp(), ng_src); + } + else + { + MultiFab::Copy(*Efield_aux[0][0], *Efield_fp[0][0], 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *Efield_fp[0][1], 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *Efield_fp[0][2], 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][0], *Bfield_fp[0][0], 0, 0, Bfield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][1], *Bfield_fp[0][1], 0, 0, Bfield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][2], *Bfield_fp[0][2], 0, 0, Bfield_aux[0][2]->nComp(), ng_src); + } for (int lev = 1; lev <= finest_level; ++lev) { const amrex::Periodicity& crse_period = Geom(lev-1).periodicity(); @@ -308,8 +348,6 @@ WarpX::UpdateAuxilaryDataSameType () dBy.setVal(0.0); dBz.setVal(0.0); - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs @@ -379,8 +417,6 @@ WarpX::UpdateAuxilaryDataSameType () dEy.setVal(0.0); dEz.setVal(0.0); - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; // Copy Efield_aux to the dE MultiFabs, using up to ng_src (=ng_FieldGather) guard // cells from Efield_aux and filling up to ng (=nGrow) guard cells in the dE MultiFabs ablastr::utils::communication::ParallelCopy(dEx, *Efield_aux[lev - 1][0], 0, 0, diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 52586a5c88c..d33bd17ccdd 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -192,6 +192,12 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { RemakeMultiFab(Efield_fp_external[lev][idim], true); } + if (mypc->m_B_ext_particle_s == "read_from_file") { + RemakeMultiFab(B_external_particle_field[lev][idim], true); + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + RemakeMultiFab(E_external_particle_field[lev][idim], true); + } RemakeMultiFab(current_fp[lev][idim], false); RemakeMultiFab(current_store[lev][idim], false); if (current_deposition_algo == CurrentDepositionAlgo::Vay) { diff --git a/Source/Particles/Gather/GetExternalFields.cpp b/Source/Particles/Gather/GetExternalFields.cpp index 8e3c679aa3a..bb55f79f394 100644 --- a/Source/Particles/Gather/GetExternalFields.cpp +++ b/Source/Particles/Gather/GetExternalFields.cpp @@ -83,6 +83,18 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset m_repeated_plasma_lens_strengths_B = mypc.d_repeated_plasma_lens_strengths_B.data(); } + // When the external particle fields are read from file, + // the external fields are not added directly inside the gather kernel. + // (Hence of `None`, which ensures that the gather kernel is compiled without support + // for external fields.) Instead, the external fields are added to the MultiFab + // Efield_aux and Bfield_aux before the particles gather from these MultiFab. + if (mypc.m_E_ext_particle_s == "read_from_file") { + m_Etype = ExternalFieldInitType::None; + } + if (mypc.m_B_ext_particle_s == "read_from_file") { + m_Btype = ExternalFieldInitType::None; + } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_Etype != Unknown, "Unknown E_ext_particle_init_style"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_Btype != Unknown, "Unknown B_ext_particle_init_style"); diff --git a/Source/WarpX.H b/Source/WarpX.H index 768f2d486dc..3372dd869cc 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1512,6 +1512,8 @@ private: // Same as Bfield_fp/Efield_fp for reading external field data amrex::Vector, 3 > > Efield_fp_external; amrex::Vector, 3 > > Bfield_fp_external; + amrex::Vector, 3 > > E_external_particle_field; + amrex::Vector, 3 > > B_external_particle_field; //! EB: Lengths of the mesh edges amrex::Vector, 3 > > m_edge_lengths; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 5e3380cc1a6..4dfebe90c67 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -343,6 +343,8 @@ WarpX::WarpX () // Same as Bfield_fp/Efield_fp for reading external field data Bfield_fp_external.resize(1); Efield_fp_external.resize(1); + B_external_particle_field.resize(1); + E_external_particle_field.resize(1); m_edge_lengths.resize(nlevs_max); m_face_areas.resize(nlevs_max); @@ -2315,18 +2317,6 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(current_fp[lev][1], amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[y]", 0.0_rt); AllocInitMultiFab(current_fp[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[z]", 0.0_rt); - // Match external field MultiFabs to fine patch - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); - } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { - AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); - } - if (do_current_centering) { amrex::BoxArray const& nodal_ba = amrex::convert(ba, amrex::IntVect::TheNodeVector()); @@ -2558,23 +2548,35 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Efield_aux[lev][1], nba, dm, ncomps, ngEB, lev, "Efield_aux[y]", 0.0_rt); AllocInitMultiFab(Efield_aux[lev][2], nba, dm, ncomps, ngEB, lev, "Efield_aux[z]", 0.0_rt); } else if (lev == 0) { - if (!WarpX::fft_do_time_averaging) { - // In this case, the aux grid is simply an alias of the fp grid - AliasInitMultiFab(Efield_aux[lev][0], *Efield_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *Efield_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *Efield_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); - - AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); - } else { - AliasInitMultiFab(Efield_aux[lev][0], *Efield_avg_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *Efield_avg_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *Efield_avg_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); - + if (WarpX::fft_do_time_averaging) { AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_avg_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_avg_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_avg_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); + + AliasInitMultiFab(Efield_aux[lev][0], *Efield_avg_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][1], *Efield_avg_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][2], *Efield_avg_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + } else { + if (mypc->m_B_ext_particle_s == "read_from_file") { + AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]"); + AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]"); + AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]"); + } else { + // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) + AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); + AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); + AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]"); + AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]"); + AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); + } else { + // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) + AliasInitMultiFab(Efield_aux[lev][0], *Efield_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][1], *Efield_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][2], *Efield_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + } } } else { AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]"); @@ -2586,6 +2588,44 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); } + // The external fields that are read from file + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + // These fields will be added directly to the grid, i.e. to fp, and need to match the index type + AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bfield_fp[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); + AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, Bfield_fp[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); + AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bfield_fp[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); + } + if (mypc->m_B_ext_particle_s == "read_from_file") { + // These fields will be added to the fields that the particles see, and need to match the index type + AllocInitMultiFab(B_external_particle_field[lev][0], amrex::convert(ba, Bfield_aux[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "B_external_particle_field[x]", 0.0_rt); + AllocInitMultiFab(B_external_particle_field[lev][1], amrex::convert(ba, Bfield_aux[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "B_external_particle_field[y]", 0.0_rt); + AllocInitMultiFab(B_external_particle_field[lev][2], amrex::convert(ba, Bfield_aux[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "B_external_particle_field[z]", 0.0_rt); + } + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { + // These fields will be added directly to the grid, i.e. to fp, and need to match the index type + AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Efield_fp[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); + AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Efield_fp[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); + AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Efield_fp[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + // These fields will be added to the fields that the particles see, and need to match the index type + AllocInitMultiFab(E_external_particle_field[lev][0], amrex::convert(ba, Efield_aux[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "E_external_particle_field[x]", 0.0_rt); + AllocInitMultiFab(E_external_particle_field[lev][1], amrex::convert(ba, Efield_aux[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "E_external_particle_field[y]", 0.0_rt); + AllocInitMultiFab(E_external_particle_field[lev][2], amrex::convert(ba, Efield_aux[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "E_external_particle_field[z]", 0.0_rt); + } + // // The coarse patch // diff --git a/requirements.txt b/requirements.txt index 60f14e1282d..bb0d8ebb704 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ periodictable~=1.5 # PICMI # note: don't forget to update the version in Docs/requirements.txt, too -picmistandard==0.28.0 +picmistandard==0.29.0 # for development against an unreleased PICMI version, use: #picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python From bca3e9af7228a9d3c928fbf67d443759882d2436 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 10 Jul 2024 20:09:54 +0200 Subject: [PATCH 45/87] Reset collision checksums and add particle data (#5042) * Turn on particle comparison * New value for checksum --- Regression/Checksum/benchmarks_json/collisionXYZ.json | 4 ++-- Regression/WarpX-tests.ini | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 7436302a99e..417e8a0b144 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,7 +6,7 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 351570.64632548566, - "T_ion": 350085.3998917431 + "T_electron": 353384.4616461907, + "T_ion": 348277.2774202612 } } diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 3d1dbfe3b0f..fdb4d14a75f 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -256,7 +256,7 @@ useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 -compareParticles = 0 +compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_rz.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -274,7 +274,7 @@ useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 -compareParticles = 0 +compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_3d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -292,7 +292,7 @@ useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 -compareParticles = 0 +compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_2d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py From 45278d772468779bad0b05c9b515a2c7f3482d9d Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:03:58 -0700 Subject: [PATCH 46/87] CI: fix dimensionality of test `Python_ohms_law_solver_EM_modes_rz` (#5037) --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index fdb4d14a75f..82e628a2bd4 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3571,7 +3571,7 @@ buildDir = . inputFile = Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py runtime_params = warpx.abort_on_warning_threshold = medium customRunCmd = python3 PICMI_inputs_rz.py --test -dim = 1 +dim = 2 addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_APP=OFF -DWarpX_QED=OFF -DWarpX_PYTHON=ON target = pip_install From a0ce802dd7fc323477800aa55af63f390dcf5f32 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 10 Jul 2024 23:10:01 +0200 Subject: [PATCH 47/87] Specify the unit of the potential, in the documentation (#5035) --- Docs/source/usage/parameters.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index d6420440935..172a9cc7cb8 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -500,7 +500,7 @@ Domain Boundary Conditions * ``open``: For the electrostatic Poisson solver based on a Integrated Green Function method. * ``boundary.potential_lo_x/y/z`` and ``boundary.potential_hi_x/y/z`` (default `0`) - Gives the value of the electric potential at the boundaries, for ``pec`` boundaries. With electrostatic solvers + Gives the value of the electric potential, in Volts, at the boundaries, for ``pec`` boundaries. With electrostatic solvers (i.e., with ``warpx.do_electrostatic = ...``), this is used in order to compute the potential in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver), setting these variables will trigger an electrostatic solve at ``t=0``, to compute the initial @@ -603,7 +603,7 @@ Whether the embedded boundary is defined with an analytical function or an STL f additionally define the electric potential at the embedded boundary with an analytical function: * ``warpx.eb_potential(x,y,z,t)`` (`string`) - Gives the value of the electric potential at the surface of the embedded boundary, + Gives the value of the electric potential, in Volts, at the surface of the embedded boundary, as a function of `x`, `y`, `z` and `t`. With electrostatic solvers (i.e., with ``warpx.do_electrostatic = ...``), this is used in order to compute the potential in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver), From 5f6715148bd46f160851a36aa7c1f9d293799590 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 11 Jul 2024 05:23:03 +0200 Subject: [PATCH 48/87] Docs: correct formatting of `gaussian_parse_momentum_function` (#5038) --- Docs/source/usage/parameters.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 172a9cc7cb8..4a8c4be0b5f 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1156,8 +1156,8 @@ Particle initialization * ``gaussian_parse_momentum_function``: Gaussian momentum distribution where the mean and the standard deviation are given by functions of position in the input file. Both are assumed to be non-relativistic. The mean is the normalized momentum, :math:`u_m = \gamma v_m/c`. - The standard deviation is normalized, :math:`u_th = v_th/c`. - For example, this might be `u_th = sqrt(T*q_e/mass)/clight` given the temperature (in eV) and mass. + The standard deviation is normalized, :math:`u_{th} = v_{th}/c`. + For example, this might be ``u_th = sqrt(T*q_e/mass)/clight`` given the temperature (in eV) and mass. It requires the following arguments: * ``.momentum_function_ux_m(x,y,z)``: mean :math:`u_{x}` From ade19e5f10451f86c8a5f6817ffa6b56194e701b Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 10 Jul 2024 21:10:03 -0700 Subject: [PATCH 49/87] Remove work-around in regression testing script (#5034) --- run_test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/run_test.sh b/run_test.sh index 37264bef243..2ca880833b0 100755 --- a/run_test.sh +++ b/run_test.sh @@ -80,8 +80,6 @@ cd - # Clone the AMReX regression test utility git clone https://github.com/AMReX-Codes/regression_testing.git -# FIXME: https://github.com/AMReX-Codes/regression_testing/issues/136 -cd regression_testing && git checkout 93ddfb11456f47d6555c39388ba1a4ead61fbf4e && cd - # Prepare regression tests mkdir -p rt-WarpX/WarpX-benchmarks From 892f65b30c448e71737acd9f970e7b1f59d7d596 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 10 Jul 2024 21:11:34 -0700 Subject: [PATCH 50/87] CI: restore input for test `Langmuir_multi_nodal` (#5039) --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 82e628a2bd4..a4cf87a53dd 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -1509,7 +1509,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_nodal] buildDir = . -inputFile = +inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = warpx.grid_type=collocated algo.current_deposition=direct dim = 3 addToCompileString = From f5ae2ec3267c07b15a5611b4c4cd08aac88d7c7d Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Thu, 11 Jul 2024 05:45:46 -0700 Subject: [PATCH 51/87] fixed ComputeTemperature routine called from Coulomb collisions to work for particles of varying weight. (#5040) --- .../BinaryCollision/Coulomb/ComputeTemperature.H | 15 +++++++++------ .../Coulomb/ElasticCollisionPerez.H | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H index 04440288ba3..6d0b48615a4 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H @@ -14,6 +14,7 @@ template AMREX_GPU_HOST_DEVICE T_R ComputeTemperature ( T_index const Is, T_index const Ie, T_index const * AMREX_RESTRICT I, + T_R const * AMREX_RESTRICT w, T_R const * AMREX_RESTRICT ux, T_R const * AMREX_RESTRICT uy, T_R const * AMREX_RESTRICT uz, T_R const m ) { @@ -26,6 +27,7 @@ T_R ComputeTemperature ( T_R vx = T_R(0.0); T_R vy = T_R(0.0); T_R vz = T_R(0.0); T_R vs = T_R(0.0); T_R gm = T_R(0.0); T_R us = T_R(0.0); + T_R wtot = T_R(0.0); for (int i = Is; i < static_cast(Ie); ++i) { @@ -33,14 +35,15 @@ T_R ComputeTemperature ( uy[ I[i] ] * uy[ I[i] ] + uz[ I[i] ] * uz[ I[i] ] ); gm = std::sqrt( T_R(1.0) + us*inv_c2 ); - vx += ux[ I[i] ] / gm; - vy += uy[ I[i] ] / gm; - vz += uz[ I[i] ] / gm; - vs += us / gm / gm; + wtot += w[ I[i] ]; + vx += w[ I[i] ] * ux[ I[i] ] / gm; + vy += w[ I[i] ] * uy[ I[i] ] / gm; + vz += w[ I[i] ] * uz[ I[i] ] / gm; + vs += w[ I[i] ] * us / gm / gm; } - vx = vx / N; vy = vy / N; - vz = vz / N; vs = vs / N; + vx = vx / wtot; vy = vy / wtot; + vz = vz / wtot; vs = vs / wtot; return m/T_R(3.0)*(vs-(vx*vx+vy*vy+vz*vz)); } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index af4e6ba38c9..a782fac5f6e 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -74,12 +74,12 @@ void ElasticCollisionPerez ( T_PR T1t; T_PR T2t; if ( T1 <= T_PR(0.0) && L <= T_PR(0.0) ) { - T1t = ComputeTemperature(I1s,I1e,I1,u1x,u1y,u1z,m1); + T1t = ComputeTemperature(I1s,I1e,I1,w1,u1x,u1y,u1z,m1); } else { T1t = T1; } if ( T2 <= T_PR(0.0) && L <= T_PR(0.0) ) { - T2t = ComputeTemperature(I2s,I2e,I2,u2x,u2y,u2z,m2); + T2t = ComputeTemperature(I2s,I2e,I2,w2,u2x,u2y,u2z,m2); } else { T2t = T2; } From a305fc6e5c031f2e1285f25300c73a86d96a6404 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 11 Jul 2024 17:36:38 +0200 Subject: [PATCH 52/87] Use latest PICMI version in documentation (#5044) --- Docs/requirements.txt | 2 +- Python/setup.py | 2 +- Tools/machines/karolina-it4i/install_dependencies.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Docs/requirements.txt b/Docs/requirements.txt index b16bf161144..1ce1ee6c1a0 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -13,7 +13,7 @@ openpmd-viewer # for checksumAPI # PICMI API docs # note: keep in sync with version in ../requirements.txt -picmistandard==0.28.0 +picmistandard==0.29.0 # for development against an unreleased PICMI version, use: # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python diff --git a/Python/setup.py b/Python/setup.py index e4adbb6c458..31f35eeceac 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -59,7 +59,7 @@ package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", package_data = package_data, - install_requires = ['numpy', 'picmistandard==0.28.0', 'periodictable'], + install_requires = ['numpy', 'picmistandard==0.29.0', 'periodictable'], python_requires = '>=3.8', zip_safe=False ) diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh index 0435b5e2926..cba455f3d29 100755 --- a/Tools/machines/karolina-it4i/install_dependencies.sh +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -53,7 +53,7 @@ python -m pip install --user --upgrade matplotlib #python -m pip install --user --upgrade yt # install or update WarpX dependencies -python -m pip install --user --upgrade picmistandard==0.28.0 +python -m pip install --user --upgrade picmistandard==0.29.0 python -m pip install --user --upgrade lasy # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) From 63bedffff42a11b073cca3d708f4910dff60a997 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:28:03 -0700 Subject: [PATCH 53/87] CI: remove unused params, check particles in `collisionXYZ` (#5043) * CI: remove unused params, check particles in `collisionXYZ` * Update benchmark --------- Co-authored-by: Remi Lehe --- .../Tests/collision/analysis_collision_3d.py | 2 +- .../benchmarks_json/collisionXYZ.json | 20 +- Regression/WarpX-tests.ini | 866 ------------------ run_test.sh | 4 +- 4 files changed, 22 insertions(+), 870 deletions(-) diff --git a/Examples/Tests/collision/analysis_collision_3d.py b/Examples/Tests/collision/analysis_collision_3d.py index ee0c81d8f75..86a434caab2 100755 --- a/Examples/Tests/collision/analysis_collision_3d.py +++ b/Examples/Tests/collision/analysis_collision_3d.py @@ -111,4 +111,4 @@ dim, species_name) test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 417e8a0b144..5ec1f21c32a 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -8,5 +8,23 @@ "Ez": 0.0, "T_electron": 353384.4616461907, "T_ion": 348277.2774202612 + }, + "electron": { + "particle_momentum_x": 8.378820415249081e-19, + "particle_momentum_y": 8.238482720534644e-19, + "particle_momentum_z": 8.189221038447925e-19, + "particle_position_x": 21268719.554914076, + "particle_position_y": 21282437.437205918, + "particle_position_z": 21231947.47401523, + "particle_weight": 7.168263344048695e+28 + }, + "ion": { + "particle_momentum_x": 2.013855005436386e-18, + "particle_momentum_y": 1.8231305152698165e-18, + "particle_momentum_z": 1.8194149390229157e-18, + "particle_position_x": 21217737.157987915, + "particle_position_y": 21263096.30516862, + "particle_position_z": 21261343.069983635, + "particle_weight": 7.168263344048695e+28 } -} +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index a4cf87a53dd..6ee9820aa28 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -81,10 +81,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [averaged_galilean_2d_psatd_hybrid] @@ -99,10 +95,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [averaged_galilean_3d_psatd] @@ -117,10 +109,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [averaged_galilean_3d_psatd_hybrid] @@ -135,10 +123,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [background_mcc] @@ -153,10 +137,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons he_ions analysisRoutine = Examples/analysis_default_regression.py [background_mcc_dp_psp] @@ -171,10 +151,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons he_ions analysisRoutine = Examples/analysis_default_regression.py [bilinear_filter] @@ -189,8 +165,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/single_particle/analysis_bilinear_filter.py [BTD_rz] @@ -205,8 +179,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/btd_rz/analysis_BTD_laser_antenna.py [collider_diagnostics] @@ -221,8 +193,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py [collisionISO] @@ -237,8 +207,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/collision/analysis_collision_3d_isotropization.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -254,9 +222,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_rz.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -272,9 +237,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_3d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -290,9 +252,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_2d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -308,10 +267,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions beam analysisRoutine = Examples/analysis_default_regression.py [Deuterium_Deuterium_Fusion_3D] @@ -326,8 +281,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [Deuterium_Deuterium_Fusion_3D_intraspecies] @@ -342,8 +295,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py [Deuterium_Tritium_Fusion_3D] @@ -358,8 +309,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [Deuterium_Tritium_Fusion_RZ] @@ -374,8 +323,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [dirichletbc] @@ -390,9 +337,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py [divb_cleaning_3d] @@ -407,9 +351,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/divb_cleaning/analysis.py [dive_cleaning_2d] @@ -423,9 +364,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/dive_cleaning/analysis.py analysisOutputImage = Comparison.png @@ -441,9 +379,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/dive_cleaning/analysis.py analysisOutputImage = Comparison.png @@ -460,9 +395,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereLabFrame_MR_emass_10] @@ -477,9 +409,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereEB] @@ -494,9 +423,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis.py [ElectrostaticSphereEB_mixedBCs] @@ -511,9 +437,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [EmbeddedBoundaryDiffraction] @@ -528,9 +451,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 outputFile = EmbeddedBoundaryDiffraction_plt analysisRoutine = Examples/Tests/embedded_boundary_diffraction/analysis_fields.py @@ -546,9 +466,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereEB_RZ_MR] @@ -563,9 +480,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereLabFrame] @@ -580,9 +494,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereRZ] @@ -597,9 +508,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereRelNodal] @@ -614,9 +522,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [embedded_boundary_cube] @@ -631,9 +536,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields.py [embedded_boundary_cube_2d] @@ -648,9 +550,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py [embedded_boundary_cube_macroscopic] @@ -665,9 +564,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields.py [embedded_boundary_python_API] @@ -684,9 +580,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_python_api/analysis.py [embedded_boundary_rotated_cube] @@ -701,9 +594,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py [embedded_boundary_rotated_cube_2d] @@ -718,9 +608,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py [embedded_circle] @@ -735,10 +622,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ar_ions analysisRoutine = Examples/Tests/embedded_circle/analysis.py [FieldProbe] @@ -753,9 +636,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/field_probe/analysis_field_probe.py [FluxInjection] @@ -770,10 +650,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/flux_injection/analysis_flux_injection_rz.py [FluxInjection3D] @@ -788,9 +664,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/flux_injection/analysis_flux_injection_3d.py [galilean_2d_psatd] @@ -805,10 +678,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_2d_psatd_current_correction] @@ -823,10 +692,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_2d_psatd_current_correction_psb] @@ -841,10 +706,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_2d_psatd_hybrid] @@ -859,10 +720,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions beam analysisRoutine = Examples/analysis_default_regression.py [galilean_3d_psatd] @@ -877,10 +734,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_3d_psatd_current_correction] @@ -895,10 +748,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_3d_psatd_current_correction_psb] @@ -913,10 +762,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_rz_psatd] @@ -931,10 +776,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_rz_psatd_current_correction_psb] @@ -949,10 +790,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_rz_psatd_current_correction] @@ -967,10 +804,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [hard_edged_plasma_lens] @@ -985,10 +818,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [hard_edged_quadrupoles] @@ -1003,10 +832,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py [hard_edged_quadrupoles_boosted] @@ -1021,10 +846,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py [hard_edged_quadrupoles_moving] @@ -1039,10 +860,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py [initial_distribution] @@ -1057,9 +874,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/initial_distribution/analysis_distribution.py aux1File = Tools/PostProcessing/read_raw_data.py @@ -1075,8 +889,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/ionization/analysis_ionization.py [ionization_lab] @@ -1091,8 +903,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/ionization/analysis_ionization.py [ion_stopping] @@ -1107,9 +917,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ion_stopping/analysis_ion_stopping.py [Langmuir_multi] @@ -1124,10 +931,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1143,9 +946,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_1d.py analysisOutputImage = langmuir_fluid_multi_1d_analysis.png @@ -1161,9 +961,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_rz.py analysisOutputImage = langmuir_fluid_rz_analysis.png @@ -1179,9 +976,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_2d.py analysisOutputImage = langmuir_fluid_multi_2d_analysis.png @@ -1197,9 +991,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_3d.py analysisOutputImage = langmuir_fluid_multi_analysis.png @@ -1215,10 +1006,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_1d.py analysisOutputImage = langmuir_multi_1d_analysis.png @@ -1234,10 +1021,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_MR.png @@ -1253,10 +1036,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_MR.png @@ -1272,10 +1051,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_MR_momentum_conserving.png @@ -1291,10 +1066,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_MR_psatd.png @@ -1310,10 +1081,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1329,10 +1096,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1348,10 +1111,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1367,10 +1126,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1386,10 +1141,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1405,10 +1156,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_psatd_multiJ.png @@ -1424,10 +1171,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_psatd_multiJ_nodal.png @@ -1443,10 +1186,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1462,10 +1201,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1481,10 +1216,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1500,10 +1231,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1519,10 +1246,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1538,10 +1261,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1557,10 +1276,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1576,10 +1291,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1595,10 +1306,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1614,10 +1321,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1633,10 +1336,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = Langmuir_multi_psatd_multiJ.png @@ -1652,10 +1351,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = Langmuir_multi_psatd_multiJ_nodal.png @@ -1671,10 +1366,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1690,10 +1381,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1709,10 +1396,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1728,10 +1411,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1747,10 +1426,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py analysisOutputImage = Langmuir_multi_rz_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -1767,10 +1442,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py analysisOutputImage = Langmuir_multi_rz_psatd_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -1787,10 +1458,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py analysisOutputImage = Langmuir_multi_rz_psatd_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -1807,10 +1474,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py analysisOutputImage = Langmuir_multi_rz_psatd_multiJ_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -1827,10 +1490,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1846,8 +1505,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [LaserAcceleration] @@ -1862,10 +1519,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons outputFile = LaserAcceleration_plt analysisRoutine = Examples/analysis_default_openpmd_regression.py @@ -1881,10 +1534,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [LaserAcceleration_1d_fluid] @@ -1899,10 +1548,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py [LaserAcceleration_1d_fluid_boosted] @@ -1917,10 +1562,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py [LaserAccelerationBoost] @@ -1935,8 +1576,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [LaserAcceleration_BTD] @@ -1951,10 +1590,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/boosted_diags/analysis.py [LaserAccelerationMR] @@ -1969,10 +1604,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [LaserAccelerationRZ] @@ -1987,10 +1618,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [LaserAccelerationRZ_opmd] @@ -2005,10 +1632,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons beam outputFile = LaserAccelerationRZ_opmd_plt analysisRoutine = Examples/Tests/openpmd_rz/analysis_openpmd_rz.py @@ -2024,10 +1647,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons outputFile = LaserAcceleration_single_precision_comms_plt analysisRoutine = Examples/analysis_default_openpmd_regression.py @@ -2043,9 +1662,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/laser_injection/analysis_laser.py analysisOutputImage = laser_analysis.png @@ -2061,9 +1677,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/laser_injection/analysis_1d.py [LaserInjection_2d] @@ -2078,9 +1691,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/laser_injection/analysis_2d.py [LaserInjectionFromBINARYFile] @@ -2097,10 +1707,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile] buildDir = . @@ -2116,10 +1724,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_1d] buildDir = . @@ -2135,10 +1741,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_1d_boost] buildDir = . @@ -2154,10 +1758,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_2d] buildDir = . @@ -2173,10 +1775,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_RZ] buildDir = . @@ -2192,10 +1792,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromRZLASYFile] buildDir = . @@ -2211,10 +1809,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserIonAcc2d] buildDir = . @@ -2229,8 +1825,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserOnFine] @@ -2245,8 +1839,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [leveling_thinning] @@ -2261,9 +1853,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/resampling/analysis_leveling_thinning.py [LoadExternalFieldRZGrid] @@ -2279,10 +1868,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = proton analysisRoutine = Examples/Tests/LoadExternalField/analysis_rz.py [LoadExternalFieldRZParticles] @@ -2298,10 +1883,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = proton analysisRoutine = Examples/Tests/LoadExternalField/analysis_rz.py [magnetostatic_eb_3d] @@ -2316,9 +1897,6 @@ useMPI = 1 numprocs = 1 useOMP = 2 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [Maxwell_Hybrid_QED_solver] @@ -2333,8 +1911,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/maxwell_hybrid_qed/analysis_Maxwell_QED_Hybrid.py [momentum-conserving-gather] @@ -2349,10 +1925,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam driver plasma_e analysisRoutine = Examples/analysis_default_regression.py [multi_J_rz_psatd] @@ -2367,10 +1939,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = driver plasma_e plasma_p analysisRoutine = Examples/analysis_default_regression.py [nci_corrector] @@ -2385,9 +1953,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -doComparison = 0 analysisRoutine = Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py [nci_correctorMR] @@ -2402,9 +1967,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -doComparison = 0 analysisRoutine = Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py [parabolic_channel_initialization_2d_single_precision] @@ -2418,9 +1980,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/initial_plasma_profile/analysis.py @@ -2436,10 +1995,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/particle_boundary_process/analysis_absorption.py [particle_boundaries_3d] @@ -2454,9 +2009,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/boundaries/analysis.py [particle_fields_diags] @@ -2472,9 +2024,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_fields_diags/analysis_particle_diags.py [particle_fields_diags_single_precision] @@ -2490,9 +2039,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_fields_diags/analysis_particle_diags_single.py [particle_pusher] @@ -2507,9 +2053,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_pusher/analysis_pusher.py [particle_scrape] @@ -2524,10 +2067,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/particle_boundary_scrape/analysis_scrape.py [particles_in_pml] @@ -2542,9 +2081,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [particles_in_pml_2d] @@ -2559,9 +2095,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [particles_in_pml_2d_MR] @@ -2576,9 +2109,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [particles_in_pml_3d_MR] @@ -2593,9 +2123,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [PEC_field] @@ -2610,9 +2137,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/pec/analysis_pec.py [PEC_field_mr] @@ -2627,9 +2151,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/pec/analysis_pec_mr.py [PEC_particle] @@ -2644,10 +2165,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron proton analysisRoutine = Examples/analysis_default_regression.py [Performance_works_1_uniform_rest_32ppc] @@ -2662,10 +2179,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_2_uniform_rest_1ppc] @@ -2680,10 +2193,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Performance_works_3_uniform_drift_4ppc] @@ -2698,10 +2207,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_4_labdiags_2ppc] @@ -2716,10 +2221,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_5_loadimbalance] @@ -2734,10 +2235,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_6_output_2ppc] @@ -2752,10 +2249,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [photon_pusher] @@ -2770,9 +2263,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/photon_pusher/analysis_photon_pusher.py [PlasmaAccelerationBoost2d] @@ -2787,8 +2277,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationBoost3d] @@ -2803,8 +2291,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationBoost3d_hybrid] @@ -2818,8 +2304,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationMR] @@ -2834,10 +2318,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam driver plasma_e analysisRoutine = Examples/analysis_default_regression.py [Plasma_lens] @@ -2852,10 +2332,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [Plasma_lens_boosted] @@ -2870,10 +2346,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [Plasma_lens_short] @@ -2888,10 +2360,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [PlasmaMirror] @@ -2906,8 +2374,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [pml_psatd_dive_divb_cleaning] @@ -2922,8 +2388,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [pml_psatd_rz] @@ -2938,8 +2402,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_psatd_rz.py [pml_x_ckc] @@ -2954,8 +2416,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_ckc.py [pml_x_galilean] @@ -2970,8 +2430,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py [pml_x_psatd] @@ -2987,8 +2445,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py [pml_x_yee] @@ -3004,8 +2460,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py [pml_x_yee_eb] @@ -3021,8 +2475,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py [Proton_Boron_Fusion_2D] @@ -3037,8 +2489,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py [Proton_Boron_Fusion_3D] @@ -3053,8 +2503,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py [Python_background_mcc] @@ -3071,9 +2519,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_2d.py [Python_background_mcc_1d] @@ -3090,9 +2535,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py [Python_background_mcc_1d_tridiag] @@ -3109,9 +2551,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py [Python_collisionXZ] @@ -3128,9 +2567,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/collision/analysis_collision_2d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -3148,9 +2584,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py [Python_dsmc_1d] @@ -3167,9 +2600,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py [Python_ElectrostaticSphereEB] @@ -3186,9 +2616,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis.py [Python_gaussian_beam] @@ -3205,10 +2632,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_gaussian_beam_no_field_output] @@ -3225,10 +2648,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Python_gaussian_beam_opmd] @@ -3245,10 +2664,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Python_gaussian_beam_opmd_no_field_output] @@ -3265,10 +2680,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Python_id_cpu_read] @@ -3285,10 +2696,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons [Python_ionization] buildDir = . @@ -3304,8 +2711,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/ionization/analysis_ionization.py [Python_Langmuir] @@ -3322,10 +2727,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_Langmuir_2d] @@ -3342,10 +2743,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_Langmuir_rz_multimode] @@ -3362,10 +2759,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons protons analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAcceleration] @@ -3382,10 +2775,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAcceleration_1d] @@ -3402,10 +2791,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAccelerationMR] @@ -3422,10 +2807,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAccelerationRZ] @@ -3442,10 +2823,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [Python_LaserIonAcc2d] @@ -3463,8 +2840,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_openpmd_regression.py [Python_LoadExternalGridField3D] @@ -3481,10 +2856,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = proton analysisRoutine = Examples/Tests/LoadExternalField/analysis_3d.py [Python_LoadExternalParticleField3D] @@ -3501,10 +2872,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = proton analysisRoutine = Examples/Tests/LoadExternalField/analysis_3d.py [Python_magnetostatic_eb_3d] @@ -3521,10 +2888,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 2 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_magnetostatic_eb_rz] @@ -3541,10 +2904,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 2 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons analysisRoutine = Examples/Tests/magnetostatic_eb/analysis_rz.py [Python_ohms_law_solver_EM_modes_1d] @@ -3561,9 +2920,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_EM_modes/analysis.py [Python_ohms_law_solver_EM_modes_rz] @@ -3580,9 +2936,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_EM_modes/analysis_rz.py [Python_ohms_law_solver_ion_beam_1d] @@ -3599,9 +2952,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_ion_beam_instability/analysis.py [Python_ohms_law_solver_landau_damping_2d] @@ -3618,9 +2968,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py [Python_ohms_law_solver_magnetic_reconnection_2d] @@ -3637,9 +2984,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py [Python_particle_attr_access] @@ -3656,8 +3000,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/particle_data_python/analysis.py [Python_particle_attr_access_unique] @@ -3674,8 +3016,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/particle_data_python/analysis.py [Python_particle_reflection] @@ -3692,8 +3032,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/particle_boundary_process/analysis_reflection.py [Python_particle_scrape] @@ -3710,10 +3048,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/particle_boundary_scrape/analysis_scrape.py # TODO: Enable in pyAMReX, then enable lines in PICMI_inputs_2d.py again @@ -3732,10 +3066,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons # TODO: comment in again once enabled #analysisRoutine = Examples/Tests/pass_mpi_communicator/analysis.py @@ -3753,10 +3083,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam analysisRoutine = Examples/analysis_default_regression.py [Python_PlasmaAcceleration1d] @@ -3773,10 +3099,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam analysisRoutine = Examples/analysis_default_regression.py [Python_PlasmaAccelerationMR] @@ -3793,10 +3115,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam analysisRoutine = Examples/analysis_default_regression.py [Python_plasma_lens] @@ -3813,10 +3131,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [Python_prev_positions] @@ -3833,9 +3147,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/analysis_default_regression.py [Python_reduced_diags_loadbalancecosts_timers] @@ -3852,9 +3163,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [Python_restart_eb] @@ -3872,10 +3180,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/restart/analysis_restart.py [Python_restart_runtime_components] @@ -3893,10 +3197,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons [Python_wrappers] buildDir = . @@ -3912,9 +3212,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [qed_breit_wheeler_2d] @@ -3930,9 +3227,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_yt.py [qed_breit_wheeler_2d_opmd] @@ -3948,9 +3242,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 outputFile = qed_breit_wheeler_2d_opmd_plt analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_opmd.py @@ -3967,9 +3258,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_yt.py [qed_breit_wheeler_3d_opmd] @@ -3985,9 +3273,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 outputFile = qed_breit_wheeler_3d_opmd_plt analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_opmd.py @@ -4003,9 +3288,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/quantum_synchrotron/analysis.py [qed_quantum_sync_3d] @@ -4020,9 +3302,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/quantum_synchrotron/analysis.py [qed_schwinger1] @@ -4037,8 +3316,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [qed_schwinger2] @@ -4053,8 +3330,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [qed_schwinger3] @@ -4069,8 +3344,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [qed_schwinger4] @@ -4085,8 +3358,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [radiation_reaction] @@ -4101,9 +3372,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py [reduced_diags] @@ -4119,9 +3387,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags.py [reduced_diags_loadbalancecosts_heuristic] @@ -4136,9 +3401,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [reduced_diags_loadbalancecosts_timers] @@ -4153,9 +3415,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [reduced_diags_loadbalancecosts_timers_psatd] @@ -4170,9 +3429,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [reduced_diags_single_precision] @@ -4188,9 +3444,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_single.py [RefinedInjection] @@ -4205,10 +3458,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py [relativistic_space_charge_initialization] @@ -4222,9 +3471,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/relativistic_space_charge_initialization/analysis.py analysisOutputImage = Comparison.png @@ -4241,8 +3487,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/repelling_particles/analysis_repelling.py [resample_velocity_coincidence_thinning] @@ -4257,9 +3501,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [resample_velocity_coincidence_thinning_cartesian] @@ -4274,9 +3515,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [restart] @@ -4292,10 +3530,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = beam analysisRoutine = Examples/Tests/restart/analysis_restart.py [restart_psatd] @@ -4311,10 +3545,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = beam analysisRoutine = Examples/Tests/restart/analysis_restart.py [restart_psatd_time_avg] @@ -4330,10 +3560,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = beam analysisRoutine = Examples/Tests/restart/analysis_restart.py [RigidInjection_BTD] @@ -4348,10 +3574,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py [RigidInjection_lab] @@ -4366,9 +3588,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/rigid_injection/analysis_rigid_injection_LabFrame.py [scraping] @@ -4383,9 +3602,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/scraping/analysis_rz.py [scraping_filter] @@ -4400,9 +3616,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/scraping/analysis_rz_filter.py [silver_mueller_1d] @@ -4417,8 +3630,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [silver_mueller_2d_x] @@ -4433,8 +3644,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [silver_mueller_2d_z] @@ -4449,8 +3658,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [silver_mueller_rz_z] @@ -4465,8 +3672,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [space_charge_initialization] @@ -4480,9 +3685,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py analysisOutputImage = Comparison.png @@ -4498,9 +3700,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py analysisOutputImage = Comparison.png @@ -4517,9 +3716,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [Uniform_2d] @@ -4534,8 +3730,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [uniform_plasma_restart] @@ -4551,10 +3745,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons analysisRoutine = Examples/Tests/restart/analysis_restart.py [uniform_plasma_multiJ] @@ -4569,10 +3759,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_multiJ.py [VayDeposition2D] @@ -4587,10 +3773,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron ion analysisRoutine = Examples/Tests/vay_deposition/analysis.py [VayDeposition3D] @@ -4605,10 +3787,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron ion analysisRoutine = Examples/Tests/vay_deposition/analysis.py [NodalElectrostaticSolver] @@ -4623,9 +3801,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/nodal_electrostatic/analysis_3d.py [BeamBeamCollision] @@ -4640,8 +3815,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 outputFile = BeamBeamCollision_plt analysisRoutine = Examples/analysis_default_openpmd_regression.py @@ -4659,10 +3832,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons protons outputFile = spacecraft_charging_plt analysisRoutine = Examples/Physics_applications/spacecraft_charging/analysis.py analysisOutputImage = min_phi_analysis.png @@ -4679,10 +3848,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons outputFile = Point_of_contact_EB_3d_plt analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py @@ -4698,10 +3863,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons outputFile = Point_of_contact_EB_rz_plt analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py @@ -4717,9 +3878,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py [ThetaImplicitJFNK_VandB_2d] @@ -4734,9 +3892,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py [SemiImplicitPicard_1d] @@ -4751,9 +3906,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py [EnergyConservingThermalPlasma] @@ -4767,9 +3919,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/energy_conserving_thermal_plasma/analysis.py [focusing_gaussian_beam] @@ -4784,10 +3933,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/gaussian_beam/analysis_focusing_beam.py [particle_boundary_interaction] @@ -4804,10 +3949,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons outputFile = particle_boundary_interaction_plt analysisRoutine = Examples/Tests/particle_boundary_interaction/analysis.py @@ -4823,9 +3964,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py [openbc_poisson_solver] @@ -4840,8 +3978,4 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py diff --git a/run_test.sh b/run_test.sh index 2ca880833b0..21e84c7030d 100755 --- a/run_test.sh +++ b/run_test.sh @@ -94,10 +94,10 @@ cd ../../regression_testing/ echo "cd $PWD" # run only tests specified in variable tests_arg (single test or multiple tests) if [[ ! -z "${tests_arg}" ]]; then - python3 regtest.py ../rt-WarpX/ci-tests.ini --no_update all "${tests_run}" + python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all "${tests_run}" # run all tests (variables tests_arg and tests_run are empty) else - python3 regtest.py ../rt-WarpX/ci-tests.ini --no_update all + python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all fi # clean up python virtual environment From dc087bf18b1e3d2367acdf86ea44ee3fc41d1755 Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Thu, 11 Jul 2024 21:52:40 -0700 Subject: [PATCH 54/87] shuffling the full list of particles for intra-species binary collisions (#5045) * shuffling the full list of particles for intra-species binary collisions * fixing merge issue with collision json files. * moved checksum to before error assert for intraspecies DD fusion regression test analysis script. * updating benchmark values for DD_fusion_3D_intraspecies. * adding fixed random seed to DD_3D_intraspecies fusion regression test. * putting checksum back after assert. * adjusting json. * updating json. --- ...sis_deuterium_deuterium_3d_intraspecies.py | 4 +- ...inputs_deuterium_deuterium_3d_intraspecies | 2 + ...rium_Deuterium_Fusion_3D_intraspecies.json | 48 +++++++++---------- .../benchmarks_json/collisionISO.json | 12 ++--- .../benchmarks_json/collisionXYZ.json | 30 ++++++------ .../BinaryCollision/BinaryCollision.H | 4 +- 6 files changed, 51 insertions(+), 49 deletions(-) diff --git a/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py b/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py index 70178fe97a6..858df0b26a6 100755 --- a/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py +++ b/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py @@ -47,7 +47,9 @@ sigma_th = 2.603e-18 error = np.abs(sigma-sigma_th)/sigma_th -tolerance = 1e-2 +tolerance = 2e-2 +print('error = ', error) +print('tolerance = ', tolerance) assert error < tolerance # Compare checksums with benchmark diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies b/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies index 985cc2d7612..0030ca9b945 100644 --- a/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies +++ b/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies @@ -8,6 +8,8 @@ # plasma simulations. Journal of Computational Physics, 388, pp.439-453. # DOI: https://doi.org/10.1016/j.jcp.2019.03.020 +warpx.random_seed = 1034958209 + # algo algo.particle_shape = 3 diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json index 59818f62826..45e29874cda 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json @@ -1,34 +1,34 @@ { "lev=0": { "rho": 0.0, - "rho_deuterium": 8203144355.71195, - "rho_helium3": 10.368009592276463 + "rho_deuterium": 8203144355.767546, + "rho_helium3": 10.312416024595505 }, "neutron": { - "particle_momentum_x": 2.2543499835759282e-15, - "particle_momentum_y": 2.2526527390783875e-15, - "particle_momentum_z": 2.2619641737859965e-15, - "particle_position_x": 61.961041864660686, - "particle_position_y": 61.78141653674165, - "particle_position_z": 61.741022731492514, - "particle_weight": 505562702.7678892 + "particle_momentum_x": 2.260586487910896e-15, + "particle_momentum_y": 2.2564723094408887e-15, + "particle_momentum_z": 2.263164930227645e-15, + "particle_position_x": 62.03084683414219, + "particle_position_y": 61.95011296352698, + "particle_position_z": 62.02404756675538, + "particle_weight": 502851860.91505826 }, "deuterium": { - "particle_momentum_x": 1.3370046499332103e-14, - "particle_momentum_y": 1.3364310231320824e-14, - "particle_momentum_z": 1.3372728873714894e-14, - "particle_position_x": 2560.1613417364665, - "particle_position_y": 2560.082464065988, - "particle_position_z": 2560.0018477161034, - "particle_weight": 7.999999989888742e+17 + "particle_momentum_x": 1.3380470537895298e-14, + "particle_momentum_y": 1.3367685546044215e-14, + "particle_momentum_z": 1.3372922616599391e-14, + "particle_position_x": 2559.782513518522, + "particle_position_y": 2559.8747993471684, + "particle_position_z": 2560.2864832238383, + "particle_weight": 7.999999989942968e+17 }, "helium3": { - "particle_momentum_x": 2.2749239620327265e-15, - "particle_momentum_y": 2.268697031603961e-15, - "particle_momentum_z": 2.278045756364995e-15, - "particle_position_x": 61.961041864660686, - "particle_position_y": 61.78141653674165, - "particle_position_z": 61.741022731492514, - "particle_weight": 505562702.7678892 + "particle_momentum_x": 2.278275200450756e-15, + "particle_momentum_y": 2.2738610200497133e-15, + "particle_momentum_z": 2.2792408973056887e-15, + "particle_position_x": 62.03084683414219, + "particle_position_y": 61.95011296352698, + "particle_position_z": 62.02404756675538, + "particle_weight": 502851860.91505826 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json index 5146837be6f..a2fd6116cb8 100644 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ b/Regression/Checksum/benchmarks_json/collisionISO.json @@ -11,12 +11,12 @@ "jz": 0.0 }, "electron": { - "particle_momentum_x": 3.579130200773753e-19, - "particle_momentum_y": 3.5788119408700804e-19, - "particle_momentum_z": 3.584163522201744e-19, - "particle_position_x": 1.024188253213835, - "particle_position_y": 1.0238795904737117, - "particle_position_z": 1.02399735048655, + "particle_momentum_x": 3.579989064013309e-19, + "particle_momentum_y": 3.5822945977746767e-19, + "particle_momentum_z": 3.579753452653627e-19, + "particle_position_x": 1.0241322532163375, + "particle_position_y": 1.0238995904625479, + "particle_position_z": 1.02402135051502, "particle_weight": 714240000000.0 } } diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 5ec1f21c32a..39df1b9eee1 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,25 +6,25 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 353384.4616461907, - "T_ion": 348277.2774202612 + "T_electron": 353604.6247926339, + "T_ion": 347976.6168136309 }, "electron": { - "particle_momentum_x": 8.378820415249081e-19, - "particle_momentum_y": 8.238482720534644e-19, - "particle_momentum_z": 8.189221038447925e-19, - "particle_position_x": 21268719.554914076, - "particle_position_y": 21282437.437205918, - "particle_position_z": 21231947.47401523, + "particle_momentum_x": 8.370755929299189e-19, + "particle_momentum_y": 8.228112213603589e-19, + "particle_momentum_z": 8.204295817378347e-19, + "particle_position_x": 21284971.94721422, + "particle_position_y": 21212829.42991966, + "particle_position_z": 21214774.536558084, "particle_weight": 7.168263344048695e+28 }, "ion": { - "particle_momentum_x": 2.013855005436386e-18, - "particle_momentum_y": 1.8231305152698165e-18, - "particle_momentum_z": 1.8194149390229157e-18, - "particle_position_x": 21217737.157987915, - "particle_position_y": 21263096.30516862, - "particle_position_z": 21261343.069983635, + "particle_momentum_x": 2.0074097598289766e-18, + "particle_momentum_y": 1.8203553942782305e-18, + "particle_momentum_z": 1.823420185235695e-18, + "particle_position_x": 21227192.857240494, + "particle_position_y": 21286501.692027714, + "particle_position_z": 21245587.6706009, "particle_weight": 7.168263344048695e+28 } -} \ No newline at end of file +} diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index a386cb3012e..84e0fef8c9c 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -368,14 +368,12 @@ public: // given by the `indices_1[cell_start_1:cell_stop_1]` index_type const cell_start_1 = cell_offsets_1[i_cell]; index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; - index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; // Do not collide if there is only one particle in the cell if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } // shuffle - ShuffleFisherYates( - indices_1, cell_start_1, cell_half_1, engine ); + ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); } ); From f9748eb2201439d2f4391eff92e05bb0dfde90ea Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 15 Jul 2024 21:53:27 +0200 Subject: [PATCH 55/87] Fix Lassen: BLAS++/LAPACK++ (#5050) Update TOSS3 install scripts. --- .../lassen-llnl/install_v100_dependencies_toss3.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh index 916986ee119..9876334159b 100644 --- a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh @@ -82,13 +82,12 @@ if [ -d ${SRC_DIR}/blaspp ] then cd ${SRC_DIR}/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi -cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 # LAPACK++ (for PSATD+RZ) @@ -96,13 +95,12 @@ if [ -d ${SRC_DIR}/lapackpp ] then cd ${SRC_DIR}/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 From b6cf99d37701ce8e42952e04e7f05aa4b6c87eb9 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:07:27 -0700 Subject: [PATCH 56/87] Initialize with value all `aux`, `cax` fields (#5049) --- Source/WarpX.cpp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 4dfebe90c67..87fc5f2633d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2579,13 +2579,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } } } else { - AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]"); - AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]"); - AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]"); + AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]", 0.0_rt); + AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]", 0.0_rt); + AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]", 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]"); - AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]"); - AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); + AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]", 0.0_rt); + AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]", 0.0_rt); + AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]", 0.0_rt); } // The external fields that are read from file @@ -2736,22 +2736,22 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (n_field_gather_buffer > 0 || mypc->nSpeciesGatherFromMainGrid() > 0) { if (aux_is_nodal) { BoxArray const& cnba = amrex::convert(cba,IntVect::TheNodeVector()); - AllocInitMultiFab(Bfield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[x]"); - AllocInitMultiFab(Bfield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[y]"); - AllocInitMultiFab(Bfield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[z]"); - AllocInitMultiFab(Efield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Efield_cax[x]"); - AllocInitMultiFab(Efield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Efield_cax[y]"); - AllocInitMultiFab(Efield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Efield_cax[z]"); + AllocInitMultiFab(Bfield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[x]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[y]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[z]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Efield_cax[x]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Efield_cax[y]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Efield_cax[z]", 0.0_rt); } else { // Create the MultiFabs for B - AllocInitMultiFab(Bfield_cax[lev][0], amrex::convert(cba,Bx_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[x]"); - AllocInitMultiFab(Bfield_cax[lev][1], amrex::convert(cba,By_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[y]"); - AllocInitMultiFab(Bfield_cax[lev][2], amrex::convert(cba,Bz_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[z]"); + AllocInitMultiFab(Bfield_cax[lev][0], amrex::convert(cba,Bx_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[x]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][1], amrex::convert(cba,By_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[y]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][2], amrex::convert(cba,Bz_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[z]", 0.0_rt); // Create the MultiFabs for E - AllocInitMultiFab(Efield_cax[lev][0], amrex::convert(cba,Ex_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[x]"); - AllocInitMultiFab(Efield_cax[lev][1], amrex::convert(cba,Ey_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[y]"); - AllocInitMultiFab(Efield_cax[lev][2], amrex::convert(cba,Ez_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[z]"); + AllocInitMultiFab(Efield_cax[lev][0], amrex::convert(cba,Ex_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[x]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][1], amrex::convert(cba,Ey_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[y]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][2], amrex::convert(cba,Ez_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[z]", 0.0_rt); } AllocInitMultiFab(gather_buffer_masks[lev], ba, dm, ncomps, amrex::IntVect(1), lev, "gather_buffer_masks"); From c54eff69ce658c80f9db194d19f28daf7cd315b8 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 16 Jul 2024 06:39:45 +0200 Subject: [PATCH 57/87] ABLASTR FFT: Fix Build (#5051) Use the correct ABLASTR (not WarpX) CMake options and compiler defines. E.g., for ImpactX, we only control ABLASTR, not WarpX. --- Source/ablastr/fields/CMakeLists.txt | 2 +- Source/ablastr/fields/PoissonSolver.H | 8 ++++---- Source/ablastr/math/fft/CMakeLists.txt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/ablastr/fields/CMakeLists.txt b/Source/ablastr/fields/CMakeLists.txt index 158e623ebaf..56acc678217 100644 --- a/Source/ablastr/fields/CMakeLists.txt +++ b/Source/ablastr/fields/CMakeLists.txt @@ -1,6 +1,6 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) - if(WarpX_FFT AND D EQUAL 3) + if(ABLASTR_FFT AND D EQUAL 3) target_sources(ablastr_${SD} PRIVATE IntegratedGreenFunctionSolver.cpp diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index fed76b2987b..ca262981010 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -15,7 +15,7 @@ #include #include -#if defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D) +#if defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D) #include #endif @@ -160,9 +160,9 @@ computePhi (amrex::Vector const & rho, {{ beta[0], beta[1], beta[2] }}; #endif -#if !defined(WARPX_USE_FFT) +#if !defined(ABLASTR_USE_FFT) ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, - "Must compile with -DWarpX_FFT=ON to use the FFT solver!"); + "Must compile with FFT support to use the IGF solver!"); #endif #if !defined(WARPX_DIM_3D) @@ -170,7 +170,7 @@ computePhi (amrex::Vector const & rho, "The FFT Poisson solver is currently only implemented for 3D!"); #endif -#if (defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D)) +#if (defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D)) // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected if(is_solver_igf_on_lev0 && lev==0){ amrex::Array const dx_igf diff --git a/Source/ablastr/math/fft/CMakeLists.txt b/Source/ablastr/math/fft/CMakeLists.txt index 2b9fc3d37f9..913a912e1ee 100644 --- a/Source/ablastr/math/fft/CMakeLists.txt +++ b/Source/ablastr/math/fft/CMakeLists.txt @@ -1,6 +1,6 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) - if(WarpX_FFT STREQUAL ON) + if(ABLASTR_FFT STREQUAL ON) if(WarpX_COMPUTE STREQUAL CUDA) target_sources(ablastr_${SD} PRIVATE WrapCuFFT.cpp) elseif(WarpX_COMPUTE STREQUAL HIP) From 7d0f91a27ec100a148a0d0e51ceac82ea7eb9f3c Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 17 Jul 2024 01:03:36 +0200 Subject: [PATCH 58/87] AMReX/pyAMReX/PICSAR: Weekly Update (#5054) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 190294bb0e1..e6ba115925a 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.07 && cd - + cd ../amrex && git checkout --detach dcb9cc0383dcc71e38dee9070574e325a812f8bf && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 0003dcff134..b32ca61c709 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.07 +branch = dcb9cc0383dcc71e38dee9070574e325a812f8bf [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 6ee9820aa28..6d7c57f0961 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.07 +branch = dcb9cc0383dcc71e38dee9070574e325a812f8bf [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 0b90b09df30..8acbd974779 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.07" +set(WarpX_amrex_branch "dcb9cc0383dcc71e38dee9070574e325a812f8bf" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 5c17681d4b6..a2fbd0ddee8 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.07" +set(WarpX_pyamrex_branch "18f0026b1dd9b2aa4869c96f74e4b77262a067d0" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 21e84c7030d..c01c3fb1176 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.07 && cd - +cd amrex && git checkout --detach dcb9cc0383dcc71e38dee9070574e325a812f8bf && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From a13a11d5541bc245634cdc793bf997bfa12fbcc1 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 17 Jul 2024 17:49:51 +0200 Subject: [PATCH 59/87] Doc: Lassen Doc String h5py (#5052) Funny typo. --- Tools/machines/lassen-llnl/install_v100_dependencies.sh | 2 +- Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh index 6368010ab7b..48668efcc2a 100755 --- a/Tools/machines/lassen-llnl/install_v100_dependencies.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies.sh @@ -119,7 +119,7 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools -# Older version for h4py +# Older version for h5py # https://github.com/h5py/h5py/issues/2268 python3 -m pip install --upgrade "cython<3" python3 -m pip install --upgrade numpy diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh index 9876334159b..a6d5aad32fb 100644 --- a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh @@ -119,7 +119,7 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools -# Older version for h4py +# Older version for h5py # https://github.com/h5py/h5py/issues/2268 python3 -m pip install --upgrade "cython<3" python3 -m pip install --upgrade numpy From b58343a9cf31ad2e2e0c47b2c5f0b15cc122a961 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 18 Jul 2024 01:19:18 +0200 Subject: [PATCH 60/87] Fix CMake Python IPO Control (#5055) `pybind11::lto` is only defined if `CMAKE_INTERPROCEDURAL_OPTIMIZATION` is not set in `pybind11Common.cmake`. Package managers like Spack use the latter. --- CMakeLists.txt | 23 +++++++++++++++++++---- cmake/WarpXFunctions.cmake | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 23427a00ec8..9c81a057131 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -146,10 +146,19 @@ set_default_build_type("Release") # Option to enable interprocedural optimization # (also know as "link-time optimization" or "whole program optimization") -option(WarpX_IPO "Compile WarpX with interprocedural optimization (will take more time)" OFF) +set(_WarpX_IPO_DEFAULT OFF) +set(_WarpX_PYTHON_IPO_DEFAULT ON) +if(DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(_WarpX_IPO_DEFAULT ${CMAKE_INTERPROCEDURAL_OPTIMIZATION}) + set(_WarpX_PYTHON_IPO_DEFAULT ${CMAKE_INTERPROCEDURAL_OPTIMIZATION}) +endif() +option(WarpX_IPO + "Compile WarpX with interprocedural optimization (will take more time)" + ${_WarpX_IPO_DEFAULT} +) option(WarpX_PYTHON_IPO "Compile Python bindings with interprocedural optimization (IPO) / link-time optimization (LTO)" - ON + ${_WarpX_PYTHON_IPO_DEFAULT} ) set(pyWarpX_VERSION_INFO "" CACHE STRING @@ -455,7 +464,7 @@ endif() # Interprocedural optimization (IPO) / Link-Time Optimization (LTO) if(WarpX_IPO) - enable_IPO("${_ALL_TARGETS}") + warpx_enable_IPO("${_ALL_TARGETS}") endif() # link dependencies @@ -488,7 +497,13 @@ foreach(D IN LISTS WarpX_DIMS) if(WarpX_PYTHON) target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras) if(WarpX_PYTHON_IPO) - target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto) + if(DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + warpx_enable_IPO(pyWarpX_${SD}) + else() + # conditionally defined target in pybind11 + # https://github.com/pybind/pybind11/blob/v2.12.0/tools/pybind11Common.cmake#L397-L403 + target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto) + endif() endif() endif() diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index c12cedca3b2..e05b4c3afc4 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -209,7 +209,7 @@ endmacro() # Enables interprocedural optimization for a list of targets # -function(enable_IPO all_targets_list) +function(warpx_enable_IPO all_targets_list) include(CheckIPOSupported) check_ipo_supported(RESULT is_IPO_available) if(is_IPO_available) From f160cfcaf594ea0c0180bf6781dbfc5b425c50da Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Fri, 19 Jul 2024 17:51:17 +0200 Subject: [PATCH 61/87] Change `if (bool_var == 1)` into `if (bool_var)` for readability (#5060) --- Source/Diagnostics/SliceDiagnostic.cpp | 2 +- Source/Evolve/WarpXEvolve.cpp | 4 ++-- Source/Initialization/WarpXInitData.cpp | 12 ++++++------ Source/Parallelization/GuardCellManager.cpp | 6 +++--- Source/Particles/PhysicalParticleContainer.cpp | 4 ++-- Source/Utils/WarpXTagging.cpp | 2 +- Source/WarpX.cpp | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 14a2d633a0c..97af967f2be 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -153,7 +153,7 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, ablastr::utils::communication::ParallelCopy(*smf, mf, 0, 0, ncomp, nghost_vect, nghost_vect, WarpX::do_single_precision_comms); // interpolate if required on refined slice // - if (interpolate == 1 ) { + if (interpolate) { InterpolateSliceValues( *smf, interp_lo, slice_cc_nd_box, dom_geom, ncomp, nghost, slice_lo, slice_hi, SliceType, real_box); } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 8b2cc2c33f1..7bdba64dc4e 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -142,7 +142,7 @@ WarpX::Evolve (int numsteps) OneStep_multiJ(cur_time); } // Electromagnetic case: no subcycling or no mesh refinement - else if (do_subcycling == 0 || finest_level == 0) + else if ( !do_subcycling || (finest_level == 0)) { OneStep_nosub(cur_time); // E: guard cells are up-to-date @@ -150,7 +150,7 @@ WarpX::Evolve (int numsteps) // F: guard cells are NOT up-to-date } // Electromagnetic case: subcycling with one level of mesh refinement - else if (do_subcycling == 1 && finest_level == 1) + else if (do_subcycling && (finest_level == 1)) { OneStep_sub1(cur_time); } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 9a4c46b1c10..8af0ce23ce8 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -294,19 +294,19 @@ WarpX::PrintMainPICparameters () " | - v_comoving = (" << m_v_comoving[0] << "," << m_v_comoving[1] << "," << m_v_comoving[2] << ")\n"; } - if (WarpX::update_with_rho==1) { + if (WarpX::update_with_rho) { amrex::Print() << " | - update with rho is ON \n"; } - if (current_correction==1) { + if (current_correction) { amrex::Print() << " | - current correction is ON \n"; } - if (WarpX::do_dive_cleaning==1) { + if (WarpX::do_dive_cleaning) { amrex::Print() << " | - div(E) cleaning is ON \n"; } - if (WarpX::do_divb_cleaning==1) { + if (WarpX::do_divb_cleaning) { amrex::Print() << " | - div(B) cleaning is ON \n"; } - if (do_multi_J == 1){ + if (do_multi_J){ amrex::Print() << " | - multi-J deposition is ON \n"; amrex::Print() << " | - do_multi_J_n_depositions = " << WarpX::do_multi_J_n_depositions << "\n"; @@ -323,7 +323,7 @@ WarpX::PrintMainPICparameters () amrex::Print() << " | - rho_in_time = constant \n"; } } - if (fft_do_time_averaging == 1){ + if (fft_do_time_averaging){ amrex::Print()<<" | - time-averaged is ON \n"; } #endif // WARPX_USE_FFT diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 321be15df7e..bf010736853 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -59,9 +59,9 @@ guardCellManager::Init ( // When using subcycling, the particles on the fine level perform two pushes // before being redistributed ; therefore, we need one extra guard cell // (the particles may move by 2*c*dt) - int ngx_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; - int ngy_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; - int ngz_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; + int ngx_tmp = (max_level > 0 && do_subcycling) ? nox+1 : nox; + int ngy_tmp = (max_level > 0 && do_subcycling) ? nox+1 : nox; + int ngz_tmp = (max_level > 0 && do_subcycling) ? nox+1 : nox; const bool galilean = (v_galilean[0] != 0. || v_galilean[1] != 0. || v_galilean[2] != 0.); const bool comoving = (v_comoving[0] != 0. || v_comoving[1] != 0. || v_comoving[2] != 0.); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 89ae435882b..3a30467c20d 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1065,7 +1065,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int static_cast(std::round((overlap_realbox.lo(dir)-problo[dir])/dx[dir])); // shifted is exact in non-moving-window direction. That's all we care. } - if (no_overlap == 1) { + if (no_overlap) { continue; // Go to the next tile } @@ -1640,7 +1640,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // shifted is exact in non-moving-window direction. That's all we care. } } - if (no_overlap == 1) { + if (no_overlap) { continue; // Go to the next tile } diff --git a/Source/Utils/WarpXTagging.cpp b/Source/Utils/WarpXTagging.cpp index f88380e2589..b6eb2fda5d2 100644 --- a/Source/Utils/WarpXTagging.cpp +++ b/Source/Utils/WarpXTagging.cpp @@ -61,7 +61,7 @@ WarpX::ErrorEst (int lev, TagBoxArray& tags, Real /*time*/, int /*ngrow*/) } else { tag_val = (pos > ftlo && pos < fthi); } - if ( tag_val == 1) { + if (tag_val) { fab(i,j,k) = TagBox::SET; } }); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 87fc5f2633d..fcd10418b23 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2457,7 +2457,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(phi_fp[lev], amrex::convert(ba, phi_nodal_flag), dm, ncomps, ngPhi, lev, "phi_fp", 0.0_rt); } - if (do_subcycling == 1 && lev == 0) + if (do_subcycling && lev == 0) { AllocInitMultiFab(current_store[lev][0], amrex::convert(ba,jx_nodal_flag),dm,ncomps,ngJ,lev, "current_store[x]"); AllocInitMultiFab(current_store[lev][1], amrex::convert(ba,jy_nodal_flag),dm,ncomps,ngJ,lev, "current_store[y]"); From 8bf0fb0d7a63248b6aa1084a5294996034a31e43 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 20 Jul 2024 00:40:53 +0200 Subject: [PATCH 62/87] Doc: Lassen Stays TOSS3, Fix SciPy (#5063) Sadly, the TOSS4 transition was not completed before the IBM contract ran out and Lassen stays with TOSS3. This simplifies the documentation again. --- Docs/source/install/hpc/lassen.rst | 161 ++++-------------- .../lassen-llnl/install_v100_dependencies.sh | 142 --------------- .../install_v100_dependencies_toss3.sh | 4 +- .../lassen_v100_warpx.profile.example | 56 ------ .../lassen_v100_warpx_toss3.profile.example | 2 +- 5 files changed, 33 insertions(+), 332 deletions(-) delete mode 100755 Tools/machines/lassen-llnl/install_v100_dependencies.sh delete mode 100644 Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index 0d885dff04d..43330dcc111 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -24,27 +24,9 @@ If you are new to this system, **please see the following resources**: Login ----- -.. tab-set:: - - .. tab-item:: TOSS4 (RHEL8) - - Lassen is currently transitioning to RHEL8. - During this transition, first SSH into lassen and then to the updated RHEL8/TOSS4 nodes. - - .. code-block:: bash - - ssh lassen.llnl.gov - ssh eatoss4 - - Approximately October/November 2023, the new software environment on these nodes will be the new default. - - .. tab-item:: TOSS3 (RHEL7) - - .. code-block:: bash - - ssh lassen.llnl.gov +.. code-block:: bash - Approximately October/November 2023, this partition will become TOSS4 (RHEL8) as well. + ssh lassen.llnl.gov .. _building-lassen-preparation: @@ -58,135 +40,54 @@ Use the following commands to download the WarpX source code: git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/lassen/src/warpx -.. tab-set:: - - .. tab-item:: TOSS4 (RHEL8) - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``. - Create it now: - - .. code-block:: bash - - cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example - :language: bash - - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: - - .. code-block:: bash - - export proj="nsldt" - - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). - - .. important:: - - Now, and as the first step on future logins to lassen, activate these environment settings: - - .. code-block:: bash +We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``. +Create it now: - source $HOME/lassen_v100_warpx.profile - - .. tab-item:: TOSS3 (RHEL7) - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``. - Create it now: - - .. code-block:: bash +.. code-block:: bash - cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile + cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example - :language: bash + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example + :language: bash - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: +Edit the 2nd line of this script, which sets the ``export proj=""`` variable. +For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``. +Enter the edit mode by typing ``i`` and edit line 2 to read: - .. code-block:: bash +.. code-block:: bash - export proj="nsldt" + export proj="nsldt" - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). +Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). - .. important:: +.. important:: - Now, and as the first step on future logins to lassen, activate these environment settings: + Now, and as the first step on future logins to lassen, activate these environment settings: - .. code-block:: bash + .. code-block:: bash - source $HOME/lassen_v100_warpx_toss3.profile + source $HOME/lassen_v100_warpx_toss3.profile Finally, since lassen does not yet provide software modules for some of our dependencies, install them once: -.. tab-set:: - - .. tab-item:: TOSS4 (RHEL8) - - .. code-block:: bash - - bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh - source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh - :language: bash - - .. dropdown:: AI/ML Dependencies (Optional) - :animate: fade-in-slide-down - - If you plan to run AI/ML workflows depending on pyTorch, run the next step as well. - This will take a while and should be skipped if not needed. - - .. code-block:: bash - - runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh - :language: bash - - For `optimas dependencies `__ (incl. scikit-learn), plan another hour of build time: - - .. code-block:: bash - - python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt - - .. tab-item:: TOSS3 (RHEL7) +.. code-block:: bash - .. code-block:: bash + bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh + source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate - bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh - source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh + :language: bash - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh - :language: bash .. _building-lassen-compilation: diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh deleted file mode 100755 index 48668efcc2a..00000000000 --- a/Tools/machines/lassen-llnl/install_v100_dependencies.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was lassen_v100_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lassen_v100_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SRC_DIR="/usr/workspace/${USER}/lassen/src" -SW_DIR="/usr/workspace/${USER}/lassen/gpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} -mkdir -p ${SRC_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# tmpfs build directory: avoids issues often seen with $HOME and is faster -build_dir=$(mktemp -d) - -# c-blosc (I/O compression) -if [ -d ${SRC_DIR}/c-blosc ] -then - cd ${SRC_DIR}/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc -fi -cmake -S ${SRC_DIR}/c-blosc -B ${build_dir}/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build ${build_dir}/c-blosc-lassen-build --target install --parallel 10 - -# HDF5 -if [ -d ${SRC_DIR}/hdf5 ] -then - cd ${SRC_DIR}/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git ${SRC_DIR}/hdf5 -fi -cmake -S ${SRC_DIR}/hdf5 -B ${build_dir}/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build ${build_dir}/hdf5-lassen-build --target install --parallel 10 - -# ADIOS2 -if [ -d ${SRC_DIR}/adios2 ] -then - cd ${SRC_DIR}/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 -fi -cmake -S ${SRC_DIR}/adios2 -B ${build_dir}/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build ${build_dir}/adios2-lassen-build --target install -j 10 - -# BLAS++ (for PSATD+RZ) -if [ -d ${SRC_DIR}/blaspp ] -then - cd ${SRC_DIR}/blaspp - git fetch --prune - git checkout v2024.05.31 - cd - -else - git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp -fi -cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 -cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 - -# LAPACK++ (for PSATD+RZ) -if [ -d ${SRC_DIR}/lapackpp ] -then - cd ${SRC_DIR}/lapackpp - git fetch --prune - git checkout v2024.05.31 - cd - -else - git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp -fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so -cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 - - -# Python ###################################################################### -# -# sometimes, the Lassen PIP Index is down -export PIP_EXTRA_INDEX_URL="https://pypi.org/simple" - -python3 -m pip install --upgrade --user virtualenv -rm -rf ${SW_DIR}/venvs/warpx-lassen -python3 -m venv ${SW_DIR}/venvs/warpx-lassen -source ${SW_DIR}/venvs/warpx-lassen/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip cache purge -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -# Older version for h5py -# https://github.com/h5py/h5py/issues/2268 -python3 -m pip install --upgrade "cython<3" -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade -Ccompile-args="-j10" scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -CC=mpicc H5PY_SETUP_REQUIRES=0 HDF5_DIR=${SW_DIR}/hdf5-1.14.1.2 HDF5_MPI=ON python3 -m pip install --upgrade h5py --no-cache-dir --no-build-isolation --no-binary h5py -MPLLOCALFREETYPE=1 python3 -m pip install --upgrade matplotlib==3.2.2 # does not try to build freetype itself -echo "matplotlib==3.2.2" > ${build_dir}/constraints.txt -python3 -m pip install --upgrade -c ${build_dir}/constraints.txt yt - -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt - -# for ML dependencies, see install_v100_ml.sh - - -# remove build temporary directory -rm -rf ${build_dir} diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh index a6d5aad32fb..1b14159cd22 100644 --- a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh @@ -119,9 +119,7 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools -# Older version for h5py -# https://github.com/h5py/h5py/issues/2268 -python3 -m pip install --upgrade "cython<3" +python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas CMAKE_PREFIX_PATH=/usr/lib64:${CMAKE_PREFIX_PATH} python3 -m pip install --upgrade -Ccompile-args="-j10" -Csetup-args=-Dblas=BLAS -Csetup-args=-Dlapack=BLAS scipy diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example deleted file mode 100644 index 80755ece308..00000000000 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example +++ /dev/null @@ -1,56 +0,0 @@ -# please set your project account -#export proj="" # edit this and comment in - -# required dependencies -module load cmake/3.23.1 -module load gcc/11.2.1 -module load cuda/12.0.0 - -# optional: for QED lookup table generation support -module load boost/1.70.0 - -# optional: for openPMD support -SRC_DIR="/usr/workspace/${USER}/lassen/src" -SW_DIR="/usr/workspace/${USER}/lassen/gpu" -export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} -export PATH=${SW_DIR}/adios2-2.8.3/bin:$PATH - -# optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH - -# optional: for Python bindings -module load python/3.8.2 - -if [ -d "${SW_DIR}/venvs/warpx-lassen" ] -then - source ${SW_DIR}/venvs/warpx-lassen/bin/activate -fi - -# optional: an alias to request an interactive node for two hours -alias getNode="bsub -G $proj -W 2:00 -nnodes 1 -Is /bin/bash" -# an alias to run a command on a batch node for up to 30min -# usage: runNode -alias runNode="bsub -q debug -P $proj -W 2:00 -nnodes 1 -I" - -# fix system defaults: do not escape $ with a \ on tab completion -shopt -s direxpand - -# optimize CUDA compilation for V100 -export AMREX_CUDA_ARCH=7.0 -export CUDAARCHS=70 - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example index 6a7067f596e..98e8d6410b3 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example @@ -28,7 +28,7 @@ export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings -module load python/3.8.2 +module load python/3.11.5 if [ -d "${SW_DIR}/venvs/warpx-lassen-toss3" ] then From c6bb95acf52b088341ee95052fa5f50ca0d55f0e Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Fri, 19 Jul 2024 18:18:41 -0700 Subject: [PATCH 63/87] =?UTF-8?q?Avoid=20N=C2=B2=20Operation=20in=20Binary?= =?UTF-8?q?-Pairing=20Coulomb=20Collsions=20(#5066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * moved density and temperature calc from each pair to once for each cell. * pulled out n12 calc to cell level. * fixed dV issue for cyl geom. --- .../BinaryCollision/BinaryCollision.H | 172 ++++++++++++++++++ .../Coulomb/ElasticCollisionPerez.H | 69 ++----- .../Coulomb/PairWiseCoulombCollisionFunc.H | 16 +- .../Collision/BinaryCollision/DSMC/DSMCFunc.H | 5 + .../NuclearFusion/NuclearFusionFunc.H | 5 + 5 files changed, 207 insertions(+), 60 deletions(-) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 84e0fef8c9c..0ba49e9b09f 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -8,6 +8,7 @@ #define WARPX_PARTICLES_COLLISION_BINARYCOLLISION_H_ #include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" +#include "Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H" #include "Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" @@ -360,6 +361,21 @@ public: End of calculations only required when creating product particles */ + // create vectors to store density and temperature on cell level + amrex::Gpu::DeviceVector n1_vec; + amrex::Gpu::DeviceVector n12_vec; + amrex::Gpu::DeviceVector T1_vec; + if (binary_collision_functor.m_computeSpeciesDensities) { + n1_vec.resize(n_cells); + n12_vec.resize(n_cells); + } + if (binary_collision_functor.m_computeSpeciesTemperatures) { + T1_vec.resize(n_cells); + } + amrex::ParticleReal* AMREX_RESTRICT n1_in_each_cell = n1_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT n12_in_each_cell = n12_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT T1_in_each_cell = T1_vec.dataPtr(); + // Loop over cells amrex::ParallelForRNG( n_cells, [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept @@ -368,12 +384,60 @@ public: // given by the `indices_1[cell_start_1:cell_stop_1]` index_type const cell_start_1 = cell_offsets_1[i_cell]; index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; // Do not collide if there is only one particle in the cell if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } + // compute local density [1/m^3] + if (binary_collision_functor.m_computeSpeciesDensities) { + amrex::ParticleReal wtot1 = 0.0; + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + for (index_type i1=cell_start_1; i1 n1_vec, n2_vec; + amrex::Gpu::DeviceVector n12_vec; + amrex::Gpu::DeviceVector T1_vec, T2_vec; + if (binary_collision_functor.m_computeSpeciesDensities) { + n1_vec.resize(n_cells); + n2_vec.resize(n_cells); + n12_vec.resize(n_cells); + } + if (binary_collision_functor.m_computeSpeciesTemperatures) { + T1_vec.resize(n_cells); + T2_vec.resize(n_cells); + } + amrex::ParticleReal* AMREX_RESTRICT n1_in_each_cell = n1_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT n2_in_each_cell = n2_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT n12_in_each_cell = n12_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT T1_in_each_cell = T1_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT T2_in_each_cell = T2_vec.dataPtr(); // Loop over cells amrex::ParallelForRNG( n_cells, @@ -579,6 +675,43 @@ public: // ux_1[ indices_1[i] ], where i is between // cell_start_1 (inclusive) and cell_start_2 (exclusive) + // compute local densities [1/m^3] + if (binary_collision_functor.m_computeSpeciesDensities) { + amrex::ParticleReal w1tot = 0.0; + amrex::ParticleReal w2tot = 0.0; + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + for (index_type i1=cell_start_1; i1 /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const n1, amrex::ParticleReal const n2, + amrex::ParticleReal const n12, + amrex::ParticleReal const T1, amrex::ParticleReal const T2, amrex::ParticleReal const q1, amrex::ParticleReal const q2, amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, index_type coll_idx, + amrex::Real const dt, amrex::Real const /*dV*/, index_type coll_idx, index_type const /*cell_start_pair*/, index_type* /*p_mask*/, index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, amrex::ParticleReal* /*p_pair_reaction_weight*/, @@ -99,12 +105,14 @@ public: ElasticCollisionPerez( I1s, I1e, I2s, I2e, I1, I2, - soa_1, soa_2, - q1, q2, m1, m2, -1.0_prt, -1.0_prt, - dt, m_CoulombLog, dV, engine, m_isSameSpecies, coll_idx); + soa_1, soa_2, n1, n2, n12, T1, T2, + q1, q2, m1, m2, + dt, m_CoulombLog, engine, coll_idx); } amrex::ParticleReal m_CoulombLog; + bool m_computeSpeciesDensities = true; + bool m_computeSpeciesTemperatures = false; bool m_isSameSpecies; }; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H index f28f3fb984c..152dca4e1a1 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -96,6 +96,9 @@ public: index_type const* AMREX_RESTRICT I2, const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*n1*/, amrex::ParticleReal const /*n2*/, + amrex::ParticleReal const /*n12*/, + amrex::ParticleReal const /*T1*/, amrex::ParticleReal const /*T2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, amrex::Real const dt, amrex::Real const dV, index_type coll_idx, @@ -189,6 +192,8 @@ public: } int m_process_count; + bool m_computeSpeciesDensities = false; + bool m_computeSpeciesTemperatures = false; ScatteringProcess::Executor* m_scattering_processes_data; }; diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index b6d6fe171de..1f3e5e56582 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -131,6 +131,9 @@ public: index_type const* AMREX_RESTRICT I2, const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*n1*/, amrex::ParticleReal const /*n2*/, + amrex::ParticleReal const /*n12*/, + amrex::ParticleReal const /*T1*/, amrex::ParticleReal const /*T2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, amrex::Real const dt, amrex::Real const dV, index_type coll_idx, @@ -232,6 +235,8 @@ public: amrex::ParticleReal m_probability_threshold; amrex::ParticleReal m_probability_target_value; NuclearFusionType m_fusion_type; + bool m_computeSpeciesDensities = false; + bool m_computeSpeciesTemperatures = false; bool m_isSameSpecies; }; From 7dad8f33390cdeccc804e4a0c20418db7b34e12c Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 22 Jul 2024 10:23:23 -0700 Subject: [PATCH 64/87] GNU Make: fix make clean (#5067) --- Source/Make.WarpX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 519eed52820..502cb8a6714 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -326,7 +326,7 @@ $(srcTempDir)/Utils/export.H: @echo "#endif" >> $@ cleanconfig:: - $(SILENT) $(RM) -r $(srcTempDir)/Utils + $(SILENT) $(RM) -r $(srcTempDir)/Utils $(srcTempDir)/ablastr include $(AMREX_HOME)/Tools/GNUMake/Make.rules From f1af6f40636f8ae3085d350de425ce52f06c5479 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 22 Jul 2024 11:38:32 -0700 Subject: [PATCH 65/87] AMReX/pyAMReX/PICSAR: Weekly Update (#5074) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index e6ba115925a..c2ec9aaa763 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach dcb9cc0383dcc71e38dee9070574e325a812f8bf && cd - + cd ../amrex && git checkout --detach 0c3273f5e591815909180f8ffaf5b793cabbf9bc && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index b32ca61c709..a2447ac3cf5 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = dcb9cc0383dcc71e38dee9070574e325a812f8bf +branch = 0c3273f5e591815909180f8ffaf5b793cabbf9bc [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 6d7c57f0961..17ecfd64f29 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = dcb9cc0383dcc71e38dee9070574e325a812f8bf +branch = 0c3273f5e591815909180f8ffaf5b793cabbf9bc [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 8acbd974779..025b4d3b6e0 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "dcb9cc0383dcc71e38dee9070574e325a812f8bf" +set(WarpX_amrex_branch "0c3273f5e591815909180f8ffaf5b793cabbf9bc" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index a2fbd0ddee8..a4598953432 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "18f0026b1dd9b2aa4869c96f74e4b77262a067d0" +set(WarpX_pyamrex_branch "ff4643869c63d4ee40a87054b901f61eefcb97a3" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index c01c3fb1176..1692f66263c 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach dcb9cc0383dcc71e38dee9070574e325a812f8bf && cd - +cd amrex && git checkout --detach 0c3273f5e591815909180f8ffaf5b793cabbf9bc && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 396c06106ac7d635c70d46e112fb1b7ff2b720ad Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 24 Jul 2024 23:05:44 -0700 Subject: [PATCH 66/87] Fix again the collisionXYZ benchmark (#5083) --- .../Tests/collision/analysis_collision_2d.py | 2 +- .../benchmarks_json/collisionXYZ.json | 28 +++++++++---------- .../Checksum/benchmarks_json/collisionXZ.json | 16 +++++++++++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/Examples/Tests/collision/analysis_collision_2d.py b/Examples/Tests/collision/analysis_collision_2d.py index bd013898a8a..29482909b2e 100755 --- a/Examples/Tests/collision/analysis_collision_2d.py +++ b/Examples/Tests/collision/analysis_collision_2d.py @@ -114,4 +114,4 @@ dim, species_name) test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 39df1b9eee1..2aa0e4686eb 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,25 +6,25 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 353604.6247926339, - "T_ion": 347976.6168136309 + "T_electron": 381488.6528070591, + "T_ion": 320091.2785835478 }, "electron": { - "particle_momentum_x": 8.370755929299189e-19, - "particle_momentum_y": 8.228112213603589e-19, - "particle_momentum_z": 8.204295817378347e-19, - "particle_position_x": 21284971.94721422, - "particle_position_y": 21212829.42991966, - "particle_position_z": 21214774.536558084, + "particle_momentum_x": 8.667025573698235e-19, + "particle_momentum_y": 8.457499789250831e-19, + "particle_momentum_z": 8.482438182280524e-19, + "particle_position_x": 21262567.138872623, + "particle_position_y": 21245135.070665065, + "particle_position_z": 21232644.283726066, "particle_weight": 7.168263344048695e+28 }, "ion": { - "particle_momentum_x": 2.0074097598289766e-18, - "particle_momentum_y": 1.8203553942782305e-18, - "particle_momentum_z": 1.823420185235695e-18, - "particle_position_x": 21227192.857240494, - "particle_position_y": 21286501.692027714, - "particle_position_z": 21245587.6706009, + "particle_momentum_x": 1.9300495097720012e-18, + "particle_momentum_y": 1.747257416857836e-18, + "particle_momentum_z": 1.7510296287537058e-18, + "particle_position_x": 21217348.883301035, + "particle_position_y": 21300859.0630925, + "particle_position_z": 21237901.246521123, "particle_weight": 7.168263344048695e+28 } } diff --git a/Regression/Checksum/benchmarks_json/collisionXZ.json b/Regression/Checksum/benchmarks_json/collisionXZ.json index 927848745a8..49138f0a1df 100644 --- a/Regression/Checksum/benchmarks_json/collisionXZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXZ.json @@ -6,5 +6,21 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0 + }, + "ion": { + "particle_momentum_x": 2.458306853810186e-19, + "particle_momentum_y": 2.272685285153902e-19, + "particle_momentum_z": 2.281205462681013e-19, + "particle_position_x": 2645436.647039526, + "particle_position_y": 2672571.48688055, + "particle_weight": 1.7256099431746894e+26 + }, + "electron": { + "particle_momentum_x": 1.0454942263455085e-19, + "particle_momentum_y": 1.0323735347957779e-19, + "particle_momentum_z": 1.0199134968670343e-19, + "particle_position_x": 2681776.3648108337, + "particle_position_y": 2663907.8843079703, + "particle_weight": 1.7256099431746894e+26 } } From 49df0ca46930229678c3bebe063f949e24b63013 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 25 Jul 2024 19:15:13 +0200 Subject: [PATCH 67/87] [WIP] Recompute the macroscopic properties everytime the moving window moves. (#5082) * Recompute the macroscopic properties everytime the moving window moves. * Minor cleanup * Separate allocation and initialization --- .../MacroscopicProperties.H | 14 +++++++++---- .../MacroscopicProperties.cpp | 20 ++++++++++--------- Source/Initialization/WarpXInitData.cpp | 5 +---- Source/Utils/WarpXMovingWindow.cpp | 12 +++++++++++ Source/WarpX.cpp | 7 +++++++ 5 files changed, 41 insertions(+), 17 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H index 129b2e2f782..ca9c77334ea 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H @@ -38,20 +38,26 @@ public: void ReadParameters (); /** - * \brief Initialize multifabs storing macroscopic multifabs + * \brief Allocate multifabs storing macroscopic multifabs * * @param[in] ba the box array associated to the multifabs E and B * @param[in] dmap the distribution mapping * @param[in] ng_EB_alloc guard cells allocated for multifabs E and B + */ + void AllocateLevelMFs ( + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm, + const amrex::IntVect& ng_EB_alloc ); + + /** + * \brief Initialize multifabs storing macroscopic multifabs + * * @param[in] geom the geometry * @param[in] Ex_stag staggering of the Ex field * @param[in] Ey_stag staggering of the Ey field * @param[in] Ez_stag staggering of the Ez field */ void InitData ( - const amrex::BoxArray& ba, - const amrex::DistributionMapping& dmap, - const amrex::IntVect& ng_EB_alloc, const amrex::Geometry& geom, const amrex::IntVect& Ex_stag, const amrex::IntVect& Ey_stag, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index 1467c7a8c0c..a6a389fe056 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -119,24 +119,26 @@ MacroscopicProperties::ReadParameters () } void -MacroscopicProperties::InitData ( +MacroscopicProperties::AllocateLevelMFs ( const amrex::BoxArray& ba, const amrex::DistributionMapping& dmap, - const amrex::IntVect& ng_EB_alloc, - const amrex::Geometry& geom, - const amrex::IntVect& Ex_stag, - const amrex::IntVect& Ey_stag, - const amrex::IntVect& Ez_stag) + const amrex::IntVect& ng_EB_alloc ) { - amrex::Print() << Utils::TextMsg::Info("we are in init data of macro"); - - // Define material property multifabs using ba and dmap from WarpX instance // sigma is cell-centered MultiFab m_sigma_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); // epsilon is cell-centered MultiFab m_eps_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); // mu is cell-centered MultiFab m_mu_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); +} + +void +MacroscopicProperties::InitData ( + const amrex::Geometry& geom, + const amrex::IntVect& Ex_stag, + const amrex::IntVect& Ey_stag, + const amrex::IntVect& Ez_stag) +{ // Initialize sigma if (m_sigma_s == "constant") { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 8af0ce23ce8..e9f63e34bb0 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -474,12 +474,9 @@ WarpX::InitData () BuildBufferMasks(); - if (WarpX::em_solver_medium==1) { + if (WarpX::em_solver_medium == MediumForEM::Macroscopic) { const int lev_zero = 0; m_macroscopic_properties->InitData( - boxArray(lev_zero), - DistributionMap(lev_zero), - getngEB(), Geom(lev_zero), getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 5fcaccf761a..d64394dafc9 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -19,6 +19,7 @@ #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include @@ -452,6 +453,17 @@ WarpX::MoveWindow (const int step, bool move_j) } } + // Recompute macroscopic properties of the medium + if (WarpX::em_solver_medium == MediumForEM::Macroscopic) { + const int lev_zero = 0; + m_macroscopic_properties->InitData( + Geom(lev_zero), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + ); + } + return num_shift_base; } diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index fcd10418b23..b725431c2f1 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2373,6 +2373,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm myfl->InitData(lev, geom[lev].Domain(),cur_time); } + // Allocate extra multifabs for macroscopic properties of the medium + if (em_solver_medium == MediumForEM::Macroscopic) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( lev==0, + "Macroscopic properties are not supported with mesh refinement."); + m_macroscopic_properties->AllocateLevelMFs(ba, dm, ngEB); + } + if (fft_do_time_averaging) { AllocInitMultiFab(Bfield_avg_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[x]", 0.0_rt); From eda6af5525d89eb251305f340f2f9b6a955b1b48 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 25 Jul 2024 19:26:15 +0200 Subject: [PATCH 68/87] Update script to compile WarpX on Frontier (OLCF) (#5076) * update profile script for Frontier * satisfy dependency for module load * change module versions and manually install adios2 * fix bug * fix bug * update instructions * update scripts * remove line for LibEnsemble --- .../frontier-olcf/frontier_warpx.profile.example | 14 +++++++------- .../machines/frontier-olcf/install_dependencies.sh | 3 --- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/Tools/machines/frontier-olcf/frontier_warpx.profile.example b/Tools/machines/frontier-olcf/frontier_warpx.profile.example index 50e12e99a38..f59f2d3d058 100644 --- a/Tools/machines/frontier-olcf/frontier_warpx.profile.example +++ b/Tools/machines/frontier-olcf/frontier_warpx.profile.example @@ -8,9 +8,9 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo # required dependencies module load cmake/3.23.2 module load craype-accel-amd-gfx90a -module load rocm/5.2.0 # waiting for 5.6 for next bump -module load cray-mpich -module load cce/15.0.0 # must be loaded after rocm +module load rocm/5.7.1 +module load cray-mpich/8.1.28 +module load cce/17.0.0 # must be loaded after rocm # optional: faster builds module load ccache @@ -26,14 +26,14 @@ export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/blaspp-2024.05.31/lib64:$LD_LIBRA export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support -module load boost/1.79.0-cxx17 +module load boost/1.79.0 # optional: for openPMD support -module load adios2/2.8.3 -module load hdf5/1.14.0 +module load adios2/2.8.3-mpi +module load hdf5/1.12.1-mpi # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.11.5 if [ -d "${HOME}/sw/frontier/gpu/venvs/warpx-frontier" ] then diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 503ab4525e9..fd1d28e76b5 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -74,7 +74,6 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B cmake --build $HOME/src/lapackpp-frontier-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-frontier-gpu-build - # Python ###################################################################### # python3 -m pip install --upgrade pip @@ -109,8 +108,6 @@ CUPY_INSTALL_USE_HIP=1 \ ROCM_HOME=${ROCM_PATH} \ HCC_AMDGPU_TARGET=${AMREX_AMD_ARCH} \ python3 -m pip install -v cupy -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2 #python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt From 675369d9797aa2bcfda8b64c8e500fff6a2b9f04 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 25 Jul 2024 10:29:56 -0700 Subject: [PATCH 69/87] PICMI (Bucket): NumPy 2.0 Compatibility (#5075) * PICMI (Bucket): NumPy 2.0 Compatibility `repr` returns `'np.float64(value)'` instead of `'value'` in NumPy 2.0 * Use fstrings * Remove outdated comment * `requirements.txt`: Relax `numpy` version Version 1 and 2 support. * Remove outdated comment --------- Co-authored-by: David Grote --- Python/pywarpx/Bucket.py | 6 ++---- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Python/pywarpx/Bucket.py b/Python/pywarpx/Bucket.py index 91a34b9d3d6..9dbe4def88e 100644 --- a/Python/pywarpx/Bucket.py +++ b/Python/pywarpx/Bucket.py @@ -59,8 +59,6 @@ def attrlist(self): for attr, value in self.argvattrs.items(): if value is None: continue - # --- repr is applied to value so that for floats, all of the digits are included. - # --- The strip of "'" is then needed when value is a string. if isinstance(value, str): if value.find('=') > -1: # --- Expressions with temporary variables need to be inside quotes @@ -73,11 +71,11 @@ def attrlist(self): continue # --- For lists, tuples, and arrays make a space delimited string of the values. # --- The lambda is needed in case this is a list of strings. - rhs = ' '.join(map(lambda s : repr(s).strip("'"), value)) + rhs = ' '.join(map(lambda s : f'{s}', value)) elif isinstance(value, bool): rhs = 1 if value else 0 else: rhs = value - attrstring = '{0}.{1} = {2}'.format(self.instancename, attr, repr(rhs).strip("'")) + attrstring = f'{self.instancename}.{attr} = {rhs}' result += [attrstring] return result diff --git a/requirements.txt b/requirements.txt index bb0d8ebb704..8e664ae3096 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # basic dependencies -numpy~=1.15 +numpy>=1.15 periodictable~=1.5 # PICMI From 5343aa87aa1d98fb68aafd6546db09b0e69f8d8b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 29 Jul 2024 11:24:32 +0200 Subject: [PATCH 70/87] Add PRL citation (#5093) --- Docs/source/highlights.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 7baec74d606..108f685a551 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -14,6 +14,11 @@ Plasma-Based Acceleration Scientific works in laser-plasma and beam-plasma acceleration. +#. Shrock JE, Rockafellow E, Miao B, Le M, Hollinger RC, Wang S, Gonsalves AJ, Picksley A, Rocca JJ, and Milchberg HM + **Guided Mode Evolution and Ionization Injection in Meter-Scale Multi-GeV Laser Wakefield Accelerators**. + Phys. Rev. Lett. **133**, 045002, 2024 + `DOI:10.1103/PhysRevLett.133.045002 `__ + #. Ross AJ, Chappell J, van de Wetering JJ, Cowley J, Archer E, Bourgeois N, Corner L, Emerson DR, Feder L, Gu XJ, Jakobsson O, Jones H, Picksley A, Reid L, Wang W, Walczak R, Hooker SM. **Resonant excitation of plasma waves in a plasma channel**. Phys. Rev. Research **6**, L022001, 2024 From b1e7932cc2df0f07ede608f2a2eaf9cbde99d600 Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Mon, 29 Jul 2024 12:05:13 -0700 Subject: [PATCH 71/87] Improving Coulomb collision method for weighted-particles (#5091) * using corrected weighted-particle Coulomb collision method. * adding CI test for weighted Coulomb collisions. * using n12 in UpdateMomentumPerezElastic * updating Checksum * fixing type issue. * updating checksums * checksum --- .../Tests/collision/analysis_collision_1d.py | 126 ++++++++++++++++++ Examples/Tests/collision/inputs_1d | 90 +++++++++++++ .../benchmarks_json/collisionISO.json | 12 +- .../benchmarks_json/collisionXYZ.json | 28 ++-- .../Checksum/benchmarks_json/collisionXZ.json | 20 +-- .../Checksum/benchmarks_json/collisionZ.json | 20 +++ Regression/WarpX-tests.ini | 14 ++ .../BinaryCollision/BinaryCollision.H | 61 +-------- .../Coulomb/ElasticCollisionPerez.H | 31 +++-- .../Coulomb/PairWiseCoulombCollisionFunc.H | 7 +- .../Coulomb/UpdateMomentumPerezElastic.H | 11 +- .../Collision/BinaryCollision/DSMC/DSMCFunc.H | 1 - .../NuclearFusion/NuclearFusionFunc.H | 1 - 13 files changed, 313 insertions(+), 109 deletions(-) create mode 100755 Examples/Tests/collision/analysis_collision_1d.py create mode 100644 Examples/Tests/collision/inputs_1d create mode 100644 Regression/Checksum/benchmarks_json/collisionZ.json diff --git a/Examples/Tests/collision/analysis_collision_1d.py b/Examples/Tests/collision/analysis_collision_1d.py new file mode 100755 index 00000000000..7775a476dae --- /dev/null +++ b/Examples/Tests/collision/analysis_collision_1d.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Justin Angus +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from the script `inputs_1d`. +# run locally: python analysis_vandb_1d.py diags/diag1000600/ +# +# This is a 1D intra-species Coulomb scattering relaxation test consisting +# of a low-density population streaming into a higher density population at rest. +# Both populations belong to the same carbon12 ion species. +# See test T1b from JCP 413 (2020) by D. Higginson, et al. +# +import os +import sys + +import numpy as np +import yt +from scipy.constants import e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] +ds = yt.load(fn) +data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) + +# carbon 12 ion (mass = 12*amu - 6*me) +mass = 1.992100316897910e-26 + +# Separate macroparticles from group A (low weight) and group B (high weight) +# by sorting based on weight +sorted_indices = data['ions','particle_weight'].argsort() +sorted_wp = data['ions', 'particle_weight'][sorted_indices].value +sorted_px = data['ions', 'particle_momentum_x'][sorted_indices].value +sorted_py = data['ions', 'particle_momentum_y'][sorted_indices].value +sorted_pz = data['ions', 'particle_momentum_z'][sorted_indices].value + +# Find the index 'Npmin' that separates macroparticles from group A and group B +Np = len(sorted_wp) +wpmin = sorted_wp.min(); +wpmax = sorted_wp.max(); +for i in range(len(sorted_wp)): + if sorted_wp[i] > wpmin: + Npmin = i + break + +NpA = Npmin +wpA = wpmin; +NpB = Np - Npmin +wpB = wpmax; +NpAs = 0 +NpAe = Npmin +NpBs = Npmin +NpBe = Np + +############# + +sorted_px_sum = np.abs(sorted_px).sum(); +sorted_py_sum = np.abs(sorted_py).sum(); +sorted_pz_sum = np.abs(sorted_pz).sum(); +sorted_wp_sum = np.abs(sorted_wp).sum(); + +# compute mean velocities +wAtot = wpA*NpA +wBtot = wpB*NpB + +uBx = uBy = uBz = 0. +for i in range(NpBs,NpBe): + uBx += wpB*sorted_px[i] + uBy += wpB*sorted_py[i] + uBz += wpB*sorted_pz[i] +uBx /= (mass*wBtot) # [m/s] +uBy /= (mass*wBtot) # [m/s] +uBz /= (mass*wBtot) # [m/s] + +uAx = uAy = uAz = 0. +for i in range(NpAs,NpAe): + uAx += wpA*sorted_px[i] + uAy += wpA*sorted_py[i] + uAz += wpA*sorted_pz[i] +uAx /= (mass*wAtot) # [m/s] +uAy /= (mass*wAtot) # [m/s] +uAz /= (mass*wAtot) # [m/s] + +# compute temperatures +TBx = TBy = TBz = 0. +for i in range(NpBs,NpBe): + TBx += wpB*(sorted_px[i]/mass - uBx)**2 + TBy += wpB*(sorted_py[i]/mass - uBy)**2 + TBz += wpB*(sorted_pz[i]/mass - uBz)**2 +TBx *= mass/(e*wBtot) +TBy *= mass/(e*wBtot) +TBz *= mass/(e*wBtot) + +TAx = TAy = TAz = 0. +for i in range(NpAs,NpAe): + TAx += wpA*(sorted_px[i]/mass - uAx)**2 + TAy += wpA*(sorted_py[i]/mass - uAy)**2 + TAz += wpA*(sorted_pz[i]/mass - uAz)**2 +TAx *= mass/(e*wAtot) +TAy *= mass/(e*wAtot) +TAz *= mass/(e*wAtot) + +TApar = TAz +TAperp = (TAx + TAy)/2.0 +TA = (TAx + TAy + TAz)/3.0 + +TBpar = TBz +TBperp = (TBx + TBy)/2.0 +TB = (TBx + TBy + TBz)/3.0 + +TApar_30ps_soln = 6.15e3 # TA parallel solution at t = 30 ps +error = np.abs(TApar-TApar_30ps_soln)/TApar_30ps_soln +tolerance = 0.02 +print('TApar at 30ps error = ', error); +print('tolerance = ', tolerance); +assert error < tolerance + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/collision/inputs_1d b/Examples/Tests/collision/inputs_1d new file mode 100644 index 00000000000..b2de17192ae --- /dev/null +++ b/Examples/Tests/collision/inputs_1d @@ -0,0 +1,90 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.nA = 1.e25 # m^-3 +my_constants.NpA = 400 # m^-3 +my_constants.UA = 6.55e5 # m/s +my_constants.TA = 500. # eV +# +my_constants.nB = 1.e26 # m^-3 +my_constants.NpB = 400 # m^-3 +my_constants.UB = 0. # m/s +my_constants.TB = 500. # eV +# +my_constants.q_c12 = 6.*q_e +my_constants.m_c12 = 12.*m_u - 6.*m_e + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 600 +amr.n_cell = 180 +amr.max_level = 0 +amr.blocking_factor = 4 +geometry.dims = 1 +geometry.prob_lo = 0. +geometry.prob_hi = 0.01 + +################################# +###### Boundary Condition ####### +################################# +boundary.field_lo = periodic +boundary.field_hi = periodic + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = 0.05e-12 +warpx.use_filter = 0 + +# Do not evolve the E and B fields +algo.maxwell_solver = none + +# Order of particle shape factors +algo.particle_shape = 1 + +################################# +############ PLASMA ############# +################################# +particles.species_names = ions + +ions.charge = q_c12 +ions.mass = m_c12 +ions.do_not_deposit = 1 + +ions.injection_sources = groupA groupB + +ions.groupA.injection_style = "NUniformPerCell" +ions.groupA.num_particles_per_cell_each_dim = NpA +ions.groupA.profile = constant +ions.groupA.density = nA # number per m^3 +ions.groupA.momentum_distribution_type = "gaussian" +ions.groupA.uz_m = UA/clight +ions.groupA.ux_th = sqrt(TA*q_e/m_c12)/clight +ions.groupA.uy_th = sqrt(TA*q_e/m_c12)/clight +ions.groupA.uz_th = sqrt(TA*q_e/m_c12)/clight + +ions.groupB.injection_style = "NUniformPerCell" +ions.groupB.num_particles_per_cell_each_dim = NpB +ions.groupB.profile = constant +ions.groupB.density = nB # number per m^3 +ions.groupB.momentum_distribution_type = "gaussian" +ions.groupB.uz_m = UB/clight +ions.groupB.ux_th = sqrt(TB*q_e/m_c12)/clight +ions.groupB.uy_th = sqrt(TB*q_e/m_c12)/clight +ions.groupB.uz_th = sqrt(TB*q_e/m_c12)/clight + +################################# +############ COLLISION ########## +################################# +collisions.collision_names = collision1 +collision1.species = ions ions +collision1.CoulombLog = 10.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 600 +diag1.diag_type = Full diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json index a2fd6116cb8..350848d4aee 100644 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ b/Regression/Checksum/benchmarks_json/collisionISO.json @@ -11,12 +11,12 @@ "jz": 0.0 }, "electron": { - "particle_momentum_x": 3.579989064013309e-19, - "particle_momentum_y": 3.5822945977746767e-19, - "particle_momentum_z": 3.579753452653627e-19, - "particle_position_x": 1.0241322532163375, - "particle_position_y": 1.0238995904625479, - "particle_position_z": 1.02402135051502, + "particle_momentum_x": 3.5790777034053853e-19, + "particle_momentum_y": 3.5815348106229496e-19, + "particle_momentum_z": 3.577963316718249e-19, + "particle_position_x": 1.024180253191667, + "particle_position_y": 1.023919590453571, + "particle_position_z": 1.0240653505082926, "particle_weight": 714240000000.0 } } diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 2aa0e4686eb..c13f404857a 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,25 +6,25 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 381488.6528070591, - "T_ion": 320091.2785835478 + "T_electron": 381957.7394223898, + "T_ion": 319565.6269784763 }, "electron": { - "particle_momentum_x": 8.667025573698235e-19, - "particle_momentum_y": 8.457499789250831e-19, - "particle_momentum_z": 8.482438182280524e-19, - "particle_position_x": 21262567.138872623, - "particle_position_y": 21245135.070665065, - "particle_position_z": 21232644.283726066, + "particle_momentum_x": 8.631833485301232e-19, + "particle_momentum_y": 8.476316745254719e-19, + "particle_momentum_z": 8.514139891331418e-19, + "particle_position_x": 21253105.73686561, + "particle_position_y": 21282643.519070115, + "particle_position_z": 21239057.457948968, "particle_weight": 7.168263344048695e+28 }, "ion": { - "particle_momentum_x": 1.9300495097720012e-18, - "particle_momentum_y": 1.747257416857836e-18, - "particle_momentum_z": 1.7510296287537058e-18, - "particle_position_x": 21217348.883301035, - "particle_position_y": 21300859.0630925, - "particle_position_z": 21237901.246521123, + "particle_momentum_x": 1.9215585867122464e-18, + "particle_momentum_y": 1.7471481568315848e-18, + "particle_momentum_z": 1.7510887207292533e-18, + "particle_position_x": 21228900.948879313, + "particle_position_y": 21304564.011731848, + "particle_position_z": 21250585.221808463, "particle_weight": 7.168263344048695e+28 } } diff --git a/Regression/Checksum/benchmarks_json/collisionXZ.json b/Regression/Checksum/benchmarks_json/collisionXZ.json index 49138f0a1df..f90c34bc86d 100644 --- a/Regression/Checksum/benchmarks_json/collisionXZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXZ.json @@ -8,19 +8,19 @@ "Ez": 0.0 }, "ion": { - "particle_momentum_x": 2.458306853810186e-19, - "particle_momentum_y": 2.272685285153902e-19, - "particle_momentum_z": 2.281205462681013e-19, - "particle_position_x": 2645436.647039526, - "particle_position_y": 2672571.48688055, + "particle_momentum_x": 2.4932317055825563e-19, + "particle_momentum_y": 2.274916403334278e-19, + "particle_momentum_z": 2.2528161767665816e-19, + "particle_position_x": 2648815.601036139, + "particle_position_y": 2662836.7581390506, "particle_weight": 1.7256099431746894e+26 }, "electron": { - "particle_momentum_x": 1.0454942263455085e-19, - "particle_momentum_y": 1.0323735347957779e-19, - "particle_momentum_z": 1.0199134968670343e-19, - "particle_position_x": 2681776.3648108337, - "particle_position_y": 2663907.8843079703, + "particle_momentum_x": 1.0489203687862582e-19, + "particle_momentum_y": 1.0209657029567292e-19, + "particle_momentum_z": 1.0248962872393911e-19, + "particle_position_x": 2657004.8285825616, + "particle_position_y": 2670174.272797987, "particle_weight": 1.7256099431746894e+26 } } diff --git a/Regression/Checksum/benchmarks_json/collisionZ.json b/Regression/Checksum/benchmarks_json/collisionZ.json new file mode 100644 index 00000000000..3be8d5ae893 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/collisionZ.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "ions": { + "particle_momentum_x": 3.425400072687143e-16, + "particle_momentum_y": 3.421937133999805e-16, + "particle_momentum_z": 5.522701882677923e-16, + "particle_position_x": 720.0011611411148, + "particle_weight": 1.0999999999999999e+24 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 17ecfd64f29..c1a6316e7e2 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -195,6 +195,20 @@ useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py +[collisionZ] +buildDir = . +inputFile = Examples/Tests/collision/inputs_1d +runtime_params = +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +analysisRoutine = Examples/Tests/collision/analysis_collision_1d.py + [collisionISO] buildDir = . inputFile = Examples/Tests/collision/inputs_3d_isotropization diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 0ba49e9b09f..6f8f61e66b1 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -363,17 +363,14 @@ public: // create vectors to store density and temperature on cell level amrex::Gpu::DeviceVector n1_vec; - amrex::Gpu::DeviceVector n12_vec; amrex::Gpu::DeviceVector T1_vec; if (binary_collision_functor.m_computeSpeciesDensities) { n1_vec.resize(n_cells); - n12_vec.resize(n_cells); } if (binary_collision_functor.m_computeSpeciesTemperatures) { T1_vec.resize(n_cells); } amrex::ParticleReal* AMREX_RESTRICT n1_in_each_cell = n1_vec.dataPtr(); - amrex::ParticleReal* AMREX_RESTRICT n12_in_each_cell = n12_vec.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT T1_in_each_cell = T1_vec.dataPtr(); // Loop over cells @@ -384,7 +381,6 @@ public: // given by the `indices_1[cell_start_1:cell_stop_1]` index_type const cell_start_1 = cell_offsets_1[i_cell]; index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; - index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; // Do not collide if there is only one particle in the cell if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } @@ -415,29 +411,6 @@ public: // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); - - // compute n12 for intra-species - if (binary_collision_functor.m_computeSpeciesDensities) { - amrex::ParticleReal n12 = 0.0; - index_type const NI1 = cell_half_1 - cell_start_1; - index_type const NI2 = cell_stop_1 - cell_half_1; - index_type const max_N = amrex::max(NI1,NI2); - index_type i1 = cell_start_1; - index_type i2 = cell_half_1; - amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; - for (index_type k=0; k n1_vec, n2_vec; - amrex::Gpu::DeviceVector n12_vec; amrex::Gpu::DeviceVector T1_vec, T2_vec; if (binary_collision_functor.m_computeSpeciesDensities) { n1_vec.resize(n_cells); n2_vec.resize(n_cells); - n12_vec.resize(n_cells); } if (binary_collision_functor.m_computeSpeciesTemperatures) { T1_vec.resize(n_cells); @@ -655,7 +624,6 @@ public: } amrex::ParticleReal* AMREX_RESTRICT n1_in_each_cell = n1_vec.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT n2_in_each_cell = n2_vec.dataPtr(); - amrex::ParticleReal* AMREX_RESTRICT n12_in_each_cell = n12_vec.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT T1_in_each_cell = T1_vec.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT T2_in_each_cell = T2_vec.dataPtr(); @@ -719,29 +687,6 @@ public: // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); - - // compute n12 for inter-species - if (binary_collision_functor.m_computeSpeciesDensities) { - amrex::ParticleReal n12 = 0.0; - index_type const NI1 = cell_stop_1 - cell_start_1; - index_type const NI2 = cell_stop_2 - cell_start_2; - index_type const max_N = amrex::max(NI1,NI2); - index_type i1 = cell_start_1; - index_type i2 = cell_start_2; - amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; - amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; - for (index_type k=0; k( 1.0/std::cbrt(4.0*MathConst::pi/3.0*maxn) ); + + // bmax (screening length) cannot be smaller than atomic spacing + const T_PR bmax = amrex::max(lmdD, rmin); #if (defined WARPX_DIM_RZ) T_PR * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; @@ -108,12 +110,23 @@ void ElasticCollisionPerez ( u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); #endif + // Compute the effective density n12 used to compute the normalized + // scattering path s12 in UpdateMomentumPerezElastic(). + // s12 is defined such that the expected value of the change in particle + // velocity is equal to that from the full NxN pairing method, as described + // here https://arxiv.org/submit/5758216/view. This method is a direct extension + // of the original method by Takizuka and Abe JCP 25 (1977) to weighted particles. + T_PR n12; + const T_PR wpmax = amrex::max(w1[ I1[i1] ],w2[ I2[i2] ]); + if (isSameSpecies) { n12 = wpmax*static_cast(min_N+max_N-1)/dV; } + else { n12 = wpmax*static_cast(min_N)/dV; } + UpdateMomentumPerezElastic( u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], n1, n2, n12, q1, m1, w1[ I1[i1] ], q2, m2, w2[ I2[i2] ], - dt, L, lmdD, + dt, L, bmax, engine); #if (defined WARPX_DIM_RZ) diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index cd3cd5a5c83..3322c19ed2d 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -91,11 +91,10 @@ public: const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const n1, amrex::ParticleReal const n2, - amrex::ParticleReal const n12, amrex::ParticleReal const T1, amrex::ParticleReal const T2, amrex::ParticleReal const q1, amrex::ParticleReal const q2, amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const /*dV*/, index_type coll_idx, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, index_type const /*cell_start_pair*/, index_type* /*p_mask*/, index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, amrex::ParticleReal* /*p_pair_reaction_weight*/, @@ -105,9 +104,9 @@ public: ElasticCollisionPerez( I1s, I1e, I2s, I2e, I1, I2, - soa_1, soa_2, n1, n2, n12, T1, T2, + soa_1, soa_2, n1, n2, T1, T2, q1, q2, m1, m2, - dt, m_CoulombLog, engine, coll_idx); + dt, m_CoulombLog, dV, engine, m_isSameSpecies, coll_idx); } amrex::ParticleReal m_CoulombLog; diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H b/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H index 3101047e211..93acc3d8aef 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H @@ -18,9 +18,10 @@ /* \brief Update particle velocities according to * F. Perez et al., Phys.Plasmas.19.083104 (2012), * which is based on Nanbu's method, PhysRevE.55.4642 (1997). - * @param[in] LmdD is max(Debye length, minimal interparticle distance). + * @param[in] bmax is max(Debye length, minimal interparticle distance). * @param[in] L is the Coulomb log. A fixed L will be used if L > 0, * otherwise L will be calculated based on the algorithm. + * @param[in] n12 = max(w1,w2)*min(N1,N2)/dV is the effective density used for s12 * To see if there are nan or inf updated velocities, * compile with USE_ASSERTION=TRUE. * @@ -36,7 +37,7 @@ void UpdateMomentumPerezElastic ( T_PR const n1, T_PR const n2, T_PR const n12, T_PR const q1, T_PR const m1, T_PR const w1, T_PR const q2, T_PR const m2, T_PR const w2, - T_R const dt, T_PR const L, T_PR const lmdD, + T_R const dt, T_PR const L, T_PR const bmax, amrex::RandomEngine const& engine) { @@ -128,13 +129,13 @@ void UpdateMomentumPerezElastic ( // Compute the Coulomb log lnLmd lnLmd = amrex::max( T_PR(2.0), - T_PR(0.5)*std::log(T_PR(1.0)+lmdD*lmdD/(bmin*bmin)) ); + T_PR(0.5)*std::log(T_PR(1.0) + bmax*bmax/(bmin*bmin)) ); } // Compute s const auto tts = m1*g1s*m2*g2s/(inv_c2*p1sm*p1sm) + T_PR(1.0); const auto tts2 = tts*tts; - s = n1*n2/n12 * dt*lnLmd*q1*q1*q2*q2 / + s = n12 * dt*lnLmd*q1*q1*q2*q2 / ( T_PR(4.0) * MathConst::pi * PhysConst::ep0 * PhysConst::ep0 * m1*g1*m2*g2/(inv_c2*inv_c2) ) * gc*p1sm/mass_g * tts2; @@ -144,7 +145,7 @@ void UpdateMomentumPerezElastic ( const auto coeff = static_cast( std::pow(4.0*MathConst::pi/3.0,1.0/3.0)); T_PR const vrel = mass_g*p1sm/(m1*g1s*m2*g2s*gc); - T_PR const sp = coeff * n1*n2/n12 * dt * vrel * (m1+m2) / + T_PR const sp = coeff * n12 * dt * vrel * (m1+m2) / amrex::max( m1*cbrt_n1*cbrt_n1, m2*cbrt_n2*cbrt_n2); diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H index 152dca4e1a1..30b466e2ec2 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -97,7 +97,6 @@ public: const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const /*n1*/, amrex::ParticleReal const /*n2*/, - amrex::ParticleReal const /*n12*/, amrex::ParticleReal const /*T1*/, amrex::ParticleReal const /*T2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index 1f3e5e56582..3113dc69839 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -132,7 +132,6 @@ public: const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const /*n1*/, amrex::ParticleReal const /*n2*/, - amrex::ParticleReal const /*n12*/, amrex::ParticleReal const /*T1*/, amrex::ParticleReal const /*T2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, From e26f7ef0d42903a96cb67c3bf1fbe56af88712c1 Mon Sep 17 00:00:00 2001 From: Thomas Marks Date: Tue, 30 Jul 2024 02:34:10 +0300 Subject: [PATCH 72/87] Fix FFT python interface, take 2 (#5081) --- Python/pywarpx/picmi.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 42c0313a6e9..d06fe90d2d6 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1351,6 +1351,9 @@ def init(self, kw): def solver_initialize_inputs(self): + # Open BC means FieldBoundaryType::Open for electrostatic sims, rather than perfectly-matched layer + BC_map['open'] = 'open' + self.grid.grid_initialize_inputs() if self.relativistic: @@ -1371,6 +1374,8 @@ def solver_initialize_inputs(self): pywarpx.boundary.potential_hi_y = self.grid.potential_ymax pywarpx.boundary.potential_hi_z = self.grid.potential_zmax + pywarpx.warpx.poisson_solver = self.method + class GaussianLaser(picmistandard.PICMI_GaussianLaser): def laser_initialize_inputs(self): From 24c0711fc92577b3ed8c59af316c78d76894e10f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 29 Jul 2024 18:35:13 -0500 Subject: [PATCH 73/87] AMReX/pyAMReX/PICSAR: Weekly Update (#5095) * AMReX: Weekly Update * pyAMReX: Weekly Update * Increase Clang-Tidy `timeout-minutes` * Cast long long to int when using IParser --------- Co-authored-by: Edoardo Zoni --- .github/workflows/clang_tidy.yml | 1 + .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- Source/ablastr/utils/SignalHandling.cpp | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 8 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index 28b46b52530..af8632ed698 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -13,6 +13,7 @@ jobs: dim: [1, 2, RZ, 3] name: clang-tidy-${{ matrix.dim }}D runs-on: ubuntu-22.04 + timeout-minutes: 120 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index c2ec9aaa763..8f5e59ccaa4 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 0c3273f5e591815909180f8ffaf5b793cabbf9bc && cd - + cd ../amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index a2447ac3cf5..68d59d850a0 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 0c3273f5e591815909180f8ffaf5b793cabbf9bc +branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index c1a6316e7e2..8e2f723b8c5 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 0c3273f5e591815909180f8ffaf5b793cabbf9bc +branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/Source/ablastr/utils/SignalHandling.cpp b/Source/ablastr/utils/SignalHandling.cpp index 4095b9207ce..5eeaeec259f 100644 --- a/Source/ablastr/utils/SignalHandling.cpp +++ b/Source/ablastr/utils/SignalHandling.cpp @@ -104,7 +104,7 @@ SignalHandling::parseSignalNameToNumber (const std::string &str) auto spf = signals_parser.compileHost<0>(); - const int sig = spf(); + const auto sig = int(spf()); ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(sig < NUM_SIGNALS, "Parsed signal value is outside the supported range of [1, 31]"); diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 025b4d3b6e0..f1e5b3f62e2 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "0c3273f5e591815909180f8ffaf5b793cabbf9bc" +set(WarpX_amrex_branch "20e6f2eadf0c297517588ba38973ec7c7084fa31" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index a4598953432..133747aaeae 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "ff4643869c63d4ee40a87054b901f61eefcb97a3" +set(WarpX_pyamrex_branch "e007e730d48cb5fdbe1e10462d7d0a14e2bc8f8e" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 1692f66263c..f56a957c17a 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 0c3273f5e591815909180f8ffaf5b793cabbf9bc && cd - +cd amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 3413568d0d372d508de3c56ffdebd25b884f8d97 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:36:10 -0700 Subject: [PATCH 74/87] CI: remove unused parameter `analysisOutputImage` (#5097) --- Regression/WarpX-tests.ini | 45 -------------------------------------- 1 file changed, 45 deletions(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 8e2f723b8c5..a1dc0a168f7 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -380,7 +380,6 @@ useOMP = 1 numthreads = 1 runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/dive_cleaning/analysis.py -analysisOutputImage = Comparison.png [dive_cleaning_3d] buildDir = . @@ -395,7 +394,6 @@ useOMP = 1 numthreads = 1 runtime_params = analysisRoutine = Examples/Tests/dive_cleaning/analysis.py -analysisOutputImage = Comparison.png [ElectrostaticSphere] buildDir = . @@ -946,7 +944,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_fluid_1D] buildDir = . @@ -961,7 +958,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_1d.py -analysisOutputImage = langmuir_fluid_multi_1d_analysis.png [Langmuir_fluid_RZ] buildDir = . @@ -976,7 +972,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_rz.py -analysisOutputImage = langmuir_fluid_rz_analysis.png [Langmuir_fluid_2D] buildDir = . @@ -991,7 +986,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_2d.py -analysisOutputImage = langmuir_fluid_multi_2d_analysis.png [Langmuir_fluid_multi] buildDir = . @@ -1006,7 +1000,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_3d.py -analysisOutputImage = langmuir_fluid_multi_analysis.png [Langmuir_multi_1d] buildDir = . @@ -1021,7 +1014,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_1d.py -analysisOutputImage = langmuir_multi_1d_analysis.png [Langmuir_multi_2d_MR] buildDir = . @@ -1036,7 +1028,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_anisotropic] buildDir = . @@ -1051,7 +1042,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_momentum_conserving] buildDir = . @@ -1066,7 +1056,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR_momentum_conserving.png [Langmuir_multi_2d_MR_psatd] buildDir = . @@ -1081,7 +1070,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR_psatd.png [Langmuir_multi_2d_nodal] buildDir = . @@ -1096,7 +1084,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd] buildDir = . @@ -1111,7 +1098,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction] buildDir = . @@ -1126,7 +1112,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction_nodal] buildDir = . @@ -1141,7 +1126,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_momentum_conserving] buildDir = . @@ -1156,7 +1140,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_multiJ] buildDir = . @@ -1171,7 +1154,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_psatd_multiJ.png [Langmuir_multi_2d_psatd_multiJ_nodal] buildDir = . @@ -1186,7 +1168,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_psatd_multiJ_nodal.png [Langmuir_multi_2d_psatd_nodal] buildDir = . @@ -1201,7 +1182,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition] buildDir = . @@ -1216,7 +1196,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4] buildDir = . @@ -1231,7 +1210,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_nodal] buildDir = . @@ -1246,7 +1224,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_nodal] buildDir = . @@ -1261,7 +1238,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd] buildDir = . @@ -1276,7 +1252,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_current_correction] buildDir = . @@ -1291,7 +1266,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_current_correction_nodal] buildDir = . @@ -1306,7 +1280,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_div_cleaning] buildDir = . @@ -1321,7 +1294,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_momentum_conserving] buildDir = . @@ -1336,7 +1308,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_multiJ] buildDir = . @@ -1351,7 +1322,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = Langmuir_multi_psatd_multiJ.png [Langmuir_multi_psatd_multiJ_nodal] buildDir = . @@ -1366,7 +1336,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = Langmuir_multi_psatd_multiJ_nodal.png [Langmuir_multi_psatd_nodal] buildDir = . @@ -1381,7 +1350,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_single_precision] buildDir = . @@ -1396,7 +1364,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_Vay_deposition] buildDir = . @@ -1411,7 +1378,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_Vay_deposition_nodal] buildDir = . @@ -1426,7 +1392,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_rz] buildDir = . @@ -1441,7 +1406,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd] @@ -1457,7 +1421,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_psatd_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_current_correction] @@ -1473,7 +1436,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_psatd_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_multiJ] @@ -1489,7 +1451,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_psatd_multiJ_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_single_precision] @@ -1505,7 +1466,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Larmor] buildDir = . @@ -1677,7 +1637,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/laser_injection/analysis_laser.py -analysisOutputImage = laser_analysis.png [LaserInjection_1d] buildDir = . @@ -3487,7 +3446,6 @@ useOMP = 1 numthreads = 1 runtime_params = analysisRoutine = Examples/Tests/relativistic_space_charge_initialization/analysis.py -analysisOutputImage = Comparison.png [RepellingParticles] buildDir = . @@ -3701,7 +3659,6 @@ useOMP = 1 numthreads = 1 runtime_params = analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py -analysisOutputImage = Comparison.png [space_charge_initialization_2d] buildDir = . @@ -3716,7 +3673,6 @@ useOMP = 1 numthreads = 1 runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py -analysisOutputImage = Comparison.png [subcyclingMR] buildDir = . @@ -3848,7 +3804,6 @@ useOMP = 1 numthreads = 1 outputFile = spacecraft_charging_plt analysisRoutine = Examples/Physics_applications/spacecraft_charging/analysis.py -analysisOutputImage = min_phi_analysis.png [Point_of_contact_EB_3d] buildDir = . From 4ac5962ef4fce6e4f812ab64bed2f584ea412e91 Mon Sep 17 00:00:00 2001 From: Olga Shapoval <30510597+oshapoval@users.noreply.github.com> Date: Mon, 29 Jul 2024 17:07:18 -0700 Subject: [PATCH 75/87] Fix bug with ES solver and MR: `E_aux=E_fp` in `UpdateAuxilaryData` (#4922) * Removal of asserttion which prevented from usung the averaged PSATD algorithms with PML BC * Clean-up * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fixed to arr_aux(j,k,l) = fine when ES solve is used * Removed temporary print statements * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Clean-up * Added CI test ElectrostaticSphereEB_RZ_MR_lev_1 to check the fields on the level=1 * Updated becnmarks for ElectrostaticSphereLabFrame_MR_emass_10 * Imported regular expression (re) in the analysis script. * Fixed typo * United two CI tests for different levels of MR in one test. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated CI test ElectrostaticSphereEB_RZ_MR and the corresponding analysis script with smaller MR patch. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Clean-up * Do deepcopy for lev>0 and collocated grid * Fix bugs to resolve failure of CI tests * Preserve plotfile output, update benchmark file * Working on CI test * Update benchmark of `ElectrostaticSphereEB_RZ_MR` * Initialize with value all `aux`, `cax` fields * Remove changes related to averaged Galilean PSATD with PML * Remove style changes (e.g., changes to empty lines) * Replace `MFIter`/`ParallelFor` loop with simple copy * Apply suggestions from code review * Revert part of the code to its previous, equivalent state * Removed DeepCopy & no need for ghost cells in temp phi_cp. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Source/ablastr/fields/PoissonSolver.H * Add inline comments * Update analysis script * Use `TilingIfNotGPU`, `growntilebox` in E loop --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edoardo Zoni Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Co-authored-by: Remi Lehe Co-authored-by: Weiqun Zhang --- .../electrostatic_sphere_eb/analysis_rz_mr.py | 99 ++++ .../electrostatic_sphere_eb/inputs_rz_mr | 1 + .../ElectrostaticSphereEB_RZ_MR.json | 8 +- ...ectrostaticSphereLabFrame_MR_emass_10.json | 12 +- Regression/WarpX-tests.ini | 5 +- Source/Parallelization/WarpXComm.cpp | 510 ++++++++++-------- Source/Parallelization/WarpXComm_K.H | 90 +++- Source/ablastr/fields/PoissonSolver.H | 21 +- 8 files changed, 508 insertions(+), 238 deletions(-) create mode 100755 Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py diff --git a/Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py b/Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py new file mode 100755 index 00000000000..0b01b128362 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +# Copyright 2024 Olga Shapoval, Edoardo Zoni +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the embedded boundary in RZ. +# A cylindrical surface (r=0.1) has a fixed potential 1 V. +# The outer surface has 0 V fixed. +# Thus the analytical solution has the form: +# phi(r) = A+B*log(r), Er(r) = -B/r. + +import os +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +tolerance = 0.004 +print(f'tolerance = {tolerance}') + +fn = sys.argv[1] + +def find_first_non_zero_from_bottom_left(matrix): + for i in range(matrix.shape[0]): + for j in range(matrix.shape[1]): + if (matrix[i][j] != 0) and (matrix[i][j] != np.nan): + return (i, j) + return i, j + +def find_first_non_zero_from_upper_right(matrix): + for i in range(matrix.shape[0]-1, -1, -1): + for j in range(matrix.shape[1]-1, -1, -1): + if (matrix[i][j] != 0) and (matrix[i][j] != np.nan): + return (i, j) + return i,j + +def get_fields(ts, level): + if level == 0: + Er, info = ts.get_field('E', 'r', iteration=0) + phi, info = ts.get_field('phi', iteration=0) + else: + Er, info = ts.get_field(f'E_lvl{level}', 'r', iteration=0) + phi, info = ts.get_field(f'phi_lvl{level}', iteration=0) + return Er, phi, info + +def get_error_per_lev(ts,level): + Er, phi, info = get_fields(ts, level) + + nr_half = info.r.shape[0] // 2 + dr = info.dr + + Er_patch = Er[:,nr_half:] + phi_patch = phi[:,nr_half:] + r1 = info.r[nr_half:] + patch_left_lower_i, patch_left_lower_j = find_first_non_zero_from_bottom_left(Er_patch) + patch_right_upper_i, patch_right_upper_j = find_first_non_zero_from_upper_right(Er_patch) + + # phi and Er field on the MR patch + phi_sim = phi_patch[patch_left_lower_i:patch_right_upper_i+1, patch_left_lower_j:patch_right_upper_j+1] + Er_sim = Er_patch[patch_left_lower_i:patch_right_upper_i+1, patch_left_lower_j:patch_right_upper_j+1] + r = r1[patch_left_lower_j:patch_right_upper_j+1] + + B = 1.0/np.log(0.1/0.5) + A = -B*np.log(0.5) + + # outside EB and last cutcell + rmin = np.min(np.argwhere(r >= (0.1+dr))) + rmax = -1 + r = r[rmin:rmax] + phi_sim = phi_sim[:,rmin:rmax] + Er_sim = Er_sim[:,rmin:rmax] + + phi_theory = A + B*np.log(r) + phi_theory = np.tile(phi_theory, (phi_sim.shape[0],1)) + phi_error = np.max(np.abs(phi_theory-phi_sim) / np.abs(phi_theory)) + + Er_theory = -B/r + Er_theory = np.tile(Er_theory, (Er_sim.shape[0],1)) + Er_error = np.max(np.abs(Er_theory-Er_sim) / np.abs(Er_theory)) + + print(f'max error of phi[lev={level}]: {phi_error}') + print(f'max error of Er[lev={level}]: {Er_error}') + assert(phi_error < tolerance) + assert(Er_error < tolerance) + +ts = OpenPMDTimeSeries(fn) +level_fields = [field for field in ts.avail_fields if 'lvl' in field] +nlevels = 0 if level_fields == [] else int(level_fields[-1][-1]) +for level in range(nlevels+1): + get_error_per_lev(ts,level) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn, output_format="openpmd") diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr b/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr index 3bea63d76fb..722fc916416 100644 --- a/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr +++ b/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr @@ -30,3 +30,4 @@ diagnostics.diags_names = diag1 diag1.intervals = 1 diag1.diag_type = Full diag1.fields_to_plot = Er phi +diag1.format = openpmd diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json index ffa8d68f9d9..6bbfce0e3b3 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json @@ -1,10 +1,10 @@ { "lev=0": { - "Er": 8487.661571739109, - "phi": 2036.0428085225362 + "Er": 16975.32314347822, + "phi": 4072.085617045073 }, "lev=1": { - "Er": 19519.172334977942, - "phi": 3291.0262856782897 + "Er": 26818.189739547757, + "phi_lvl1": 8731.176548788893 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json index 024127a1bd2..21d5208c59a 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json @@ -6,15 +6,15 @@ "rho": 0.0 }, "lev=1": { - "Ex": 14.281015560380963, - "Ey": 14.281015560380965, - "Ez": 14.281015560380965, + "Ex": 7.170105936287823, + "Ey": 7.17010593628782, + "Ez": 7.170105936287821, "rho": 2.6092568008333786e-10 }, "electron": { - "particle_momentum_x": 1.80842228672388e-24, - "particle_momentum_y": 1.8084222867238806e-24, - "particle_momentum_z": 1.7598771525647628e-24, + "particle_momentum_x": 9.257577597262615e-25, + "particle_momentum_y": 9.257577597262618e-25, + "particle_momentum_z": 9.257577597262624e-25, "particle_position_x": 327.46875, "particle_position_y": 327.46875, "particle_position_z": 327.46875, diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index a1dc0a168f7..8048318de7a 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -483,7 +483,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereEB_RZ_MR] buildDir = . inputFile = Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr -runtime_params = warpx.abort_on_warning_threshold = medium +runtime_params = warpx.abort_on_warning_threshold = medium amr.ref_ratio_vect = 2 2 2 dim = 2 addToCompileString = USE_EB=TRUE USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON @@ -492,7 +492,8 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py +outputFile = ElectrostaticSphereEB_RZ_MR_plt +analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py [ElectrostaticSphereLabFrame] buildDir = . diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index e7df489236e..2887bd4d056 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -171,136 +171,191 @@ WarpX::UpdateAuxilaryDataStagToNodal () // Bfield { - Array,3> Btmp; - if (Bfield_cax[lev][0]) { - for (int i = 0; i < 3; ++i) { - Btmp[i] = std::make_unique( - *Bfield_cax[lev][i], amrex::make_alias, 0, 1); + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { + Array,3> Btmp; + if (Bfield_cax[lev][0]) { + for (int i = 0; i < 3; ++i) { + Btmp[i] = std::make_unique( + *Bfield_cax[lev][i], amrex::make_alias, 0, 1); + } + } else { + const IntVect ngtmp = Bfield_aux[lev-1][0]->nGrowVect(); + for (int i = 0; i < 3; ++i) { + Btmp[i] = std::make_unique(cnba, dm, 1, ngtmp); + } } - } else { - const IntVect ngtmp = Bfield_aux[lev-1][0]->nGrowVect(); + Btmp[0]->setVal(0.0); + Btmp[1]->setVal(0.0); + Btmp[2]->setVal(0.0); + // ParallelCopy from coarse level for (int i = 0; i < 3; ++i) { - Btmp[i] = std::make_unique(cnba, dm, 1, ngtmp); + const IntVect ng = Btmp[i]->nGrowVect(); + // Guard cells may not be up to date beyond ng_FieldGather + const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + // Copy Bfield_aux to Btmp, using up to ng_src (=ng_FieldGather) guard cells from + // Bfield_aux and filling up to ng (=nGrow) guard cells in Btmp + ablastr::utils::communication::ParallelCopy(*Btmp[i], *Bfield_aux[lev - 1][i], 0, 0, 1, + ng_src, ng, WarpX::do_single_precision_comms, cperiod); } - } - Btmp[0]->setVal(0.0); - Btmp[1]->setVal(0.0); - Btmp[2]->setVal(0.0); - // ParallelCopy from coarse level - for (int i = 0; i < 3; ++i) { - const IntVect ng = Btmp[i]->nGrowVect(); - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; - // Copy Bfield_aux to Btmp, using up to ng_src (=ng_FieldGather) guard cells from - // Bfield_aux and filling up to ng (=nGrow) guard cells in Btmp - ablastr::utils::communication::ParallelCopy(*Btmp[i], *Bfield_aux[lev - 1][i], 0, 0, 1, - ng_src, ng, WarpX::do_single_precision_comms, cperiod); - } - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); - const amrex::IntVect& Bx_cp_stag = Bfield_cp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& By_cp_stag = Bfield_cp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_cp_stag = Bfield_cp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_cp_stag = Bfield_cp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_cp_stag = Bfield_cp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_cp_stag = Bfield_cp[lev][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Bfield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); - Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); - Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); - Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); - Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); - Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); - Array4 const& bx_cp = Bfield_cp[lev][0]->const_array(mfi); - Array4 const& by_cp = Bfield_cp[lev][1]->const_array(mfi); - Array4 const& bz_cp = Bfield_cp[lev][2]->const_array(mfi); - Array4 const& bx_c = Btmp[0]->const_array(mfi); - Array4 const& by_c = Btmp[1]->const_array(mfi); - Array4 const& bz_c = Btmp[2]->const_array(mfi); - - const Box& bx = mfi.growntilebox(); - amrex::ParallelFor(bx, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Bfield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, bx_aux, bx_fp, bx_cp, bx_c, Bx_fp_stag, Bx_cp_stag, refinement_ratio); - warpx_interp(j, k, l, by_aux, by_fp, by_cp, by_c, By_fp_stag, By_cp_stag, refinement_ratio); - warpx_interp(j, k, l, bz_aux, bz_fp, bz_cp, bz_c, Bz_fp_stag, Bz_cp_stag, refinement_ratio); - }); + Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); + Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); + Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); + Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); + Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); + Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); + Array4 const& bx_cp = Bfield_cp[lev][0]->const_array(mfi); + Array4 const& by_cp = Bfield_cp[lev][1]->const_array(mfi); + Array4 const& bz_cp = Bfield_cp[lev][2]->const_array(mfi); + Array4 const& bx_c = Btmp[0]->const_array(mfi); + Array4 const& by_c = Btmp[1]->const_array(mfi); + Array4 const& bz_c = Btmp[2]->const_array(mfi); + + const Box& bx = mfi.growntilebox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bx_aux, bx_fp, bx_cp, bx_c, Bx_fp_stag, Bx_cp_stag, refinement_ratio); + warpx_interp(j, k, l, by_aux, by_fp, by_cp, by_c, By_fp_stag, By_cp_stag, refinement_ratio); + warpx_interp(j, k, l, bz_aux, bz_fp, bz_cp, bz_c, Bz_fp_stag, Bz_cp_stag, refinement_ratio); + }); + } + } + else { // electrostatic + const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*Bfield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); + Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); + Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); + Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); + Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); + Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); + + const Box& bx = mfi.growntilebox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bx_aux, bx_fp, Bx_fp_stag); + warpx_interp(j, k, l, by_aux, by_fp, By_fp_stag); + warpx_interp(j, k, l, bz_aux, bz_fp, Bz_fp_stag); + }); + } } } - // Efield { - Array,3> Etmp; - if (Efield_cax[lev][0]) { - for (int i = 0; i < 3; ++i) { - Etmp[i] = std::make_unique( - *Efield_cax[lev][i], amrex::make_alias, 0, 1); + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { + Array,3> Etmp; + if (Efield_cax[lev][0]) { + for (int i = 0; i < 3; ++i) { + Etmp[i] = std::make_unique( + *Efield_cax[lev][i], amrex::make_alias, 0, 1); + } + } else { + const IntVect ngtmp = Efield_aux[lev-1][0]->nGrowVect(); + for (int i = 0; i < 3; ++i) { + Etmp[i] = std::make_unique( + cnba, dm, 1, ngtmp); + } } - } else { - const IntVect ngtmp = Efield_aux[lev-1][0]->nGrowVect(); + Etmp[0]->setVal(0.0); + Etmp[1]->setVal(0.0); + Etmp[2]->setVal(0.0); + // ParallelCopy from coarse level for (int i = 0; i < 3; ++i) { - Etmp[i] = std::make_unique( - cnba, dm, 1, ngtmp); + const IntVect ng = Etmp[i]->nGrowVect(); + // Guard cells may not be up to date beyond ng_FieldGather + const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + // Copy Efield_aux to Etmp, using up to ng_src (=ng_FieldGather) guard cells from + // Efield_aux and filling up to ng (=nGrow) guard cells in Etmp + ablastr::utils::communication::ParallelCopy(*Etmp[i], *Efield_aux[lev - 1][i], 0, 0, 1, + ng_src, ng, WarpX::do_single_precision_comms, cperiod); } - } - Etmp[0]->setVal(0.0); - Etmp[1]->setVal(0.0); - Etmp[2]->setVal(0.0); - // ParallelCopy from coarse level - for (int i = 0; i < 3; ++i) { - const IntVect ng = Etmp[i]->nGrowVect(); - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; - // Copy Efield_aux to Etmp, using up to ng_src (=ng_FieldGather) guard cells from - // Efield_aux and filling up to ng (=nGrow) guard cells in Etmp - ablastr::utils::communication::ParallelCopy(*Etmp[i], *Efield_aux[lev - 1][i], 0, 0, 1, - ng_src, ng, WarpX::do_single_precision_comms, cperiod); - } - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); - const amrex::IntVect& Ex_cp_stag = Efield_cp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_cp_stag = Efield_cp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_cp_stag = Efield_cp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_cp_stag = Efield_cp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_cp_stag = Efield_cp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_cp_stag = Efield_cp[lev][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) - { - Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); - Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); - Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); - Array4 const& ex_cp = Efield_cp[lev][0]->const_array(mfi); - Array4 const& ey_cp = Efield_cp[lev][1]->const_array(mfi); - Array4 const& ez_cp = Efield_cp[lev][2]->const_array(mfi); - Array4 const& ex_c = Etmp[0]->const_array(mfi); - Array4 const& ey_c = Etmp[1]->const_array(mfi); - Array4 const& ez_c = Etmp[2]->const_array(mfi); - - const Box& bx = mfi.fabbox(); - amrex::ParallelFor(bx, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, ex_aux, ex_fp, ex_cp, ex_c, Ex_fp_stag, Ex_cp_stag, refinement_ratio); - warpx_interp(j, k, l, ey_aux, ey_fp, ey_cp, ey_c, Ey_fp_stag, Ey_cp_stag, refinement_ratio); - warpx_interp(j, k, l, ez_aux, ez_fp, ez_cp, ez_c, Ez_fp_stag, Ez_cp_stag, refinement_ratio); - }); + Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); + Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); + Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); + Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); + Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); + Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_cp = Efield_cp[lev][0]->const_array(mfi); + Array4 const& ey_cp = Efield_cp[lev][1]->const_array(mfi); + Array4 const& ez_cp = Efield_cp[lev][2]->const_array(mfi); + Array4 const& ex_c = Etmp[0]->const_array(mfi); + Array4 const& ey_c = Etmp[1]->const_array(mfi); + Array4 const& ez_c = Etmp[2]->const_array(mfi); + + const Box& bx = mfi.fabbox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ex_aux, ex_fp, ex_cp, ex_c, Ex_fp_stag, Ex_cp_stag, refinement_ratio); + warpx_interp(j, k, l, ey_aux, ey_fp, ey_cp, ey_c, Ey_fp_stag, Ey_cp_stag, refinement_ratio); + warpx_interp(j, k, l, ez_aux, ez_fp, ez_cp, ez_c, Ez_fp_stag, Ez_cp_stag, refinement_ratio); + }); + } + } + else { // electrostatic + const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*Efield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); + Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); + Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); + Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); + Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); + Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + + const Box& bx = mfi.growntilebox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ex_aux, ex_fp, Ex_fp_stag); + warpx_interp(j, k, l, ey_aux, ey_fp, Ey_fp_stag); + warpx_interp(j, k, l, ez_aux, ez_fp, Ez_fp_stag); + }); + } } } } @@ -341,141 +396,158 @@ WarpX::UpdateAuxilaryDataSameType () // B field { - MultiFab dBx(Bfield_cp[lev][0]->boxArray(), dm, Bfield_cp[lev][0]->nComp(), ng); - MultiFab dBy(Bfield_cp[lev][1]->boxArray(), dm, Bfield_cp[lev][1]->nComp(), ng); - MultiFab dBz(Bfield_cp[lev][2]->boxArray(), dm, Bfield_cp[lev][2]->nComp(), ng); - dBx.setVal(0.0); - dBy.setVal(0.0); - dBz.setVal(0.0); - - // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard - // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs - - ablastr::utils::communication::ParallelCopy(dBx, *Bfield_aux[lev - 1][0], 0, 0, - Bfield_aux[lev - 1][0]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dBy, *Bfield_aux[lev - 1][1], 0, 0, - Bfield_aux[lev - 1][1]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dBz, *Bfield_aux[lev - 1][2], 0, 0, - Bfield_aux[lev - 1][2]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, - crse_period); - - if (Bfield_cax[lev][0]) + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab::Copy(*Bfield_cax[lev][0], dBx, 0, 0, Bfield_cax[lev][0]->nComp(), ng); - MultiFab::Copy(*Bfield_cax[lev][1], dBy, 0, 0, Bfield_cax[lev][1]->nComp(), ng); - MultiFab::Copy(*Bfield_cax[lev][2], dBz, 0, 0, Bfield_cax[lev][2]->nComp(), ng); - } - MultiFab::Subtract(dBx, *Bfield_cp[lev][0], 0, 0, Bfield_cp[lev][0]->nComp(), ng); - MultiFab::Subtract(dBy, *Bfield_cp[lev][1], 0, 0, Bfield_cp[lev][1]->nComp(), ng); - MultiFab::Subtract(dBz, *Bfield_cp[lev][2], 0, 0, Bfield_cp[lev][2]->nComp(), ng); + MultiFab dBx(Bfield_cp[lev][0]->boxArray(), dm, Bfield_cp[lev][0]->nComp(), ng); + MultiFab dBy(Bfield_cp[lev][1]->boxArray(), dm, Bfield_cp[lev][1]->nComp(), ng); + MultiFab dBz(Bfield_cp[lev][2]->boxArray(), dm, Bfield_cp[lev][2]->nComp(), ng); + dBx.setVal(0.0); + dBy.setVal(0.0); + dBz.setVal(0.0); + + // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard + // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs + + ablastr::utils::communication::ParallelCopy(dBx, *Bfield_aux[lev - 1][0], 0, 0, + Bfield_aux[lev - 1][0]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dBy, *Bfield_aux[lev - 1][1], 0, 0, + Bfield_aux[lev - 1][1]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dBz, *Bfield_aux[lev - 1][2], 0, 0, + Bfield_aux[lev - 1][2]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, + crse_period); + + if (Bfield_cax[lev][0]) + { + MultiFab::Copy(*Bfield_cax[lev][0], dBx, 0, 0, Bfield_cax[lev][0]->nComp(), ng); + MultiFab::Copy(*Bfield_cax[lev][1], dBy, 0, 0, Bfield_cax[lev][1]->nComp(), ng); + MultiFab::Copy(*Bfield_cax[lev][2], dBz, 0, 0, Bfield_cax[lev][2]->nComp(), ng); + } + MultiFab::Subtract(dBx, *Bfield_cp[lev][0], 0, 0, Bfield_cp[lev][0]->nComp(), ng); + MultiFab::Subtract(dBy, *Bfield_cp[lev][1], 0, 0, Bfield_cp[lev][1]->nComp(), ng); + MultiFab::Subtract(dBz, *Bfield_cp[lev][2], 0, 0, Bfield_cp[lev][2]->nComp(), ng); - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Bx_stag = Bfield_aux[lev-1][0]->ixType().toIntVect(); - const amrex::IntVect& By_stag = Bfield_aux[lev-1][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_stag = Bfield_aux[lev-1][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_stag = Bfield_aux[lev-1][0]->ixType().toIntVect(); + const amrex::IntVect& By_stag = Bfield_aux[lev-1][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_stag = Bfield_aux[lev-1][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Bfield_aux[lev][0]); mfi.isValid(); ++mfi) - { - Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); - Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); - Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); - Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); - Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); - Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); - Array4 const& bx_c = dBx.const_array(mfi); - Array4 const& by_c = dBy.const_array(mfi); - Array4 const& bz_c = dBz.const_array(mfi); - - amrex::ParallelFor(Box(bx_aux), Box(by_aux), Box(bz_aux), - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, bx_aux, bx_fp, bx_c, Bx_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, by_aux, by_fp, by_c, By_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Bfield_aux[lev][0]); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, bz_aux, bz_fp, bz_c, Bz_stag, refinement_ratio); - }); + Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); + Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); + Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); + Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); + Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); + Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); + Array4 const& bx_c = dBx.const_array(mfi); + Array4 const& by_c = dBy.const_array(mfi); + Array4 const& bz_c = dBz.const_array(mfi); + + amrex::ParallelFor(Box(bx_aux), Box(by_aux), Box(bz_aux), + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bx_aux, bx_fp, bx_c, Bx_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, by_aux, by_fp, by_c, By_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bz_aux, bz_fp, bz_c, Bz_stag, refinement_ratio); + }); + } + } + else // electrostatic + { + MultiFab::Copy(*Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, 0, Bfield_aux[lev][0]->nComp(), Bfield_aux[lev][0]->nGrowVect()); + MultiFab::Copy(*Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, 0, Bfield_aux[lev][1]->nComp(), Bfield_aux[lev][1]->nGrowVect()); + MultiFab::Copy(*Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, 0, Bfield_aux[lev][2]->nComp(), Bfield_aux[lev][2]->nGrowVect()); } } - // E field { - MultiFab dEx(Efield_cp[lev][0]->boxArray(), dm, Efield_cp[lev][0]->nComp(), ng); - MultiFab dEy(Efield_cp[lev][1]->boxArray(), dm, Efield_cp[lev][1]->nComp(), ng); - MultiFab dEz(Efield_cp[lev][2]->boxArray(), dm, Efield_cp[lev][2]->nComp(), ng); - dEx.setVal(0.0); - dEy.setVal(0.0); - dEz.setVal(0.0); - - // Copy Efield_aux to the dE MultiFabs, using up to ng_src (=ng_FieldGather) guard - // cells from Efield_aux and filling up to ng (=nGrow) guard cells in the dE MultiFabs - ablastr::utils::communication::ParallelCopy(dEx, *Efield_aux[lev - 1][0], 0, 0, - Efield_aux[lev - 1][0]->nComp(), ng_src, ng, - WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dEy, *Efield_aux[lev - 1][1], 0, 0, - Efield_aux[lev - 1][1]->nComp(), ng_src, ng, - WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dEz, *Efield_aux[lev - 1][2], 0, 0, - Efield_aux[lev - 1][2]->nComp(), ng_src, ng, - WarpX::do_single_precision_comms, - crse_period); - - if (Efield_cax[lev][0]) + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab::Copy(*Efield_cax[lev][0], dEx, 0, 0, Efield_cax[lev][0]->nComp(), ng); - MultiFab::Copy(*Efield_cax[lev][1], dEy, 0, 0, Efield_cax[lev][1]->nComp(), ng); - MultiFab::Copy(*Efield_cax[lev][2], dEz, 0, 0, Efield_cax[lev][2]->nComp(), ng); - } - MultiFab::Subtract(dEx, *Efield_cp[lev][0], 0, 0, Efield_cp[lev][0]->nComp(), ng); - MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, Efield_cp[lev][1]->nComp(), ng); - MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, Efield_cp[lev][2]->nComp(), ng); + MultiFab dEx(Efield_cp[lev][0]->boxArray(), dm, Efield_cp[lev][0]->nComp(), ng); + MultiFab dEy(Efield_cp[lev][1]->boxArray(), dm, Efield_cp[lev][1]->nComp(), ng); + MultiFab dEz(Efield_cp[lev][2]->boxArray(), dm, Efield_cp[lev][2]->nComp(), ng); + dEx.setVal(0.0); + dEy.setVal(0.0); + dEz.setVal(0.0); + + // Copy Efield_aux to the dE MultiFabs, using up to ng_src (=ng_FieldGather) guard + // cells from Efield_aux and filling up to ng (=nGrow) guard cells in the dE MultiFabs + ablastr::utils::communication::ParallelCopy(dEx, *Efield_aux[lev - 1][0], 0, 0, + Efield_aux[lev - 1][0]->nComp(), ng_src, ng, + WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dEy, *Efield_aux[lev - 1][1], 0, 0, + Efield_aux[lev - 1][1]->nComp(), ng_src, ng, + WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dEz, *Efield_aux[lev - 1][2], 0, 0, + Efield_aux[lev - 1][2]->nComp(), ng_src, ng, + WarpX::do_single_precision_comms, + crse_period); + + if (Efield_cax[lev][0]) + { + MultiFab::Copy(*Efield_cax[lev][0], dEx, 0, 0, Efield_cax[lev][0]->nComp(), ng); + MultiFab::Copy(*Efield_cax[lev][1], dEy, 0, 0, Efield_cax[lev][1]->nComp(), ng); + MultiFab::Copy(*Efield_cax[lev][2], dEz, 0, 0, Efield_cax[lev][2]->nComp(), ng); + } + MultiFab::Subtract(dEx, *Efield_cp[lev][0], 0, 0, Efield_cp[lev][0]->nComp(), ng); + MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, Efield_cp[lev][1]->nComp(), ng); + MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, Efield_cp[lev][2]->nComp(), ng); - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Ex_stag = Efield_aux[lev-1][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_stag = Efield_aux[lev-1][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_stag = Efield_aux[lev-1][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_stag = Efield_aux[lev-1][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_stag = Efield_aux[lev-1][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_stag = Efield_aux[lev-1][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) - { - Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); - Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); - Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); - Array4 const& ex_c = dEx.const_array(mfi); - Array4 const& ey_c = dEy.const_array(mfi); - Array4 const& ez_c = dEz.const_array(mfi); - - amrex::ParallelFor(Box(ex_aux), Box(ey_aux), Box(ez_aux), - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, ex_aux, ex_fp, ex_c, Ex_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, ey_aux, ey_fp, ey_c, Ey_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, ez_aux, ez_fp, ez_c, Ez_stag, refinement_ratio); - }); + Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); + Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); + Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); + Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); + Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); + Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_c = dEx.const_array(mfi); + Array4 const& ey_c = dEy.const_array(mfi); + Array4 const& ez_c = dEz.const_array(mfi); + + amrex::ParallelFor(Box(ex_aux), Box(ey_aux), Box(ez_aux), + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ex_aux, ex_fp, ex_c, Ex_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ey_aux, ey_fp, ey_c, Ey_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ez_aux, ez_fp, ez_c, Ez_stag, refinement_ratio); + }); + } + } + else // electrostatic + { + MultiFab::Copy(*Efield_aux[lev][0], *Efield_fp[lev][0], 0, 0, Efield_aux[lev][0]->nComp(), Efield_aux[lev][0]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][1], *Efield_fp[lev][1], 0, 0, Efield_aux[lev][1]->nComp(), Efield_aux[lev][1]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][2], *Efield_fp[lev][2], 0, 0, Efield_aux[lev][2]->nComp(), Efield_aux[lev][2]->nGrowVect()); } } } diff --git a/Source/Parallelization/WarpXComm_K.H b/Source/Parallelization/WarpXComm_K.H index a2b8fe38ed4..c3362087ad9 100644 --- a/Source/Parallelization/WarpXComm_K.H +++ b/Source/Parallelization/WarpXComm_K.H @@ -12,7 +12,7 @@ /** * \brief Interpolation function called within WarpX::UpdateAuxilaryDataSameType - * to interpolate data from the coarse and fine grids to the fine aux grid, + * with electromagnetic solver to interpolate data from the coarse and fine grids to the fine aux grid, * assuming that all grids have the same staggering (either collocated or staggered). * * \param[in] j index along x of the output array @@ -285,6 +285,94 @@ void warpx_interp (int j, int k, int l, // Final result arr_aux(j,k,l) = tmp + (fine - coarse); } +/** + * \brief Interpolation function called within WarpX::UpdateAuxilaryDataStagToNodal + * to interpolate data from the coarse and fine grids to the fine aux grid, + * with momentum-conserving field gathering, hence between grids with different staggering, + * and assuming that the aux grid is collocated. + * + * \param[in] j index along x of the output array + * \param[in] k index along y (in 3D) or z (in 2D) of the output array + * \param[in] l index along z (in 3D, l=0 in 2D) of the output array + * \param[in,out] arr_aux output array where interpolated values are stored + * \param[in] arr_fine input fine-patch array storing the values to interpolate + * \param[in] arr_fine_stag IndexType of the fine-patch arrays + */ +AMREX_GPU_DEVICE AMREX_FORCE_INLINE +void warpx_interp (int j, int k, int l, + amrex::Array4 const& arr_aux, + amrex::Array4 const& arr_fine, + const amrex::IntVect& arr_fine_stag) +{ + using namespace amrex; + + // Pad input arrays with zeros beyond ghost cells + // for out-of-bound accesses due to large-stencil operations + const auto arr_fine_zeropad = [arr_fine] (const int jj, const int kk, const int ll) noexcept + { + return arr_fine.contains(jj,kk,ll) ? arr_fine(jj,kk,ll) : 0.0_rt; + }; + + // NOTE Indices (j,k,l) in the following refer to: + // - (z,-,-) in 1D + // - (x,z,-) in 2D + // - (r,z,-) in RZ + // - (x,y,z) in 3D + + // Staggering of fine array (0: cell-centered; 1: nodal) + const int sj_fp = arr_fine_stag[0]; +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const int sk_fp = arr_fine_stag[1]; +#elif defined(WARPX_DIM_3D) + const int sk_fp = arr_fine_stag[1]; + const int sl_fp = arr_fine_stag[2]; +#endif + + // Number of points used for interpolation from coarse grid to fine grid + int nj; + int nk; + int nl; + + amrex::Real fine = 0.0_rt; + + // 3) Interpolation from fine staggered to fine nodal + + nj = (sj_fp == 0) ? 2 : 1; +#if defined(WARPX_DIM_1D_Z) + nk = 1; + nl = 1; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + nk = (sk_fp == 0) ? 2 : 1; + nl = 1; +#else + nk = (sk_fp == 0) ? 2 : 1; + nl = (sl_fp == 0) ? 2 : 1; +#endif + + const int jm = (sj_fp == 0) ? j-1 : j; +#if defined(WARPX_DIM_1D_Z) + const int km = k; + const int lm = l; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const int km = (sk_fp == 0) ? k-1 : k; + const int lm = l; +#else + const int km = (sk_fp == 0) ? k-1 : k; + const int lm = (sl_fp == 0) ? l-1 : l; +#endif + + for (int jj = 0; jj < nj; jj++) { + for (int kk = 0; kk < nk; kk++) { + for (int ll = 0; ll < nl; ll++) { + fine += arr_fine_zeropad(jm+jj,km+kk,lm+ll); + } + } + } + fine = fine/static_cast(nj*nk*nl); + + // Final result + arr_aux(j,k,l) = fine; +} /** * \brief Arbitrary-order interpolation function used to center a given MultiFab between two grids diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index ca262981010..589c6ec1835 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -263,10 +263,15 @@ computePhi (amrex::Vector const & rho, mlmg.setVerbose(verbosity); mlmg.setMaxIter(max_iters); mlmg.setAlwaysUseBNorm(always_use_bnorm); + if (WarpX::grid_type == GridType::Collocated) { + // In this case, computeE needs to use ghost nodes data. So we + // ask MLMG to fill BC for us after it solves the problem. + mlmg.setFinalFillBC(true); + } // Solve Poisson equation at lev mlmg.solve( {phi[lev]}, {rho[lev]}, - relative_tolerance, absolute_tolerance ); + relative_tolerance, absolute_tolerance ); // needed for solving the levels by levels: // - coarser level is initial guess for finer level @@ -280,10 +285,14 @@ computePhi (amrex::Vector const & rho, const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; ba.coarsen(refratio); const int ncomp = linop.getNComp(); - amrex::MultiFab phi_cp(ba, phi[lev+1]->DistributionMap(), ncomp, 1); + const int ng = (WarpX::grid_type == GridType::Collocated) ? 1 : 0; + amrex::MultiFab phi_cp(ba, phi[lev+1]->DistributionMap(), ncomp, ng); + if (ng > 0) { + // Set all values outside the domain to zero + phi_cp.setDomainBndry(0.0_rt, geom[lev]); + } // Copy from phi[lev] to phi_cp (in parallel) - const amrex::IntVect& ng = amrex::IntVect::TheUnitVector(); const amrex::Periodicity& crse_period = geom[lev].periodicity(); ablastr::utils::communication::ParallelCopy( @@ -292,8 +301,8 @@ computePhi (amrex::Vector const & rho, 0, 0, 1, - ng, - ng, + amrex::IntVect(0), + amrex::IntVect(ng), do_single_precision_comms, crse_period ); @@ -308,7 +317,7 @@ computePhi (amrex::Vector const & rho, details::PoissonInterpCPtoFP const interp(phi_fp_arr, phi_cp_arr, refratio); - amrex::Box const b = mfi.tilebox(phi[lev + 1]->ixType().toIntVect()); + amrex::Box const& b = mfi.growntilebox(ng); amrex::ParallelFor(b, interp); } From c8a2479a8b4cb94e8be7e6d2c268ec9bd8581a0f Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 30 Jul 2024 06:57:53 -0700 Subject: [PATCH 76/87] Enable tiling in some `MFIter` loops (#5096) --- Source/Parallelization/WarpXComm.cpp | 4 ++-- Source/Utils/WarpXMovingWindow.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 2887bd4d056..6c44df061fd 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -306,7 +306,7 @@ WarpX::UpdateAuxilaryDataStagToNodal () #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) + for (MFIter mfi(*Efield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) { Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); @@ -321,7 +321,7 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& ey_c = Etmp[1]->const_array(mfi); Array4 const& ez_c = Etmp[2]->const_array(mfi); - const Box& bx = mfi.fabbox(); + const Box& bx = mfi.growntilebox(); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept { diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index d64394dafc9..73696838cd4 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -534,7 +534,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (amrex::MFIter mfi(tmpmf); mfi.isValid(); ++mfi ) + for (amrex::MFIter mfi(tmpmf, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { @@ -545,7 +545,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, auto const& dstfab = mf.array(mfi); auto const& srcfab = tmpmf.array(mfi); - const amrex::Box& outbox = mfi.fabbox() & adjBox; + const amrex::Box& outbox = mfi.growntilebox() & adjBox; if (outbox.ok()) { if (!useparser) { From 870577711ffe44bcba48c193af1f7246b98a4c21 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 30 Jul 2024 12:05:59 -0700 Subject: [PATCH 77/87] Add documentation for binomial smoother (#5099) --- Docs/source/usage/python.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 4d53b311764..c068447ddbc 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -53,6 +53,10 @@ Field solvers define the updates of electric and magnetic fields. .. autoclass:: pywarpx.picmi.ElectrostaticSolver +Object that allows smoothing of fields. + +.. autoclass:: pywarpx.picmi.BinomialSmoother + Constants --------- From 128ed9e9e5516a403245d8d6f4467edcfe9396e3 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 30 Jul 2024 17:35:43 -0700 Subject: [PATCH 78/87] Expose more diagnostics options in PICMI (#5100) --- Python/pywarpx/picmi.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index d06fe90d2d6..2aaa6b6122b 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2329,6 +2329,11 @@ class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): warpx_openpmd_backend: {bp, h5, json}, optional Openpmd backend file format + warpx_openpmd_encoding: 'v' (variable based), 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Variable based is an experimental feature with ADIOS2. Default: `'f'`. + warpx_file_prefix: string, optional Prefix on the diagnostic file name @@ -2338,6 +2343,9 @@ class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): warpx_dump_rz_modes: bool, optional Flag whether to dump the data for all RZ modes + warpx_dump_last_timestep: bool, optional + If true, the last timestep is dumped regardless of the diagnostic period/intervals. + warpx_particle_fields_to_plot: list of ParticleFieldDiagnostics List of ParticleFieldDiagnostic classes to install in the simulation. Error checking is handled in the class itself. @@ -2355,9 +2363,11 @@ def init(self, kw): self.plot_crsepatch = kw.pop('warpx_plot_crsepatch', None) self.format = kw.pop('warpx_format', 'plotfile') self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.dump_rz_modes = kw.pop('warpx_dump_rz_modes', None) + self.dump_last_timestep = kw.pop('warpx_dump_last_timestep', None) self.particle_fields_to_plot = kw.pop('warpx_particle_fields_to_plot', []) self.particle_fields_species = kw.pop('warpx_particle_fields_species', None) @@ -2368,8 +2378,10 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'Full' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.dump_rz_modes = self.dump_rz_modes + self.diagnostic.dump_last_timestep = self.dump_last_timestep self.diagnostic.intervals = self.period self.diagnostic.diag_lo = self.lower_bound self.diagnostic.diag_hi = self.upper_bound @@ -2533,6 +2545,11 @@ class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnostic warpx_openpmd_backend: {bp, h5, json}, optional Openpmd backend file format + warpx_openpmd_encoding: 'v' (variable based), 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Variable based is an experimental feature with ADIOS2. Default: `'f'`. + warpx_file_prefix: string, optional Prefix on the diagnostic file name @@ -2551,6 +2568,9 @@ class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnostic a dictionary is given the keys should be species with the value specifying the stride for that species. + warpx_dump_last_timestep: bool, optional + If true, the last timestep is dumped regardless of the diagnostic period/intervals. + warpx_plot_filter_function: string, optional Analytic expression to down select the particles to in the diagnostic """ @@ -2558,11 +2578,13 @@ def init(self, kw): self.format = kw.pop('warpx_format', 'plotfile') self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.random_fraction = kw.pop('warpx_random_fraction', None) self.uniform_stride = kw.pop('warpx_uniform_stride', None) self.plot_filter_function = kw.pop('warpx_plot_filter_function', None) + self.dump_last_timestep = kw.pop('warpx_dump_last_timestep', None) self.user_defined_kw = {} if self.plot_filter_function is not None: @@ -2582,7 +2604,9 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'Full' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits + self.diagnostic.dump_last_timestep = self.dump_last_timestep self.diagnostic.intervals = self.period self.diagnostic.set_or_replace_attr('write_species', True) if 'fields_to_plot' not in self.diagnostic.argvattrs: @@ -2699,6 +2723,11 @@ class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic, warpx_openpmd_backend: string, optional Passed to .openpmd_backend + warpx_openpmd_encoding: 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Default: `'f'`. + warpx_file_prefix: string, optional Passed to .file_prefix @@ -2723,6 +2752,7 @@ def init(self, kw): self.format = kw.pop('warpx_format', None) self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.intervals = kw.pop('warpx_intervals', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) @@ -2737,6 +2767,7 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'BackTransformed' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.diag_lo = self.lower_bound self.diagnostic.diag_hi = self.upper_bound @@ -2808,6 +2839,11 @@ class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, warpx_openpmd_backend: string, optional Passed to .openpmd_backend + warpx_openpmd_encoding: 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Default: `'f'`. + warpx_file_prefix: string, optional Passed to .file_prefix @@ -2824,6 +2860,7 @@ class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, def init(self, kw): self.format = kw.pop('warpx_format', None) self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.intervals = kw.pop('warpx_intervals', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) @@ -2836,6 +2873,7 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'BackTransformed' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.do_back_transformed_particles = True From f10aa7070a6725b9979284cbe888c90b1395e237 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 31 Jul 2024 09:48:01 -0500 Subject: [PATCH 79/87] Doc: HPSF & Gov Links (#5085) Add agreed upon HPSF boilerplate text in README and manual. Update acknowledegments. --- Docs/source/acknowledgements.rst | 9 +++++++-- README.md | 10 +++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Docs/source/acknowledgements.rst b/Docs/source/acknowledgements.rst index 14877839294..6d2529705d0 100644 --- a/Docs/source/acknowledgements.rst +++ b/Docs/source/acknowledgements.rst @@ -3,13 +3,18 @@ Funding and Acknowledgements ============================ -WarpX is supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration) responsible for the planning and preparation of a capable exascale ecosystem, including software, applications, hardware, advanced system engineering, and early testbed platforms, in support of the nation's exascale computing imperative. +WarpX is hosted by the High Performance Computing Foundation (HPSF). +If your organization wants to help steer the evolution of the HPC software ecosystem, visit `hpsf.io `__ and consider joining! WarpX is supported by the CAMPA collaboration, a project of the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research and Office of High Energy Physics, Scientific Discovery through Advanced Computing (SciDAC) program. -ABLASTR seed development is supported by the Laboratory Directed Research and Development Program of Lawrence Berkeley National Laboratory under U.S. Department of Energy Contract No. DE-AC02-05CH11231. +WarpX is supported by the KISMET project, a project of the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research and Office of Fusion Energy Science, Scientific Discovery through Advanced Computing (SciDAC) program. CEA-LIDYL actively contributes to the co-development of WarpX. As part of this initiative, WarpX also receives funding from the French National Research Agency (ANR - Plasm-On-Chip), the Horizon H2020 program and CEA. +WarpX was supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration) responsible for the planning and preparation of a capable exascale ecosystem, including software, applications, hardware, advanced system engineering, and early testbed platforms, in support of the nation's exascale computing imperative. + +ABLASTR seed development was supported by the Laboratory Directed Research and Development Program of Lawrence Berkeley National Laboratory under U.S. Department of Energy Contract No. DE-AC02-05CH11231. + We acknowledge all the contributors and users of the WarpX community who participate to the code quality with valuable code improvement and important feedback. diff --git a/README.md b/README.md index b2de2211b60..7760a004081 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Discussions](https://img.shields.io/badge/chat-discussions-turquoise.svg)](https://github.com/ECP-WarpX/WarpX/discussions) [![Supported Platforms](https://img.shields.io/badge/platforms-linux%20|%20osx%20|%20win-blue)](https://warpx.readthedocs.io/en/latest/install/users.html) [![GitHub commits since last release](https://img.shields.io/github/commits-since/ECP-WarpX/WarpX/latest/development.svg)](https://github.com/ECP-WarpX/WarpX/compare/development) -[![Exascale Computing Project](https://img.shields.io/badge/supported%20by-ECP-orange)](https://www.exascaleproject.org/research/) +[![HPSF](https://img.shields.io/badge/hosted%20by-HPSF-orange)](https://hpsf.io) [![Language: C++17](https://img.shields.io/badge/language-C%2B%2B17-orange.svg)](https://isocpp.org/) [![Language: Python](https://img.shields.io/badge/language-Python-orange.svg)](https://python.org/) [![License WarpX](https://img.shields.io/badge/license-BSD--3--Clause--LBNL-blue.svg)](https://spdx.org/licenses/BSD-3-Clause-LBNL.html) @@ -45,6 +45,14 @@ To contact the developers, feel free to open an issue on this repo, or visit our [![SENSEI](https://img.shields.io/static/v1?label="runs%20on"&message="SENSEI"&color="blueviolet")](https://sensei-insitu.org) Our workflow is described in [CONTRIBUTING.rst](CONTRIBUTING.rst). +We invite you to contribute to WarpX in any form following our [Code of Conduct](https://warpx.readthedocs.io/en/latest/coc.html), e.g., contribute to [discussions](https://github.com/ECP-WarpX/WarpX/discussions), help each other in [issues](https://github.com/ECP-WarpX/WarpX/issues), fix bugs, or add [documentation](https://warpx.readthedocs.io/en/latest/developers/documentation.html) and new functionality! + +## Governance + +WarpX is hosted by the High Performance Computing Foundation (HPSF). +If your organization wants to help steer the evolution of the HPC software ecosystem, visit [hpsf.io](https://hpsf.io) and consider joining! + +The WarpX open governance model is described in [GOVERNANCE.rst](GOVERNANCE.rst). ## Copyright Notice From 9cab6f98e67b2e286cfa2388bbb9bd23f19061af Mon Sep 17 00:00:00 2001 From: Jonathan Wang Date: Thu, 1 Aug 2024 14:29:12 -0400 Subject: [PATCH 80/87] Doc: Pitzer (OSC) (#5064) * add: profile, dependencies, and sbatch scripts for OSC pitzer * add: pitzer documentation page * fix: add pitzer name to manual sidebar * chore: update pitzer dependencies installation script filename * fix formatting issues in doc * separate cpu and gpu scripts * add boost support * add separate instructions for CPU and GPU dependencies * rename venv name * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add header to dependencies intalling scripts * chores for doc * fix: dependencies installing scripts typo * fix typos * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * chore: Update batch script and profile file names for Pitzer CPU and V100 * remove gpu allocation in CPU profile * Update Docs/source/install/hpc/pitzer.rst Co-authored-by: Axel Huebl * Update Docs/source/install/hpc.rst Co-authored-by: Axel Huebl * Update Docs/source/install/hpc/pitzer.rst Co-authored-by: Axel Huebl * use built-in cuda-aware MPI * Remove repetition of source profile --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- .gitignore | 2 +- Docs/source/install/hpc.rst | 3 +- Docs/source/install/hpc/pitzer.rst | 274 ++++++++++++++++++ .../pitzer-osc/install_cpu_dependencies.sh | 156 ++++++++++ .../pitzer-osc/install_v100_dependencies.sh | 156 ++++++++++ Tools/machines/pitzer-osc/pitzer_cpu.sbatch | 19 ++ .../pitzer_cpu_warpx.profile.example | 54 ++++ Tools/machines/pitzer-osc/pitzer_v100.sbatch | 26 ++ .../pitzer_v100_warpx.profile.example | 64 ++++ 9 files changed, 752 insertions(+), 2 deletions(-) create mode 100644 Docs/source/install/hpc/pitzer.rst create mode 100644 Tools/machines/pitzer-osc/install_cpu_dependencies.sh create mode 100644 Tools/machines/pitzer-osc/install_v100_dependencies.sh create mode 100644 Tools/machines/pitzer-osc/pitzer_cpu.sbatch create mode 100644 Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example create mode 100644 Tools/machines/pitzer-osc/pitzer_v100.sbatch create mode 100644 Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example diff --git a/.gitignore b/.gitignore index 4fa65351331..094fee87efc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ Python/pywarpx/libwarpx*.so d/ f/ o/ -build/ +build*/ tmp_build_dir/ test_dir test_dir/ diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst index 5efdeae39a8..5b388b9d0b2 100644 --- a/Docs/source/install/hpc.rst +++ b/Docs/source/install/hpc.rst @@ -36,6 +36,7 @@ This section documents quick-start guides for a selection of supercomputers that hpc/crusher hpc/frontier hpc/fugaku + hpc/greatlakes hpc/hpc3 hpc/juwels hpc/karolina @@ -46,12 +47,12 @@ This section documents quick-start guides for a selection of supercomputers that hpc/lxplus hpc/ookami hpc/perlmutter + hpc/pitzer hpc/polaris hpc/quartz hpc/spock hpc/summit hpc/taurus - hpc/greatlakes .. tip:: diff --git a/Docs/source/install/hpc/pitzer.rst b/Docs/source/install/hpc/pitzer.rst new file mode 100644 index 00000000000..4f1e53bc309 --- /dev/null +++ b/Docs/source/install/hpc/pitzer.rst @@ -0,0 +1,274 @@ +.. _building-pitzer: + +Pitzer (OSC) +============ + +The `Pitzer cluster `__ is located at the Ohio Supercomputer Center (OSC). It is currently the main CPU/GPU cluster at OSC. However, the `Cardinal cluster `__ is soon going to take over Pitzer to become the next major CPU/GPU cluster at OSC in the second half of 2024. A list of all OSC clusters can be found `here `__. + +The Pitzer cluster offers a variety of partitions suitable for different computational needs, including GPU nodes, CPU nodes, and nodes with large memory capacities. For more information on the specifications and capabilities of these partitions, visit the `Ohio Supercomputer Center's Pitzer page `__. + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `Pitzer user guide `__ +* Batch system: `Slurm `__ +* `Jupyter service `__ +* `Filesystems `__: + + * ``$HOME``: per-user directory, use only for inputs, source, and scripts; backed up (500GB) + * ``/fs/ess``: per-project storage directory, use for long-term storage of data and analysis; backed up (1-5TB) + * ``/fs/scratch``: per-project production directory; fast I/O for parallel jobs; not backed up (100TB) + +.. _building-pitzer-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + +On Pitzer, you can run either on GPU nodes with V100 GPUs or CPU nodes. + +.. tab-set:: + + .. tab-item:: V100 GPUs + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/pitzer_v100_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example $HOME/pitzer_v100_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``pas2024``, then run ``nano $HOME/pitzer_v100_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="pas2024" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to pitzer, activate these environment settings: + + .. code-block:: bash + + source $HOME/pitzer_v100_warpx.profile + + Finally, since pitzer does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/pitzer-osc/install_v100_dependencies.sh + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/install_v100_dependencies.sh + :language: bash + + + .. tab-item:: CPU Nodes + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/pitzer_cpu_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example $HOME/pitzer_cpu_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``pas2024``, then run ``nano $HOME/pitzer_cpu_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="pas2024" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to pitzer, activate these environment settings: + + .. code-block:: bash + + source $HOME/pitzer_cpu_warpx.profile + + Finally, since pitzer does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/pitzer-osc/install_cpu_dependencies.sh + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/install_cpu_dependencies.sh + :language: bash + + +.. _building-pitzer-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. tab-set:: + .. tab-item:: V100 GPUs + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100 + + cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100 -j 48 + + The WarpX application executables are now in ``$HOME/src/warpx/build_v100/bin/``. Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100_py + + cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100_py -j 48 --target pip_install + + .. tab-item:: CPU Nodes + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build + + cmake -S . -B build -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build -j 48 + + The WarpX application executables are now in ``$HOME/src/warpx/build/bin/``. Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_py + + cmake -S . -B build_py -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_py -j 48 --target pip_install + +Now, you can :ref:`submit Pitzer compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Pitzer jobs (:ref:`example inputs `). For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch``. + +.. _building-pitzer-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd $HOME/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the pitzer_cpu_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. + +.. _running-pitzer: + +Running +------- + +.. tab-set:: + + .. tab-item:: V100 GPUs + + Pitzer's GPU partition includes: + + - 32 nodes, each equipped with two V100 (16GB) GPUs. + - 42 nodes, each with two V100 (32GB) GPUs. + - 4 large memory nodes, each with quad V100 (32GB) GPUs. + + To run a WarpX simulation on the GPU nodes, use the batch script provided below. Adjust the ``-N`` parameter in the script to match the number of nodes you intend to use. Each node in this partition supports running one MPI rank per GPU. + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_v100.sbatch + :language: bash + :caption: Copy this file from ``$HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_v100.sbatch``. + + After preparing your script, submit your job with the following command: + + .. code-block:: bash + + sbatch pitzer_v100.sbatch + + .. tab-item:: CPU Nodes + + For CPU-based computations, Pitzer offers: + + - 224 nodes, each with dual Intel Xeon Gold 6148 CPUs and 192 GB RAM. + - 340 nodes, each with dual Intel Xeon Platinum 8268 CPUs and 192 GB RAM. + - 16 large memory nodes. + + To submit a job to the CPU partition, use the provided batch script. Ensure you have copied the script to your working directory. + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_cpu.sbatch + :language: bash + :caption: Copy this file from ``$HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_cpu.sbatch``. + + Submit your job with: + + .. code-block:: bash + + sbatch pitzer_cpu.sbatch + +.. _post-processing-osc: + +Post-Processing +--------------- + +For post-processing, many users prefer to use the online `Jupyter service `__ (`documentation `__) that is directly connected to the cluster's fast filesystem. + +.. note:: + + This section is a stub and contributions are welcome. + We can document further details, e.g., which recommended post-processing Python software to install or how to customize Jupyter kernels here. diff --git a/Tools/machines/pitzer-osc/install_cpu_dependencies.sh b/Tools/machines/pitzer-osc/install_cpu_dependencies.sh new file mode 100644 index 00000000000..f0b6ce4b950 --- /dev/null +++ b/Tools/machines/pitzer-osc/install_cpu_dependencies.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Zhongwei Wang +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + +# Check: ###################################################################### +# +# Was pitzer_cpu_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your pitzer_cpu_warpx.profile file! Please edit its line 2 to continue!" + exit 1 +fi + +# Remove old dependencies ##################################################### +# +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + +# General extra dependencies ################################################## +# +SRC_DIR="${HOME}/src" +build_dir=$(mktemp -d) + +# boost (for QED table generation support) +cd ${SRC_DIR} +wget https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz +tar -xzvf boost_1_82_0.tar.gz +rm -rf boost_1_82_0.tar.gz +cd - + +cd ${SRC_DIR}/boost_1_82_0 +./bootstrap.sh --prefix=$SW_DIR/boost-1.82.0 +./b2 install +cd - + +# BLAS++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/blaspp ]; then + cd ${SRC_DIR}/blaspp + git fetch --prune + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp +fi +rm -rf ${build_dir}/blaspp-pitzer-cpu-build +CXX=$(which CC) cmake -S ${SRC_DIR}/blaspp \ + -B ${build_dir}/blaspp-pitzer-cpu-build \ + -Duse_openmp=ON \ + -Dgpu_backend=OFF \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 +cmake --build ${build_dir}/blaspp-pitzer-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pitzer-cpu-build + +# LAPACK++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/lapackpp ]; then + cd ${SRC_DIR}/lapackpp + git fetch --prune + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp +fi +rm -rf ${build_dir}/lapackpp-pitzer-cpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp \ + -B ${build_dir}/lapackpp-pitzer-cpu-build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 +cmake --build ${build_dir}/lapackpp-pitzer-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pitzer-cpu-build + +# c-blosc (I/O compression, for openPMD) +if [ -d ${SRC_DIR}/c-blosc ]; then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.6 + cd - +else + git clone -b v1.21.6 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi +rm -rf ${build_dir}/c-blosc-pitzer-build +cmake -S ${SRC_DIR}/c-blosc \ + -B ${build_dir}/c-blosc-pitzer-build \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DDEACTIVATE_AVX2=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.6 +cmake --build ${build_dir}/c-blosc-pitzer-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pitzer-build + +# ADIOS2 (for openPMD) +if [ -d ${SRC_DIR}/adios2 ]; then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.10.1 + cd - +else + git clone -b v2.10.1 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +rm -rf ${build_dir}/adios2-pitzer-build +cmake -S ${SRC_DIR}/adios2 \ + -B ${build_dir}/adios2-pitzer-build \ + -DBUILD_TESTING=OFF \ + -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_USE_Blosc=ON \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_SST=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.1 +cmake --build ${build_dir}/adios2-pitzer-build --target install -j 16 +rm -rf ${build_dir}/adios2-pitzer-build + +rm -rf ${build_dir} + +# Python ###################################################################### +# +python3 -m pip install --upgrade --user virtualenv +rm -rf ${SW_DIR}/venvs/${VENV_NAME} +python3 -m venv ${SW_DIR}/venvs/${VENV_NAME} +source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip cache purge +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt + +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt + +# ML dependencies +python3 -m pip install --upgrade torch diff --git a/Tools/machines/pitzer-osc/install_v100_dependencies.sh b/Tools/machines/pitzer-osc/install_v100_dependencies.sh new file mode 100644 index 00000000000..5601b4d76c9 --- /dev/null +++ b/Tools/machines/pitzer-osc/install_v100_dependencies.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Zhongwei Wang +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + +# Check: ###################################################################### +# +# Was pitzer_v100_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your pitzer_v100_warpx.profile file! Please edit its line 2 to continue!" + exit 1 +fi + +# Remove old dependencies ##################################################### +# +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + +# General extra dependencies ################################################## +# +SRC_DIR="${HOME}/src" +build_dir=$(mktemp -d) + +# boost (for QED table generation support) +cd ${SRC_DIR} +wget https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz +tar -xzvf boost_1_82_0.tar.gz +rm -rf boost_1_82_0.tar.gz +cd - + +cd ${SRC_DIR}/boost_1_82_0 +./bootstrap.sh --prefix=${SW_DIR}/boost-1.82.0 +./b2 install +cd - + +# BLAS++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/blaspp ]; then + cd ${SRC_DIR}/blaspp + git fetch + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp +fi +rm -rf ${build_dir}/blaspp-pitzer-v100-build +CXX=$(which CC) cmake -S ${SRC_DIR}/blaspp \ + -B ${build_dir}/blaspp-pitzer-v100-build \ + -Duse_openmp=ON \ + -Dgpu_backend=cuda \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 +cmake --build ${build_dir}/blaspp-pitzer-v100-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pitzer-v100-build + +# LAPACK++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/lapackpp ]; then + cd ${SRC_DIR}/lapackpp + git fetch + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp +fi +rm -rf ${build_dir}/lapackpp-pitzer-v100-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp \ + -B ${build_dir}/lapackpp-pitzer-v100-build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 +cmake --build ${build_dir}/lapackpp-pitzer-v100-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pitzer-v100-build + +# c-blosc (I/O compression, for openPMD) +if [ -d ${SRC_DIR}/c-blosc ]; then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.6 + cd - +else + git clone -b v1.21.6 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi +rm -rf ${build_dir}/c-blosc-pitzer-build +cmake -S ${SRC_DIR}/c-blosc \ + -B ${build_dir}/c-blosc-pitzer-build \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DDEACTIVATE_AVX2=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.6 +cmake --build ${build_dir}/c-blosc-pitzer-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pitzer-build + +# ADIOS2 (for openPMD) +if [ -d ${SRC_DIR}/adios2 ]; then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.10.1 + cd - +else + git clone -b v2.10.1 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +rm -rf ${build_dir}/adios2-pitzer-build +cmake -S ${SRC_DIR}/adios2 \ + -B ${build_dir}/adios2-pitzer-build \ + -DBUILD_TESTING=OFF \ + -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_USE_Blosc=ON \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_SST=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.1 +cmake --build ${build_dir}/adios2-pitzer-build --target install -j 16 +rm -rf ${build_dir}/adios2-pitzer-build + +rm -rf ${build_dir} + +# Python ###################################################################### +# +python3 -m pip install --upgrade --user virtualenv +rm -rf ${SW_DIR}/venvs/${VENV_NAME} +python3 -m venv ${SW_DIR}/venvs/${VENV_NAME} +source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip cache purge +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt + +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt + +# ML dependencies +python3 -m pip install --upgrade torch diff --git a/Tools/machines/pitzer-osc/pitzer_cpu.sbatch b/Tools/machines/pitzer-osc/pitzer_cpu.sbatch new file mode 100644 index 00000000000..8097b3842f9 --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_cpu.sbatch @@ -0,0 +1,19 @@ +#!/bin/bash +#SBATCH --time=0:20:00 +#SBATCH --nodes=1 --ntasks-per-node=6 +#SBATCH --cpus-per-task=8 +#SBATCH --job-name= +#SBATCH --account= +#SBATCH --output=./logs/%x_%j.out +#SBATCH --error=./logs/%x_%j.err + +# Pitzer cluster has 224 CPU nodes equipped with dual Intel Xeon 6148 (40 cores per node) and 340 CPU nodes with dual Intel Xeon 8268 (48 cores per node). https://www.osc.edu/resources/technical_support/supercomputers/pitzer + +source ${HOME}/pitzer_cpu_warpx.profile +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + +# executable & inputs file or python interpreter & PICMI script here +EXE=${HOME}/src/warpx/build/bin/warpx.2d +INPUTS=inputs + +srun --cpu-bind=cores ${EXE} ${INPUTS} >./logs/${SLURM_JOB_NAME}_${SLURM_JOBID}.log 2>&1 diff --git a/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example b/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example new file mode 100644 index 00000000000..d99c16b6cb6 --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example @@ -0,0 +1,54 @@ +# please set your project account +export proj="" # change me! + +# remembers the location of this script +export MY_CPU_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_CPU_PROFILE file! Please edit its line 2 to continue!" + return +fi + +export SW_DIR="${HOME}/sw/osc/pitzer/cpu" + +module purge +module load cmake/3.25.2 +module load gnu/12.3.0 +module load openmpi/4.1.5-hpcx + +# optional: for python binding support +module load miniconda3/24.1.2-py310 +export VENV_NAME="warpx-pitzer-cpu" +if [ -d "${SW_DIR}/venvs/${VENV_NAME}" ]; then + source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 --ntasks-per-node=2 --cpus-per-task=20 -t 1:00:00 -A $proj" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --ntasks-per-node=2 --cpus-per-task=20 -t 1:00:00 -A $proj" + +# optional: for PSATD in RZ geometry support +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH + +# optional: for QED lookup table generation support +# use self-installed boost +export CMAKE_PREFIX_PATH=${SW_DIR}/boost-1.82.0:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/boost-1.82.0/lib:$LD_LIBRARY_PATH + +# optional: for openPMD support (hdf5 and adios2) +module load hdf5/1.12.2 +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.6:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.1:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.6/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.1/lib64:$LD_LIBRARY_PATH +export PATH=${SW_DIR}/adios2-2.10.1/bin:${PATH} + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) diff --git a/Tools/machines/pitzer-osc/pitzer_v100.sbatch b/Tools/machines/pitzer-osc/pitzer_v100.sbatch new file mode 100644 index 00000000000..d34868da16c --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_v100.sbatch @@ -0,0 +1,26 @@ +#!/bin/bash +#SBATCH --time=0:20:00 +#SBATCH --nodes=1 --ntasks-per-node=2 +#SBATCH --cpus-per-task=24 +#SBATCH --gpus-per-task=1 +#SBATCH --gpu-bind=closest +#SBATCH --job-name= +#SBATCH --account= +#SBATCH --output=./logs/%x_%j.out +#SBATCH --error=./logs/%x_%j.err + +# Pitzer cluster has 32 GPU nodes with dual Intel Xeon 6148 and dual V100 (16GB) GPUs and 42 nodes with dual Intel Xeon 8268 and dual V100 (32GB) GPUs. https://www.osc.edu/resources/technical_support/supercomputers/pitzer + +source ${HOME}/pitzer_v100_warpx.profile +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + +echo "GPU Information:" +nvidia-smi + +GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" + +# executable & inputs file or python interpreter & PICMI script here +EXE=${HOME}/src/warpx/build_v100/bin/warpx.2d +INPUTS=inputs + +srun --cpu-bind=cores ${EXE} ${INPUTS} ${GPU_AWARE_MPI} >./logs/${SLURM_JOB_NAME}_${SLURM_JOBID}.log 2>&1 diff --git a/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example b/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example new file mode 100644 index 00000000000..061794f5f68 --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example @@ -0,0 +1,64 @@ +# please set your project account +export proj="" # change me! + +# remembers the location of this script +export MY_V100_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_V100_PROFILE file! Please edit its line 2 to continue!" + return +fi + +export SW_DIR="${HOME}/sw/osc/pitzer/v100" + +module purge +module load cmake/3.25.2 +module load intel/19.0.5 +module load cuda/11.8.0 +module load openmpi-cuda/4.1.5-hpcx +module load gcc-compatibility/11.2.0 + +# optional: for python binding support +module load miniconda3/24.1.2-py310 +export VENV_NAME="warpx-pitzer-v100" +if [ -d "${SW_DIR}/venvs/${VENV_NAME}" ]; then + source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" + +# optional: for PSATD in RZ geometry support +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH + +# optional: for QED lookup table generation support +# use self-installed boost +export CMAKE_PREFIX_PATH=${SW_DIR}/boost-1.82.0:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/boost-1.82.0/lib:$LD_LIBRARY_PATH + +# optional: for openPMD support (hdf5 and adios2) +# use self-installed hdf5 +module load hdf5/1.12.0 + +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.6:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.1:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.6/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.1/lib64:$LD_LIBRARY_PATH +export PATH=${SW_DIR}/adios2-2.10.1/bin:${PATH} + +# avoid relocation truncation error which result from large executable size +export CUDAFLAGS="--host-linker-script=use-lcs" # https://github.com/ECP-WarpX/WarpX/pull/3673 +export AMREX_CUDA_ARCH=7.0 # 7.0: V100, 8.0: V100, 9.0: H100 https://github.com/ECP-WarpX/WarpX/issues/3214 + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} From 67796e4be25484a1aab5839896595cee7230ea14 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 1 Aug 2024 14:52:29 -0500 Subject: [PATCH 81/87] ABLASTR: Pass `GridType` instead of Global (#5106) We cannot use global variables from WarpX in the ABLASTR library, otherwise it will not build on its own or in ImpactX. --- Source/BoundaryConditions/PML.H | 3 +- Source/BoundaryConditions/PML.cpp | 6 +++- Source/FieldSolver/ElectrostaticSolver.cpp | 1 + .../FiniteDifferenceSolver/ComputeDivE.cpp | 2 +- .../FiniteDifferenceSolver/EvolveBPML.cpp | 2 +- .../FiniteDifferenceSolver.H | 6 ++-- .../FiniteDifferenceSolver.cpp | 4 +-- .../PsatdAlgorithmComoving.H | 4 ++- .../PsatdAlgorithmComoving.cpp | 12 ++++--- .../PsatdAlgorithmFirstOrder.H | 4 ++- .../PsatdAlgorithmFirstOrder.cpp | 7 ++-- .../PsatdAlgorithmGalileanRZ.H | 5 ++- .../PsatdAlgorithmGalileanRZ.cpp | 2 +- .../PsatdAlgorithmJConstantInTime.H | 4 ++- .../PsatdAlgorithmJConstantInTime.cpp | 2 +- .../PsatdAlgorithmJLinearInTime.H | 7 ++-- .../PsatdAlgorithmJLinearInTime.cpp | 2 +- .../SpectralAlgorithms/PsatdAlgorithmPml.H | 5 +-- .../SpectralAlgorithms/PsatdAlgorithmPml.cpp | 2 +- .../SpectralAlgorithms/PsatdAlgorithmPmlRZ.H | 9 +++-- .../PsatdAlgorithmPmlRZ.cpp | 6 ++-- .../SpectralAlgorithms/PsatdAlgorithmRZ.H | 6 +++- .../SpectralAlgorithms/PsatdAlgorithmRZ.cpp | 6 ++-- .../SpectralBaseAlgorithm.H | 8 +++-- .../SpectralBaseAlgorithm.cpp | 10 ++++-- .../SpectralBaseAlgorithmRZ.H | 6 +++- .../SpectralSolver/SpectralKSpace.H | 10 ++++-- .../SpectralSolver/SpectralKSpace.cpp | 4 +-- .../SpectralSolver/SpectralSolver.H | 8 +++-- .../SpectralSolver/SpectralSolver.cpp | 10 ++++-- .../SpectralSolver/SpectralSolverRZ.H | 6 +++- .../SpectralSolver/SpectralSolverRZ.cpp | 3 +- Source/Parallelization/GuardCellManager.H | 4 ++- Source/Parallelization/GuardCellManager.cpp | 3 +- Source/Utils/WarpXAlgorithmSelection.H | 18 +++------- Source/Utils/WarpXAlgorithmSelection.cpp | 15 ++++---- Source/WarpX.H | 8 +++-- Source/WarpX.cpp | 8 ++--- Source/ablastr/fields/PoissonSolver.H | 7 ++-- Source/ablastr/utils/Enums.H | 35 +++++++++++++++++++ 40 files changed, 186 insertions(+), 84 deletions(-) create mode 100644 Source/ablastr/utils/Enums.H diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 7ed1def8f72..8f3ca1da1d7 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -140,7 +140,8 @@ public: const amrex::DistributionMapping& dm, bool do_similar_dm_pml, const amrex::Geometry* geom, const amrex::Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, - amrex::Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, + amrex::Real dt, int nox_fft, int noy_fft, int noz_fft, + ablastr::utils::enums::GridType grid_type, int do_moving_window, int pml_has_particles, int do_pml_in_domain, int psatd_solution_type, int J_in_time, int rho_in_time, bool do_pml_dive_cleaning, bool do_pml_divb_cleaning, diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index ed82cc07bb9..340005e9211 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -22,6 +22,7 @@ #include "WarpX.H" #include +#include #include #include @@ -548,7 +549,8 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& grid_dm, const bool do_similar_dm_pml, const Geometry* geom, const Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, - Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, + Real dt, int nox_fft, int noy_fft, int noz_fft, + ablastr::utils::enums::GridType grid_type, int do_moving_window, int /*pml_has_particles*/, int do_pml_in_domain, const int psatd_solution_type, const int J_in_time, const int rho_in_time, const bool do_pml_dive_cleaning, const bool do_pml_divb_cleaning, @@ -631,6 +633,8 @@ PML::PML (const int lev, const BoxArray& grid_ba, } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + using namespace ablastr::utils::enums; + // Increase the number of guard cells, in order to fit the extent // of the stencil for the spectral solver int ngFFt_x = (grid_type == GridType::Collocated) ? nox_fft : nox_fft/2; diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 3860bd55325..189f2f2bb0a 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -396,6 +396,7 @@ WarpX::computePhi (const amrex::Vector >& rho, this->geom, this->dmap, this->grids, + WarpX::grid_type, this->m_poisson_boundary_handler, is_solver_igf_on_lev0, WarpX::do_single_precision_comms, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp index 3cc05c58068..0702b264874 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp @@ -52,7 +52,7 @@ void FiniteDifferenceSolver::ComputeDivE ( ComputeDivECylindrical ( Efield, divEfield ); #else - if (m_grid_type == GridType::Collocated) { + if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { ComputeDivECartesian ( Efield, divEfield ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 1fb2dd85e60..0ad2c8d6802 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -53,7 +53,7 @@ void FiniteDifferenceSolver::EvolveBPML ( WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else - if (m_grid_type == GridType::Collocated) { + if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { EvolveBPMLCartesian (Bfield, Efield, dt, dive_cleaning); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index a7a4f10a713..00e87525a75 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -17,6 +17,8 @@ #include "HybridPICModel/HybridPICModel_fwd.H" #include "MacroscopicProperties/MacroscopicProperties_fwd.H" +#include + #include #include @@ -47,7 +49,7 @@ class FiniteDifferenceSolver FiniteDifferenceSolver ( int fdtd_algo, std::array cell_size, - short grid_type ); + ablastr::utils::enums::GridType grid_type ); void EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, @@ -178,7 +180,7 @@ class FiniteDifferenceSolver private: int m_fdtd_algo; - short m_grid_type; + ablastr::utils::enums::GridType m_grid_type; #ifdef WARPX_DIM_RZ amrex::Real m_dr, m_rmin; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp index 1bbc6c9b337..9af610031c0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp @@ -30,7 +30,7 @@ FiniteDifferenceSolver::FiniteDifferenceSolver ( int const fdtd_algo, std::array cell_size, - short grid_type): + ablastr::utils::enums::GridType grid_type): // Register the type of finite-difference algorithm m_fdtd_algo{fdtd_algo}, m_grid_type{grid_type} @@ -63,7 +63,7 @@ FiniteDifferenceSolver::FiniteDifferenceSolver ( "FiniteDifferenceSolver: Unknown algorithm"); } #else - if (grid_type == GridType::Collocated) { + if (grid_type == ablastr::utils::enums::GridType::Collocated) { CartesianNodalAlgorithm::InitializeStencilCoefficients( cell_size, m_h_stencil_coefs_x, m_h_stencil_coefs_y, m_h_stencil_coefs_z ); diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H index 70468277749..b7ee256f414 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H @@ -5,6 +5,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -32,7 +34,7 @@ class PsatdAlgorithmComoving : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_comoving, amrex::Real dt, bool update_with_rho); diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp index 6da6641c052..f809c3f2c20 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp @@ -5,6 +5,8 @@ #include "Utils/WarpXConst.H" #include "Utils/WarpX_Complex.H" +#include + #include #include #include @@ -26,8 +28,10 @@ using namespace amrex; PsatdAlgorithmComoving::PsatdAlgorithmComoving (const SpectralKSpace& spectral_kspace, const DistributionMapping& dm, const SpectralFieldIndex& spectral_index, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type, + const int norder_x, + const int norder_y, + const int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_comoving, const amrex::Real dt, const bool update_with_rho) @@ -35,12 +39,12 @@ PsatdAlgorithmComoving::PsatdAlgorithmComoving (const SpectralKSpace& spectral_k : SpectralBaseAlgorithm{spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type}, // Initialize the infinite-order k vectors (the argument n_order = -1 selects // the infinite order option, the argument grid_type=GridType::Staggered is then irrelevant) - kx_vec{spectral_kspace.getModifiedKComponent(dm, 0, -1, GridType::Staggered)}, + kx_vec{spectral_kspace.getModifiedKComponent(dm, 0, -1, ablastr::utils::enums::GridType::Staggered)}, #if defined(WARPX_DIM_3D) ky_vec{spectral_kspace.getModifiedKComponent(dm, 1, -1, GridType::Staggered)}, kz_vec{spectral_kspace.getModifiedKComponent(dm, 2, -1, GridType::Staggered)}, #else - kz_vec{spectral_kspace.getModifiedKComponent(dm, 1, -1, GridType::Staggered)}, + kz_vec{spectral_kspace.getModifiedKComponent(dm, 1, -1, ablastr::utils::enums::GridType::Staggered)}, #endif m_v_comoving{v_comoving}, m_dt{dt} diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H index 23d4c2f82e3..23c38f33a85 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H @@ -11,6 +11,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -50,7 +52,7 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, amrex::Real dt, bool div_cleaning, int J_in_time, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp index 041b70b8edf..14db3c9af48 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp @@ -28,18 +28,19 @@ using namespace amrex::literals; -PsatdAlgorithmFirstOrder::PsatdAlgorithmFirstOrder( +PsatdAlgorithmFirstOrder::PsatdAlgorithmFirstOrder ( const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, const SpectralFieldIndex& spectral_index, const int norder_x, const int norder_y, const int norder_z, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Real dt, const bool div_cleaning, const int J_in_time, - const int rho_in_time) + const int rho_in_time +) // Initializer list : SpectralBaseAlgorithm(spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type), m_dt(dt), diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H index ad0aca1ecf5..0861eba3abf 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H @@ -9,6 +9,9 @@ #include "SpectralBaseAlgorithmRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -20,7 +23,7 @@ class PsatdAlgorithmGalileanRZ : public SpectralBaseAlgorithmRZ amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, int n_rz_azimuthal_modes, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real dt_step, bool update_with_rho); diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp index c24648fbf07..63127969149 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp @@ -20,7 +20,7 @@ PsatdAlgorithmGalileanRZ::PsatdAlgorithmGalileanRZ (SpectralKSpaceRZ const & spe amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, int const n_rz_azimuthal_modes, int const norder_z, - short const grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real const dt, bool const update_with_rho): diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H index 5d83b5519a6..12e3203db39 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H @@ -11,6 +11,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -52,7 +54,7 @@ class PsatdAlgorithmJConstantInTime : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real dt, bool update_with_rho, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp index 59d56b99afc..991bf981924 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp @@ -36,7 +36,7 @@ PsatdAlgorithmJConstantInTime::PsatdAlgorithmJConstantInTime( const int norder_x, const int norder_y, const int norder_z, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, const amrex::Real dt, const bool update_with_rho, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H index 970657978bb..c26fc4a81a4 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H @@ -11,6 +11,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -53,11 +55,12 @@ class PsatdAlgorithmJLinearInTime : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, amrex::Real dt, bool time_averaging, bool dive_cleaning, - bool divb_cleaning); + bool divb_cleaning + ); /** * \brief Updates the E and B fields in spectral space, according to the multi-J PSATD equations diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp index a6ee55147c9..2626dd0376e 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp @@ -35,7 +35,7 @@ PsatdAlgorithmJLinearInTime::PsatdAlgorithmJLinearInTime( const int norder_x, const int norder_y, const int norder_z, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Real dt, const bool time_averaging, const bool dive_cleaning, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H index 1c19827a0a7..f01b8cad379 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H @@ -12,8 +12,9 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData_fwd.H" #include "FieldSolver/SpectralSolver/SpectralKSpace_fwd.H" -#include +#include +#include #include #include @@ -51,7 +52,7 @@ class PsatdAlgorithmPml : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real dt, bool dive_cleaning, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp index ee776a71a20..5907485a526 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp @@ -37,7 +37,7 @@ PsatdAlgorithmPml::PsatdAlgorithmPml( int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, Real dt, bool dive_cleaning, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H index 00264956900..9f2a21db595 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H @@ -9,6 +9,9 @@ #include "SpectralBaseAlgorithmRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -19,8 +22,10 @@ class PsatdAlgorithmPmlRZ : public SpectralBaseAlgorithmRZ PsatdAlgorithmPmlRZ (SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int n_rz_azimuthal_modes, int norder_z, - short grid_type, amrex::Real dt_step); + int n_rz_azimuthal_modes, + int norder_z, + ablastr::utils::enums::GridType grid_type, + amrex::Real dt_step); // Redefine functions from base class void pushSpectralFields (SpectralFieldDataRZ & f) final; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp index 66aa463503f..50522f3c9b2 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp @@ -19,8 +19,10 @@ using namespace amrex::literals; PsatdAlgorithmPmlRZ::PsatdAlgorithmPmlRZ (SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int const n_rz_azimuthal_modes, int const norder_z, - short const grid_type, amrex::Real const dt): + int const n_rz_azimuthal_modes, + int const norder_z, + ablastr::utils::enums::GridType grid_type, + amrex::Real const dt): // Initialize members of base class and member variables SpectralBaseAlgorithmRZ{spectral_kspace, dm, spectral_index, norder_z, grid_type}, m_dt{dt} diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H index 10feafec38b..c2efbd79935 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H @@ -9,6 +9,9 @@ #include "SpectralBaseAlgorithmRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -20,7 +23,8 @@ class PsatdAlgorithmRZ : public SpectralBaseAlgorithmRZ amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, int n_rz_azimuthal_modes, int norder_z, - short grid_type, amrex::Real dt_step, + ablastr::utils::enums::GridType grid_type, + amrex::Real dt_step, bool update_with_rho, bool time_averaging, int J_in_time, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp index f621d3c7af9..efff5c30a41 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp @@ -19,8 +19,10 @@ using namespace amrex::literals; PsatdAlgorithmRZ::PsatdAlgorithmRZ (SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int const n_rz_azimuthal_modes, int const norder_z, - short const grid_type, amrex::Real const dt, + int const n_rz_azimuthal_modes, + int const norder_z, + ablastr::utils::enums::GridType grid_type, + amrex::Real const dt, bool const update_with_rho, const bool time_averaging, const int J_in_time, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H index e6787c9e52d..c72e7db250d 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H @@ -13,6 +13,8 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData_fwd.H" #include "FieldSolver/SpectralSolver/SpectralFieldData.H" +#include + #include #include #include @@ -88,8 +90,10 @@ class SpectralBaseAlgorithm SpectralBaseAlgorithm(const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, const SpectralFieldIndex& spectral_index, - int norder_x, int norder_y, - int norder_z, short grid_type); + int norder_x, + int norder_y, + int norder_z, + ablastr::utils::enums::GridType grid_type); SpectralFieldIndex m_spectral_index; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp index d635a5debe3..b3f18dd6912 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp @@ -27,11 +27,15 @@ using namespace amrex; /** * \brief Constructor */ -SpectralBaseAlgorithm::SpectralBaseAlgorithm(const SpectralKSpace& spectral_kspace, +SpectralBaseAlgorithm::SpectralBaseAlgorithm ( + const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, const SpectralFieldIndex& spectral_index, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type): + const int norder_x, + const int norder_y, + const int norder_z, + ablastr::utils::enums::GridType grid_type +): m_spectral_index(spectral_index), // Compute and assign the modified k vectors modified_kx_vec(spectral_kspace.getModifiedKComponent(dm,0,norder_x,grid_type)), diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H index 95c68f6d61b..8e03a2a2559 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H @@ -10,6 +10,9 @@ #include "FieldSolver/SpectralSolver/SpectralKSpaceRZ.H" #include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. * @@ -84,7 +87,8 @@ class SpectralBaseAlgorithmRZ SpectralBaseAlgorithmRZ(SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int const norder_z, short const grid_type) + int const norder_z, + ablastr::utils::enums::GridType grid_type) // Compute and assign the modified k vectors : m_spectral_index(spectral_index), modified_kz_vec(spectral_kspace.getModifiedKComponent(dm, 1, norder_z, grid_type)) diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.H b/Source/FieldSolver/SpectralSolver/SpectralKSpace.H index 2a4dd3d1580..16f93d8292a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.H +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.H @@ -12,6 +12,8 @@ #include "Utils/WarpX_Complex.H" +#include + #include #include #include @@ -59,9 +61,11 @@ class SpectralKSpace const amrex::BoxArray& realspace_ba, int i_dim, bool only_positive_k ) const; - KVectorComponent getModifiedKComponent( - const amrex::DistributionMapping& dm, int i_dim, - int n_order, short grid_type ) const; + KVectorComponent getModifiedKComponent ( + const amrex::DistributionMapping& dm, + int i_dim, + int n_order, + ablastr::utils::enums::GridType grid_type) const; SpectralShiftFactor getSpectralShiftFactor( const amrex::DistributionMapping& dm, int i_dim, diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp index 91fe2953668..94bd384f265 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp @@ -188,10 +188,10 @@ SpectralKSpace::getSpectralShiftFactor( const DistributionMapping& dm, * \param grid_type type of grid (collocated or not) */ KVectorComponent -SpectralKSpace::getModifiedKComponent( const DistributionMapping& dm, +SpectralKSpace::getModifiedKComponent (const DistributionMapping& dm, const int i_dim, const int n_order, - const short grid_type ) const + ablastr::utils::enums::GridType grid_type) const { // Initialize an empty DeviceVector in each box KVectorComponent modified_k_comp(spectralspace_ba, dm); diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.H b/Source/FieldSolver/SpectralSolver/SpectralSolver.H index 3d751974a2b..26581900c02 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.H @@ -12,6 +12,8 @@ #include "SpectralAlgorithms/SpectralBaseAlgorithm.H" #include "SpectralFieldData.H" +#include + #include #include #include @@ -72,8 +74,10 @@ class SpectralSolver SpectralSolver (int lev, const amrex::BoxArray& realspace_ba, const amrex::DistributionMapping& dm, - int norder_x, int norder_y, - int norder_z, short grid_type, + int norder_x, + int norder_y, + int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, const amrex::Vector& v_comoving, amrex::RealVect dx, diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index 9fc8f8056ca..c6192187064 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -17,16 +17,20 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXProfilerWrapper.H" +#include + #include #if WARPX_USE_FFT -SpectralSolver::SpectralSolver( +SpectralSolver::SpectralSolver ( const int lev, const amrex::BoxArray& realspace_ba, const amrex::DistributionMapping& dm, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type, + const int norder_x, + const int norder_y, + const int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, const amrex::Vector& v_comoving, const amrex::RealVect dx, const amrex::Real dt, diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H index 30bfe5cf16c..2e94fe95da2 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H @@ -12,6 +12,9 @@ #include "SpectralAlgorithms/SpectralBaseAlgorithmRZ.H" #include "SpectralFieldDataRZ.H" +#include + + /* \brief Top-level class for the electromagnetic spectral solver * * Stores the field in spectral space, and has member functions @@ -31,7 +34,8 @@ class SpectralSolverRZ amrex::BoxArray const & realspace_ba, amrex::DistributionMapping const & dm, int n_rz_azimuthal_modes, - int norder_z, short grid_type, + int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::RealVect dx, amrex::Real dt, bool with_pml, diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp index 9b0d2f21f94..3529247ef56 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp @@ -29,7 +29,8 @@ SpectralSolverRZ::SpectralSolverRZ (const int lev, amrex::BoxArray const & realspace_ba, amrex::DistributionMapping const & dm, int const n_rz_azimuthal_modes, - int const norder_z, short const grid_type, + int const norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::RealVect const dx, amrex::Real const dt, bool const with_pml, diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 561456943f1..c70bd6d3a35 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -7,6 +7,8 @@ #ifndef WARPX_GUARDCELLMANAGER_H_ #define WARPX_GUARDCELLMANAGER_H_ +#include + #include #include #include @@ -55,7 +57,7 @@ public: const amrex::Real * dx, bool do_subcycling, bool do_fdtd_nci_corr, - short grid_type, + ablastr::utils::enums::GridType grid_type, bool do_moving_window, int moving_window_dir, int nox, diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index bf010736853..4c2f978484d 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -36,7 +36,7 @@ guardCellManager::Init ( const amrex::Real *dx, const bool do_subcycling, const bool do_fdtd_nci_corr, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const bool do_moving_window, const int moving_window_dir, const int nox, @@ -200,6 +200,7 @@ guardCellManager::Init ( // currents in the latter case). This does not seem to be necessary in x and y, // where it still seems fine to set half the number of guard cells of the nodal case. + using namespace ablastr::utils::enums; int ngFFt_x = (grid_type == GridType::Collocated) ? nox_fft : nox_fft / 2; int ngFFt_y = (grid_type == GridType::Collocated) ? noy_fft : noy_fft / 2; int ngFFt_z = (grid_type == GridType::Collocated || galilean) ? noz_fft : noz_fft / 2; diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 527b7766895..b9105557462 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -9,9 +9,13 @@ #define WARPX_UTILS_WARPXALGORITHMSELECTION_H_ #include +#include #include +using namespace ablastr::utils::enums; // NOLINT(google-global-names-in-headers) + + /** * \brief struct to determine the computational medium, i.e., vacuum or material/macroscopic default is vacuum. @@ -47,20 +51,6 @@ struct MacroscopicSolverAlgo { }; }; -struct GridType { - enum { - Collocated = 0, - Staggered = 1, - Hybrid = 2 - }; -}; - -enum struct PatchType -{ - fine, - coarse -}; - struct ElectromagneticSolverAlgo { enum { None = 0, diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 89784c3b86c..9e2360315b8 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -10,8 +10,9 @@ #include "WarpXAlgorithmSelection.H" #include "Utils/TextMsg.H" -#include +#include +#include #include #include @@ -31,10 +32,10 @@ const std::map evolve_scheme_to_int = { }; const std::map grid_to_int = { - {"collocated", GridType::Collocated}, - {"staggered", GridType::Staggered}, - {"hybrid", GridType::Hybrid}, - {"default", GridType::Staggered} + {"collocated", static_cast(ablastr::utils::enums::GridType::Collocated)}, + {"staggered", static_cast(ablastr::utils::enums::GridType::Staggered)}, + {"hybrid", static_cast(ablastr::utils::enums::GridType::Hybrid)}, + {"default", static_cast(ablastr::utils::enums::GridType::Staggered)} }; const std::map electromagnetic_solver_algo_to_int = { @@ -153,8 +154,8 @@ const std::map ReductionType_algo_to_int = { }; int -GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ - +GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ) +{ // Read user input ; use "default" if it is not found std::string algo = "default"; pp.query( pp_search_key, algo ); diff --git a/Source/WarpX.H b/Source/WarpX.H index 3372dd869cc..c10be712a22 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -49,6 +49,8 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/export.H" +#include + #include #include #include @@ -371,7 +373,7 @@ public: //! Integer that corresponds to the type of grid used in the simulation //! (collocated, staggered, hybrid) - static short grid_type; + static ablastr::utils::enums::GridType grid_type; // Global rho nodal flag to know about rho index type when rho MultiFab is not allocated amrex::IntVect m_rho_nodal_flag; @@ -1085,7 +1087,7 @@ public: * \param[in] n_order order of the finite-difference approximation * \param[in] a_grid_type type of grid (collocated or not) */ - static amrex::Vector getFornbergStencilCoefficients(int n_order, short a_grid_type); + static amrex::Vector getFornbergStencilCoefficients (int n_order, ablastr::utils::enums::GridType a_grid_type); // Device vectors of stencil coefficients used for finite-order centering of fields amrex::Gpu::DeviceVector device_field_centering_stencil_coeffs_x; @@ -1441,7 +1443,7 @@ private: int centering_nox, int centering_noy, int centering_noz, - short a_grid_type); + ablastr::utils::enums::GridType a_grid_type); void AllocLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, const amrex::IntVect& ngEB, amrex::IntVect& ngJ, diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index b725431c2f1..ca45e883089 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -209,7 +209,7 @@ IntVect WarpX::filter_npass_each_dir(1); int WarpX::n_field_gather_buffer = -1; int WarpX::n_current_deposition_buffer = -1; -short WarpX::grid_type; +ablastr::utils::enums::GridType WarpX::grid_type; amrex::IntVect m_rho_nodal_flag; WarpX* WarpX::m_instance = nullptr; @@ -1066,7 +1066,7 @@ WarpX::ReadParameters () // Integer that corresponds to the type of grid used in the simulation // (collocated, staggered, hybrid) - grid_type = static_cast(GetAlgorithmInteger(pp_warpx, "grid_type")); + grid_type = static_cast(GetAlgorithmInteger(pp_warpx, "grid_type")); // Use same shape factors in all directions, for gathering if (grid_type == GridType::Collocated) { galerkin_interpolation = false; } @@ -3203,7 +3203,7 @@ WarpX::BuildBufferMasksInBox ( const amrex::Box tbx, amrex::IArrayBox &buffer_ma }); } -amrex::Vector WarpX::getFornbergStencilCoefficients(const int n_order, const short a_grid_type) +amrex::Vector WarpX::getFornbergStencilCoefficients (const int n_order, ablastr::utils::enums::GridType a_grid_type) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(n_order % 2 == 0, "n_order must be even"); @@ -3265,7 +3265,7 @@ void WarpX::AllocateCenteringCoefficients (amrex::Gpu::DeviceVector const int centering_nox, const int centering_noy, const int centering_noz, - const short a_grid_type) + ablastr::utils::enums::GridType a_grid_type) { // Vectors of Fornberg stencil coefficients amrex::Vector Fornberg_stencil_coeffs_x; diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 589c6ec1835..26a4c72208d 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -81,6 +82,7 @@ namespace ablastr::fields { * \param[in] geom the geometry per level (e.g., from AmrMesh) * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) + * \param[in] grid_type Integer that corresponds to the type of grid used in the simulation (collocated, staggered, hybrid) * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler * \param[in] is_solver_igf_on_lev0 boolean to select the Poisson solver: 1 for FFT on level 0 & Multigrid on other levels, 0 for Multigrid on all levels * \param[in] do_single_precision_comms perform communications in single precision @@ -105,6 +107,7 @@ computePhi (amrex::Vector const & rho, amrex::Vector const& geom, amrex::Vector const& dmap, amrex::Vector const& grids, + utils::enums::GridType grid_type, T_BoundaryHandler const boundary_handler, bool is_solver_igf_on_lev0, bool const do_single_precision_comms = false, @@ -263,7 +266,7 @@ computePhi (amrex::Vector const & rho, mlmg.setVerbose(verbosity); mlmg.setMaxIter(max_iters); mlmg.setAlwaysUseBNorm(always_use_bnorm); - if (WarpX::grid_type == GridType::Collocated) { + if (grid_type == utils::enums::GridType::Collocated) { // In this case, computeE needs to use ghost nodes data. So we // ask MLMG to fill BC for us after it solves the problem. mlmg.setFinalFillBC(true); @@ -285,7 +288,7 @@ computePhi (amrex::Vector const & rho, const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; ba.coarsen(refratio); const int ncomp = linop.getNComp(); - const int ng = (WarpX::grid_type == GridType::Collocated) ? 1 : 0; + const int ng = (grid_type == utils::enums::GridType::Collocated) ? 1 : 0; amrex::MultiFab phi_cp(ba, phi[lev+1]->DistributionMap(), ncomp, ng); if (ng > 0) { // Set all values outside the domain to zero diff --git a/Source/ablastr/utils/Enums.H b/Source/ablastr/utils/Enums.H new file mode 100644 index 00000000000..1f89bede9e4 --- /dev/null +++ b/Source/ablastr/utils/Enums.H @@ -0,0 +1,35 @@ +/* Copyright 2024 Axel Huebl + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef ABLASTR_UTILS_ENUMS_H_ +#define ABLASTR_UTILS_ENUMS_H_ + +namespace ablastr::utils::enums +{ + /** Type of grids used in a simulation: + * + * Collocated at the same location (AMReX: all "NODAL"), staggered (Yee-style), or hybrid. + */ + enum struct GridType { + Collocated = 0, + Staggered = 1, + Hybrid = 2 + }; + + /** Mesh-refinement patch + * + * The fine or coarse patch (in terms of spatial resolution) on the same MR level. + * https://warpx.readthedocs.io/en/latest/theory/amr.html + */ + enum struct PatchType { + fine, + coarse + }; + +} // namespace ablastr::utils::enums + +#endif // ABLASTR_UTILS_ENUMS_H_ From 6d5486405d543e169ac923688b7f82d8bcb6a619 Mon Sep 17 00:00:00 2001 From: David Grote Date: Thu, 1 Aug 2024 13:19:05 -0700 Subject: [PATCH 82/87] PICMI interface for implicit solvers (#5101) * Implement PICMI interface for implicit solvers * Add CI test * Add ExplicitEvolveScheme * Add evolve scheme classes to PICMI documentation * Made evolve_scheme argument to Simulation * Small fix, removing unused variable * Apply suggestions from code review --------- Co-authored-by: Remi Lehe --- Docs/source/usage/python.rst | 20 ++ .../Implicit/PICMI_inputs_vandb_jfnk_2d.py | 151 ++++++++++++++ Python/pywarpx/WarpX.py | 13 +- Python/pywarpx/picmi.py | 195 ++++++++++++++++++ .../ThetaImplicitJFNK_VandB_2d_PICMI.json | 31 +++ Regression/WarpX-tests.ini | 16 ++ 6 files changed, 425 insertions(+), 1 deletion(-) create mode 100755 Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py create mode 100644 Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index c068447ddbc..38b0a31d7f3 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -57,6 +57,26 @@ Object that allows smoothing of fields. .. autoclass:: pywarpx.picmi.BinomialSmoother +Evolve Schemes +-------------- + +These define the scheme use to evolve the fields and particles. +An instance of one of these would be passed as the `evolve_scheme` into the `Simulation`. + +.. autoclass:: pywarpx.picmi.ExplicitEvolveScheme + +.. autoclass:: pywarpx.picmi.ThetaImplicitEMEvolveScheme + +.. autoclass:: pywarpx.picmi.SemiImplicitEMEvolveScheme + +There are several support classes use to specify components of the evolve schemes + +.. autoclass:: pywarpx.picmi.PicardNonlinearSolver + +.. autoclass:: pywarpx.picmi.NewtonNonlinearSolver + +.. autoclass:: pywarpx.picmi.GMRESLinearSolver + Constants --------- diff --git a/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py b/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py new file mode 100755 index 00000000000..2f919124e13 --- /dev/null +++ b/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# +# --- Tests the python interface to the Implicit solver + +import numpy as np + +from pywarpx import picmi + +constants = picmi.constants + +########################## +# physics parameters +########################## + +n0 = 1.e30 # m^-3 +Ti = 100. # eV +Te = 100. # eV +wpe = constants.q_e*np.sqrt(n0/(constants.m_e*constants.ep0)) +de0 = constants.c/wpe +nppcz = 10 # number of particles/cell in z +dt = 0.1/wpe # s + +vthe = np.sqrt(Te*constants.q_e/constants.m_e) +vthi = np.sqrt(Ti*constants.q_e/constants.m_p) + +########################## +# numerics parameters +########################## + +# --- Number of time steps +max_steps = 20 +diagnostic_intervals = "::20" + +# --- Grid +nx = 40 +ny = 40 + +xmin = 0. +ymin = 0. +xmax = 10.0*de0 +ymax = 10.0*de0 + +number_per_cell_each_dim = [nppcz, nppcz] + +########################## +# physics components +########################## + +electrons_uniform_plasma = picmi.UniformDistribution(density = n0, + rms_velocity = [vthe, vthe, vthe]) + +electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=electrons_uniform_plasma) + +protons_uniform_plasma = picmi.UniformDistribution(density = n0, + rms_velocity = [vthi, vthi, vthi]) + +protons = picmi.Species(particle_type='proton', name='protons', initial_distribution=protons_uniform_plasma) + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid(number_of_cells = [nx, ny], + lower_bound = [xmin, ymin], + upper_bound = [xmax, ymax], + lower_boundary_conditions = ['periodic', 'periodic'], + upper_boundary_conditions = ['periodic', 'periodic'], + warpx_max_grid_size = 8, + warpx_blocking_factor = 8) + +solver = picmi.ElectromagneticSolver(grid = grid, + method = 'Yee') + +GMRES_solver = picmi.GMRESLinearSolver(verbose_int = 2, + max_iterations = 1000, + relative_tolerance = 1.0e-8, + absolute_tolerance = 0.0) + +newton_solver = picmi.NewtonNonlinearSolver(verbose = True, + max_iterations = 20, + relative_tolerance = 1.0e-12, + absolute_tolerance = 0.0, + require_convergence = False, + linear_solver = GMRES_solver, + max_particle_iterations = 21, + particle_tolerance = 1.0e-12) + +evolve_scheme = picmi.ThetaImplicitEMEvolveScheme(theta = 0.5, + nonlinear_solver = newton_solver) + +########################## +# diagnostics +########################## + +field_diag1 = picmi.FieldDiagnostic(name = 'diag1', + grid = grid, + period = diagnostic_intervals, + data_list = ['Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Jx', 'Jy', 'Jz', 'rho', 'divE'], + write_dir = '.', + warpx_file_prefix = 'ThetaImplicitJFNK_VandB_2d_PICMI_plt') + +part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', + period = diagnostic_intervals, + species = [electrons, protons], + data_list = ['weighting', 'position', 'momentum'], + write_dir = '.', + warpx_file_prefix = 'ThetaImplicitJFNK_VandB_2d_PICMI_plt') + +particle_energy_diag = picmi.ReducedDiagnostic(diag_type = 'ParticleEnergy', + name = 'particle_energy', + period = 1) + +field_energy_diag = picmi.ReducedDiagnostic(diag_type = 'FieldEnergy', + name = 'field_energy', + period = 1) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation(solver = solver, + particle_shape = 2, + time_step_size = dt, + max_steps = max_steps, + verbose = 1, + warpx_evolve_scheme = evolve_scheme, + warpx_current_deposition_algo = 'villasenor', + warpx_particle_pusher_algo = 'boris', + warpx_serialize_initial_conditions = 1, + warpx_use_filter = 0) + +sim.add_species(electrons, + layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) +sim.add_species(protons, + layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) + +sim.add_diagnostic(field_diag1) +sim.add_diagnostic(part_diag1) +sim.add_diagnostic(particle_energy_diag) +sim.add_diagnostic(field_energy_diag) + +########################## +# simulation run +########################## + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +sim.write_input_file(file_name = 'inputs2d_from_PICMI') + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() diff --git a/Python/pywarpx/WarpX.py b/Python/pywarpx/WarpX.py index 6b24b35e918..6752b00f371 100644 --- a/Python/pywarpx/WarpX.py +++ b/Python/pywarpx/WarpX.py @@ -89,8 +89,19 @@ def create_argv_list(self, **kw): for diagnostic in reduced_diagnostics._diagnostics_dict.values(): argv += diagnostic.attrlist() + for bucket in self._bucket_dict.values(): + argv += bucket.attrlist() + return argv + def get_bucket(self, bucket_name): + try: + return self._bucket_dict[bucket_name] + except KeyError: + bucket = Bucket(bucket_name) + self._bucket_dict[bucket_name] = bucket + return bucket + def init(self, mpi_comm=None, **kw): # note: argv[0] needs to be an absolute path so it works with AMReX backtraces # https://github.com/AMReX-Codes/amrex/issues/3435 @@ -130,4 +141,4 @@ def write_inputs(self, filename='inputs', **kw): ff.write(f'{arg}\n') -warpx = WarpX('warpx') +warpx = WarpX('warpx', _bucket_dict = {}) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 2aaa6b6122b..8be6d9c6212 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1233,6 +1233,194 @@ def solver_initialize_inputs(self): pywarpx.warpx.do_pml_j_damping = self.do_pml_j_damping +class ExplicitEvolveScheme(picmistandard.base._ClassWithInit): + """ + Sets up the explicit evolve scheme + """ + def solver_scheme_initialize_inputs(self): + pywarpx.algo.evolve_scheme = 'explicit' + + +class ThetaImplicitEMEvolveScheme(picmistandard.base._ClassWithInit): + """ + Sets up the "theta implicit" electromagnetic evolve scheme + + Parameters + ---------- + nonlinear_solver: nonlinear solver instance + The nonlinear solver to use for the iterations + + theta: float, optional + The "theta" parameter, determining the level of implicitness + """ + def __init__(self, nonlinear_solver, theta = None): + self.nonlinear_solver = nonlinear_solver + self.theta = theta + + def solver_scheme_initialize_inputs(self): + pywarpx.algo.evolve_scheme = 'theta_implicit_em' + implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') + implicit_evolve.theta = self.theta + + self.nonlinear_solver.nonlinear_solver_initialize_inputs() + + +class SemiImplicitEMEvolveScheme(picmistandard.base._ClassWithInit): + """ + Sets up the "semi-implicit" electromagnetic evolve scheme + + Parameters + ---------- + nonlinear_solver: nonlinear solver instance + The nonlinear solver to use for the iterations + """ + def __init__(self, nonlinear_solver): + self.nonlinear_solver = nonlinear_solver + + def solver_scheme_initialize_inputs(self): + pywarpx.algo.evolve_scheme = 'semi_implicit_em' + + self.nonlinear_solver.nonlinear_solver_initialize_inputs() + + +class PicardNonlinearSolver(picmistandard.base._ClassWithInit): + """ + Sets up the iterative Picard nonlinear solver for the implicit evolve scheme + + Parameters + ---------- + verbose: bool, default=True + Whether there is verbose output from the solver + + absolute_tolerance: float, default=0. + Absoluate tolerence of the convergence + + relative_tolerance: float, default=1.e-6 + Relative tolerance of the convergence + + max_iterations: integer, default=100 + Maximum number of iterations + + require_convergence: bool, default True + Whether convergence is required. If True and convergence is not obtained, the code will exit. + """ + def __init__(self, verbose=None, absolute_tolerance=None, relative_tolerance=None, + max_iterations=None, require_convergence=None): + self.verbose = verbose + self.absolute_tolerance = absolute_tolerance + self.relative_tolerance = relative_tolerance + self.max_iterations = max_iterations + self.require_convergence = require_convergence + + def nonlinear_solver_initialize_inputs(self): + implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') + implicit_evolve.nonlinear_solver = 'picard' + + picard = pywarpx.warpx.get_bucket('picard') + picard.verbose = self.verbose + picard.absolute_tolerance = self.absolute_tolerance + picard.relative_tolerance = self.relative_tolerance + picard.max_iterations = self.max_iterations + picard.require_convergence = self.require_convergence + + +class NewtonNonlinearSolver(picmistandard.base._ClassWithInit): + """ + Sets up the iterative Newton nonlinear solver for the implicit evolve scheme + + Parameters + ---------- + verbose: bool, default=True + Whether there is verbose output from the solver + + absolute_tolerance: float, default=0. + Absoluate tolerence of the convergence + + relative_tolerance: float, default=1.e-6 + Relative tolerance of the convergence + + max_iterations: integer, default=100 + Maximum number of iterations + + require_convergence: bool, default True + Whether convergence is required. If True and convergence is not obtained, the code will exit. + + linear_solver: linear solver instance, optional + Specifies input arguments to the linear solver + + max_particle_iterations: integer, optional + The maximum number of particle iterations + + particle_tolerance: float, optional + The tolerance of parrticle quantities for convergence + + """ + def __init__(self, verbose=None, absolute_tolerance=None, relative_tolerance=None, + max_iterations=None, require_convergence=None, linear_solver=None, + max_particle_iterations=None, particle_tolerance=None): + self.verbose = verbose + self.absolute_tolerance = absolute_tolerance + self.relative_tolerance = relative_tolerance + self.max_iterations = max_iterations + self.require_convergence = require_convergence + self.linear_solver = linear_solver + self.max_particle_iterations = max_particle_iterations + self.particle_tolerance = particle_tolerance + + def nonlinear_solver_initialize_inputs(self): + implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') + implicit_evolve.nonlinear_solver = 'newton' + implicit_evolve.max_particle_iterations = self.max_particle_iterations + implicit_evolve.particle_tolerance = self.particle_tolerance + + newton = pywarpx.warpx.get_bucket('newton') + newton.verbose = self.verbose + newton.absolute_tolerance = self.absolute_tolerance + newton.relative_tolerance = self.relative_tolerance + newton.max_iterations = self.max_iterations + newton.require_convergence = self.require_convergence + + self.linear_solver.linear_solver_initialize_inputs() + + +class GMRESLinearSolver(picmistandard.base._ClassWithInit): + """ + Sets up the iterative GMRES linear solver for the implicit Newton nonlinear solver + + Parameters + ---------- + verbose_int: integer, default=2 + Level of verbosity of output + + restart_length: integer, default=30 + How often to restart the GMRES iterations + + absolute_tolerance: float, default=0. + Absoluate tolerence of the convergence + + relative_tolerance: float, default=1.e-4 + Relative tolerance of the convergence + + max_iterations: integer, default=1000 + Maximum number of iterations + """ + def __init__(self, verbose_int=None, restart_length=None, absolute_tolerance=None, relative_tolerance=None, + max_iterations=None): + self.verbose_int = verbose_int + self.restart_length = restart_length + self.absolute_tolerance = absolute_tolerance + self.relative_tolerance = relative_tolerance + self.max_iterations = max_iterations + + def linear_solver_initialize_inputs(self): + gmres = pywarpx.warpx.get_bucket('gmres') + gmres.verbose_int = self.verbose_int + gmres.restart_length = self.restart_length + gmres.absolute_tolerance = self.absolute_tolerance + gmres.relative_tolerance = self.relative_tolerance + gmres.max_iterations = self.max_iterations + + class HybridPICSolver(picmistandard.base._ClassWithInit): """ Hybrid-PIC solver based on Ohm's law. @@ -1883,6 +2071,9 @@ class Simulation(picmistandard.PICMI_Simulation): Parameters ---------- + warpx_evolve_scheme: solver scheme instance, optional + Which evolve scheme to use + warpx_current_deposition_algo: {'direct', 'esirkepov', and 'vay'}, optional Current deposition algorithm. The default depends on conditions. @@ -2046,6 +2237,7 @@ class Simulation(picmistandard.PICMI_Simulation): def init(self, kw): + self.evolve_scheme = kw.pop('warpx_evolve_scheme', None) self.current_deposition_algo = kw.pop('warpx_current_deposition_algo', None) self.charge_deposition_algo = kw.pop('warpx_charge_deposition_algo', None) self.field_gathering_algo = kw.pop('warpx_field_gathering_algo', None) @@ -2113,6 +2305,9 @@ def initialize_inputs(self): pywarpx.warpx.sort_idx_type = self.sort_idx_type pywarpx.warpx.sort_bin_size = self.sort_bin_size + if self.evolve_scheme is not None: + self.evolve_scheme.solver_scheme_initialize_inputs() + pywarpx.algo.current_deposition = self.current_deposition_algo pywarpx.algo.charge_deposition = self.charge_deposition_algo pywarpx.algo.field_gathering = self.field_gathering_algo diff --git a/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json b/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json new file mode 100644 index 00000000000..d97eb04883f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 72730.70321925254, + "By": 89276.6097395453, + "Bz": 66911.00019634314, + "Ex": 92036838733000.64, + "Ey": 15583500940725.84, + "Ez": 89163420502164.97, + "divE": 8.998871921763322e+22, + "jx": 2.7748639888523993e+19, + "jy": 2.9501400595579277e+19, + "jz": 2.6976140199337787e+19, + "rho": 796777020986.2787 + }, + "protons": { + "particle_momentum_x": 2.0873315539608036e-17, + "particle_momentum_y": 2.0858882907322405e-17, + "particle_momentum_z": 2.0877345477243595e-17, + "particle_position_x": 0.004251275869323399, + "particle_position_y": 0.0042512738905209615, + "particle_weight": 2823958719279159.5 + }, + "electrons": { + "particle_momentum_x": 4.882673707817137e-19, + "particle_momentum_y": 4.879672470952739e-19, + "particle_momentum_z": 4.872329687213274e-19, + "particle_position_x": 0.004251641684258687, + "particle_position_y": 0.004251751978637919, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 8048318de7a..96dbea5c380 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3864,6 +3864,22 @@ useOMP = 0 numthreads = 1 analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py +[ThetaImplicitJFNK_VandB_2d_PICMI] +buildDir = . +inputFile = Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_vandb_jfnk_2d.py +dim = 2 +addToCompileString = USE_PYTHON_MAIN=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PYTHON=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py + [SemiImplicitPicard_1d] buildDir = . inputFile = Examples/Tests/Implicit/inputs_1d_semiimplicit From f1f3c385e0cfd277c0662e41687a7b4db5eb234a Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:58:10 -0700 Subject: [PATCH 83/87] Treat external field types consistently (#5104) * use external field multifabs consistently for different external field types * make external field MF vectors length equal to `nlevs_max` * avoid adding external grid fields multiple times * load parse ext grid fields after restart * rename `LoadExternalFieldsFromFile` to `LoadExternalFields` * Update inline comment on why external fields are added to ES solutions Co-authored-by: Remi Lehe --------- Co-authored-by: Remi Lehe --- Source/Evolve/WarpXEvolve.cpp | 7 +- Source/Initialization/WarpXInitData.cpp | 131 +++++++++++++----------- Source/WarpX.H | 4 +- Source/WarpX.cpp | 10 +- 4 files changed, 82 insertions(+), 70 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 7bdba64dc4e..32f7a493916 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -246,7 +246,12 @@ WarpX::Evolve (int numsteps) // This is currently a lab frame calculation. ComputeMagnetostaticField(); } - AddExternalFields(); + // Since the fields were reset above, the external fields are added + // back on to the fine patch fields. This make it so that the net fields + // are the sum of the field solution and any external field. + for (int lev = 0; lev <= max_level; ++lev) { + AddExternalFields(lev); + } } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { // Hybrid-PIC case: // The particles are now at p^{n+1/2} and x^{n+1}. The fields diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index e9f63e34bb0..1d5e859c227 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -513,12 +513,6 @@ WarpX::InitData () if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { ComputeMagnetostaticField(); } - - // Set up an invariant condition through the rest of - // execution, that any code besides the field solver that - // looks at field values will see the composite of the field - // solution and any external field - AddExternalFields(); } if (restart_chkfile.empty() || write_diagnostics_on_restart) { @@ -539,15 +533,27 @@ WarpX::InitData () } void -WarpX::AddExternalFields () { - for (int lev = 0; lev <= finest_level; ++lev) { - // FIXME: RZ multimode has more than one component for all these - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { +WarpX::AddExternalFields (int const lev) { + // FIXME: RZ multimode has more than one component for all these + if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant) { + Efield_fp[lev][0]->plus(m_p_ext_field_params->E_external_grid[0], guard_cells.ng_alloc_EB.min()); + Efield_fp[lev][1]->plus(m_p_ext_field_params->E_external_grid[1], guard_cells.ng_alloc_EB.min()); + Efield_fp[lev][2]->plus(m_p_ext_field_params->E_external_grid[2], guard_cells.ng_alloc_EB.min()); + } + else { amrex::MultiFab::Add(*Efield_fp[lev][0], *Efield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][1], *Efield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][2], *Efield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); } - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + } + if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant) { + Bfield_fp[lev][0]->plus(m_p_ext_field_params->B_external_grid[0], guard_cells.ng_alloc_EB.min()); + Bfield_fp[lev][1]->plus(m_p_ext_field_params->B_external_grid[1], guard_cells.ng_alloc_EB.min()); + Bfield_fp[lev][2]->plus(m_p_ext_field_params->B_external_grid[2], guard_cells.ng_alloc_EB.min()); + } + else { amrex::MultiFab::Add(*Bfield_fp[lev][0], *Bfield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][1], *Bfield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][2], *Bfield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); @@ -788,7 +794,7 @@ WarpX::PostRestart () { mypc->PostRestart(); for (int lev = 0; lev <= maxLevel(); ++lev) { - LoadExternalFieldsFromFile(lev); + LoadExternalFields(lev); } } @@ -810,7 +816,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::default_zero; if ( is_B_ext_const && (lev <= maxlevel_extEMfield_init) ) { - Bfield_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); if (fft_do_time_averaging) { Bfield_avg_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); } @@ -831,7 +836,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::default_zero; if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) { - Efield_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); if (fft_do_time_averaging) { Efield_avg_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); } @@ -856,13 +860,12 @@ WarpX::InitLevelData (int lev, Real /*time*/) // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation if ((m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) - && (lev <= maxlevel_extEMfield_init)) { + && (lev > 0) && (lev <= maxlevel_extEMfield_init)) { - // Initialize Bfield_fp with external function InitializeExternalFieldsOnGridUsingParser( - Bfield_fp[lev][0].get(), - Bfield_fp[lev][1].get(), - Bfield_fp[lev][2].get(), + Bfield_aux[lev][0].get(), + Bfield_aux[lev][1].get(), + Bfield_aux[lev][2].get(), m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), @@ -871,31 +874,17 @@ WarpX::InitLevelData (int lev, Real /*time*/) 'B', lev, PatchType::fine); - if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser( - Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser( - Bfield_cp[lev][0].get(), - Bfield_cp[lev][1].get(), - Bfield_cp[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::coarse); - } + InitializeExternalFieldsOnGridUsingParser( + Bfield_cp[lev][0].get(), + Bfield_cp[lev][1].get(), + Bfield_cp[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::coarse); } // if the input string for the E-field is "parse_e_ext_grid_function", @@ -906,19 +895,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ((m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) && (lev <= maxlevel_extEMfield_init)) { - // Initialize Efield_fp with external function - InitializeExternalFieldsOnGridUsingParser( - Efield_fp[lev][0].get(), - Efield_fp[lev][1].get(), - Efield_fp[lev][2].get(), - m_p_ext_field_params->Exfield_parser->compile<3>(), - m_p_ext_field_params->Eyfield_parser->compile<3>(), - m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::fine); - #ifdef AMREX_USE_EB // We initialize ECTRhofield consistently with the Efield if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { @@ -963,7 +939,10 @@ WarpX::InitLevelData (int lev, Real /*time*/) } } - LoadExternalFieldsFromFile(lev); + // load external grid fields into E/Bfield_fp_external multifabs + LoadExternalFields(lev); + // add the external fields to the fine patch fields as initial conditions for the fields + AddExternalFields(lev); if (costs[lev]) { const auto iarr = costs[lev]->IndexArray(); @@ -1354,7 +1333,7 @@ void WarpX::CheckKnownIssues() } void -WarpX::LoadExternalFieldsFromFile (int const lev) +WarpX::LoadExternalFields (int const lev) { // External fields from file are currently not compatible with the moving window // In order to support the moving window, the MultiFab containing the external @@ -1370,8 +1349,21 @@ WarpX::LoadExternalFieldsFromFile (int const lev) } // External grid fields - - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + // Initialize Bfield_fp_external with external function + InitializeExternalFieldsOnGridUsingParser( + Bfield_fp_external[lev][0].get(), + Bfield_fp_external[lev][1].get(), + Bfield_fp_external[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::fine); + } + else if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); @@ -1384,7 +1376,22 @@ WarpX::LoadExternalFieldsFromFile (int const lev) ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); #endif } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { + + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + // Initialize Efield_fp_external with external function + InitializeExternalFieldsOnGridUsingParser( + Efield_fp_external[lev][0].get(), + Efield_fp_external[lev][1].get(), + Efield_fp_external[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::fine); + } + else if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); diff --git a/Source/WarpX.H b/Source/WarpX.H index c10be712a22..ad6e8a3abfd 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1050,7 +1050,7 @@ public: * \brief Load field values from a user-specified openPMD file, * for the fields Ex, Ey, Ez, Bx, By, Bz */ - void LoadExternalFieldsFromFile (int lev); + void LoadExternalFields (int lev); /** * \brief Load field values from a user-specified openPMD file @@ -1293,7 +1293,7 @@ private: void FillBoundaryB_avg (int lev, PatchType patch_type, amrex::IntVect ng); void FillBoundaryE_avg (int lev, PatchType patch_type, amrex::IntVect ng); - void AddExternalFields (); + void AddExternalFields (int lev); void OneStep_nosub (amrex::Real cur_time); void OneStep_sub1 (amrex::Real cur_time); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index ca45e883089..cd86587f2d8 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -285,7 +285,7 @@ WarpX::WarpX () // Loop over species (particles and lasers) // and set current injection position per species - if (do_moving_window){ + if (do_moving_window){ const int n_containers = mypc->nContainers(); for (int i=0; iB_ext_grid_type == ExternalFieldType::read_from_file) { + if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bfield_fp[lev][0]->ixType()), dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); @@ -2614,7 +2614,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(B_external_particle_field[lev][2], amrex::convert(ba, Bfield_aux[lev][2]->ixType()), dm, ncomps, ngEB, lev, "B_external_particle_field[z]", 0.0_rt); } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { + if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Efield_fp[lev][0]->ixType()), dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); From 2837f55d19cf1728751b788619911e44034bbe1e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 6 Aug 2024 14:01:35 -0700 Subject: [PATCH 84/87] Update test to use lasy 0.5.0 (#5111) --- Examples/Tests/laser_injection_from_file/inputs.1d_boost_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.1d_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.2d_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.3d_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.RZ_test | 2 +- .../Tests/laser_injection_from_file/inputs.from_RZ_file_test | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test index adfde819531..ffc1865ee0f 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test +++ b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test @@ -47,7 +47,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_test b/Examples/Tests/laser_injection_from_file/inputs.1d_test index 6124b4552a2..6c392883418 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.1d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.2d_test b/Examples/Tests/laser_injection_from_file/inputs.2d_test index a3003a4978d..e5814471753 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.2d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.2d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.3d_test b/Examples/Tests/laser_injection_from_file/inputs.3d_test index e3096bf92d9..ad8159cb650 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.3d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.3d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.RZ_test b/Examples/Tests/laser_injection_from_file/inputs.RZ_test index fe5bb675212..2a539883fec 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.RZ_test +++ b/Examples/Tests/laser_injection_from_file/inputs.RZ_test @@ -41,7 +41,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test index ca50430c21a..f92440188b7 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test +++ b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test @@ -40,7 +40,7 @@ lasy_RZ_laser.e_max = 1.e14 # Maximum amplitude of the laser field lasy_RZ_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_RZ_laser.profile = from_file lasy_RZ_laser.time_chunk_size = 50 -lasy_RZ_laser.lasy_file_name = "laguerrelaserRZ_00000.h5" +lasy_RZ_laser.lasy_file_name = "diags/laguerrelaserRZ_00000.h5" lasy_RZ_laser.delay = 0.0 # Diagnostics From 53605cd77f0c3478065dec270ca6bf5955795e2f Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 7 Aug 2024 08:56:11 -0700 Subject: [PATCH 85/87] Fix bug in `PrintMainPICparameters` (uninitialized values) (#5114) --- Source/WarpX.H | 2 +- Source/WarpX.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index ad6e8a3abfd..da60824945e 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -233,7 +233,7 @@ public: //! If true, a correction is applied to the current in Fourier space, // to satisfy the continuity equation and charge conservation - bool current_correction; + bool current_correction = true; //! If true, the PSATD update equation for E contains both J and rho //! (default is false for standard PSATD and true for Galilean PSATD) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index cd86587f2d8..4d51dd283a3 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1544,7 +1544,6 @@ WarpX::ReadParameters () // Current correction activated by default, unless a charge-conserving // current deposition (Esirkepov, Vay) or the div(E) cleaning scheme // are used - current_correction = true; if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor || WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay || From db2fbcda27c318a87a4de3c6ce4ae62904a2ef42 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 7 Aug 2024 10:59:17 -0500 Subject: [PATCH 86/87] Temporarily disable `openbc_poisson_solver` (#5117) This test uncoveres a bug of an invalid pc passed early to I/O routines. We will add it back with the fix. --- .../{inputs_3d => README} | 0 Regression/WarpX-tests.ini | 26 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) rename Examples/Tests/openbc_poisson_solver/{inputs_3d => README} (100%) diff --git a/Examples/Tests/openbc_poisson_solver/inputs_3d b/Examples/Tests/openbc_poisson_solver/README similarity index 100% rename from Examples/Tests/openbc_poisson_solver/inputs_3d rename to Examples/Tests/openbc_poisson_solver/README diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 96dbea5c380..bc470d7706b 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3952,16 +3952,16 @@ useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py -[openbc_poisson_solver] -buildDir = . -inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d -runtime_params = warpx.abort_on_warning_threshold = high -dim = 3 -addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py +#[openbc_poisson_solver] +#buildDir = . +#inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d +#runtime_params = warpx.abort_on_warning_threshold = high +#dim = 3 +#addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE +#cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON +#restartTest = 0 +#useMPI = 1 +#numprocs = 2 +#useOMP = 1 +#numthreads = 1 +#analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py From ffb16f3509b166d1a31b90b81c5bd19e24f54c7a Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 7 Aug 2024 13:09:51 -0700 Subject: [PATCH 87/87] CI: Clean out Pip Caches (#5110) * Debugging CI tests * Run all Cartesian 3D tests * Clear cache after python installation commands * Update Regression/WarpX-tests.ini * `run_test.sh`: Detailed `df -h` Prints * Update checksum * Cleanup --------- Co-authored-by: Remi Lehe Co-authored-by: Axel Huebl --- .../{README => inputs_3d} | 0 .../benchmarks_json/collisionXYZ.json | 28 +++++++++---------- Regression/WarpX-tests.ini | 26 ++++++++--------- run_test.sh | 1 + 4 files changed, 28 insertions(+), 27 deletions(-) rename Examples/Tests/openbc_poisson_solver/{README => inputs_3d} (100%) diff --git a/Examples/Tests/openbc_poisson_solver/README b/Examples/Tests/openbc_poisson_solver/inputs_3d similarity index 100% rename from Examples/Tests/openbc_poisson_solver/README rename to Examples/Tests/openbc_poisson_solver/inputs_3d diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index c13f404857a..726573ef8b2 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,25 +6,25 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 381957.7394223898, - "T_ion": 319565.6269784763 + "T_electron": 351982.0169218243, + "T_ion": 349599.6939052666 }, "electron": { - "particle_momentum_x": 8.631833485301232e-19, - "particle_momentum_y": 8.476316745254719e-19, - "particle_momentum_z": 8.514139891331418e-19, - "particle_position_x": 21253105.73686561, - "particle_position_y": 21282643.519070115, - "particle_position_z": 21239057.457948968, + "particle_momentum_x": 8.359982321196841e-19, + "particle_momentum_y": 8.192841151167721e-19, + "particle_momentum_z": 8.182985690701241e-19, + "particle_position_x": 21255110.08090505, + "particle_position_y": 21303488.6242626, + "particle_position_z": 21238676.122703437, "particle_weight": 7.168263344048695e+28 }, "ion": { - "particle_momentum_x": 1.9215585867122464e-18, - "particle_momentum_y": 1.7471481568315848e-18, - "particle_momentum_z": 1.7510887207292533e-18, - "particle_position_x": 21228900.948879313, - "particle_position_y": 21304564.011731848, - "particle_position_z": 21250585.221808463, + "particle_momentum_x": 2.0034830240966893e-18, + "particle_momentum_y": 1.8323959076577197e-18, + "particle_momentum_z": 1.827953230828629e-18, + "particle_position_x": 21246214.748882487, + "particle_position_y": 21280709.710960124, + "particle_position_z": 21206153.002106402, "particle_weight": 7.168263344048695e+28 } } diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index bc470d7706b..96dbea5c380 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3952,16 +3952,16 @@ useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py -#[openbc_poisson_solver] -#buildDir = . -#inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d -#runtime_params = warpx.abort_on_warning_threshold = high -#dim = 3 -#addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE -#cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON -#restartTest = 0 -#useMPI = 1 -#numprocs = 2 -#useOMP = 1 -#numthreads = 1 -#analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py +[openbc_poisson_solver] +buildDir = . +inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d +runtime_params = warpx.abort_on_warning_threshold = high +dim = 3 +addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py diff --git a/run_test.sh b/run_test.sh index f56a957c17a..4efa86eb36c 100755 --- a/run_test.sh +++ b/run_test.sh @@ -65,6 +65,7 @@ python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools wheel python3 -m pip install --upgrade cmake python3 -m pip install --upgrade -r warpx/Regression/requirements.txt +python3 -m pip cache purge # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git