From dde6f7b75b0edf3eed8c5a0f6508a8aefa66c776 Mon Sep 17 00:00:00 2001 From: Ryan Sandberg Date: Wed, 20 Mar 2024 14:18:25 -0700 Subject: [PATCH 001/190] fix variable add in picmi lab particle diagnostics (#4793) --- Python/pywarpx/picmi.py | 82 ++++++++++++++++++++--------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index dfebd13eeb3..7b1daab412a 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2688,47 +2688,47 @@ def diagnostic_initialize_inputs(self): # --- Use a set to ensure that fields don't get repeated. variables = set() - - for dataname in self.data_list: - if dataname == 'position': - if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z - variables.add('x') - if pywarpx.geometry.dims == '3': - variables.add('y') - variables.add('z') - if pywarpx.geometry.dims == 'RZ': - variables.add('theta') - elif dataname == 'momentum': - variables.add('ux') - variables.add('uy') - variables.add('uz') - elif dataname == 'weighting': - variables.add('w') - elif dataname == 'fields': - variables.add('Ex') - variables.add('Ey') - variables.add('Ez') - variables.add('Bx') - variables.add('By') - variables.add('Bz') - elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: - if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): - raise RuntimeError( - f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" - f"chosen by dim={pywarpx.geometry.dims} in pywarpx." - ) - elif pywarpx.geometry.dims != '3' and dataname == 'y': - raise RuntimeError( - f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" - f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." - ) - elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': - raise RuntimeError( - f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." - f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." - ) - else: - variables.add(dataname) + if self.data_list is not None: + for dataname in self.data_list: + if dataname == 'position': + if pywarpx.geometry.dims != '1': # because then it's WARPX_DIM_1D_Z + variables.add('x') + if pywarpx.geometry.dims == '3': + variables.add('y') + variables.add('z') + if pywarpx.geometry.dims == 'RZ': + variables.add('theta') + elif dataname == 'momentum': + variables.add('ux') + variables.add('uy') + variables.add('uz') + elif dataname == 'weighting': + variables.add('w') + elif dataname == 'fields': + variables.add('Ex') + variables.add('Ey') + variables.add('Ez') + variables.add('Bx') + variables.add('By') + variables.add('Bz') + elif dataname in ['x', 'y', 'z', 'theta', 'ux', 'uy', 'uz', 'Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Er', 'Et', 'Br', 'Bt']: + if pywarpx.geometry.dims == '1' and (dataname == 'x' or dataname == 'y'): + raise RuntimeError( + f"The attribute {dataname} is not available in mode WARPX_DIM_1D_Z" + f"chosen by dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != '3' and dataname == 'y': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_3D" + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + elif pywarpx.geometry.dims != 'RZ' and dataname == 'theta': + raise RuntimeError( + f"The attribute {dataname} is not available outside of mode WARPX_DIM_RZ." + f"The chosen value was dim={pywarpx.geometry.dims} in pywarpx." + ) + else: + variables.add(dataname) # --- Convert the set to a sorted list so that the order # --- is the same on all processors. From 1bdd4509c0c59aa72c80598fd42db020b39a31ea Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 21 Mar 2024 01:52:20 +0100 Subject: [PATCH 002/190] update Adastra doc (#4786) --- Tools/machines/adastra-cines/submit.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Tools/machines/adastra-cines/submit.sh b/Tools/machines/adastra-cines/submit.sh index 15a2b292b58..e8bcb19975e 100644 --- a/Tools/machines/adastra-cines/submit.sh +++ b/Tools/machines/adastra-cines/submit.sh @@ -31,6 +31,14 @@ export MPICH_GPU_SUPPORT_ENABLED=1 # or, less invasive: export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor +# note +# On machines with similar architectures (Frontier, OLCF) these settings +# seem to prevent the following issue: +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors +export MPICH_SMP_SINGLE_COPY_MODE=NONE +export FI_CXI_RX_MATCH_MODE=software + # note # this environment setting is needed to avoid that rocFFT writes a cache in # the home directory, which does not scale. From 2b50441499e4397bf67426620c3d8cd56fcac30c Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 20 Mar 2024 20:04:17 -0700 Subject: [PATCH 003/190] Add function to get total weight of all particles in WarpXParticleContainer (#4795) --- .../ReducedDiags/ParticleNumber.cpp | 18 ++---------------- Source/Particles/WarpXParticleContainer.H | 1 + Source/Particles/WarpXParticleContainer.cpp | 16 ++++++++++------ .../Particles/WarpXParticleContainer.cpp | 4 ++++ 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp index 8c0377a3a82..7fe3f75d4ce 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp @@ -112,27 +112,13 @@ void ParticleNumber::ComputeDiags (int step) for (int i_s = 0; i_s < nSpecies; ++i_s) { // get WarpXParticleContainer class object - const auto & myspc = mypc.GetParticleContainer(i_s); + auto & myspc = mypc.GetParticleContainer(i_s); // Save total number of macroparticles for this species m_data[idx_first_species_macroparticles + i_s] = myspc.TotalNumberOfParticles(); - using PType = typename WarpXParticleContainer::SuperParticleType; - - // Reduction to compute sum of weights for this species - auto Wtot = ReduceSum( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> amrex::Real - { - return p.rdata(PIdx::w); - }); - - // MPI reduction - amrex::ParallelDescriptor::ReduceRealSum - (Wtot, amrex::ParallelDescriptor::IOProcessorNumber()); - // Save sum of particles weight for this species - m_data[idx_first_species_sum_weight + i_s] = Wtot; - + m_data[idx_first_species_sum_weight + i_s] = myspc.sumParticleWeight(false); // Increase total number of macroparticles and total weight (all species) m_data[idx_total_macroparticles] += m_data[idx_first_species_macroparticles + i_s]; diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 7d2d5619da9..c100cc0bff4 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -280,6 +280,7 @@ public: /// This is needed when solving Poisson's equation with periodic boundary conditions. /// amrex::ParticleReal sumParticleCharge(bool local = false); + amrex::ParticleReal sumParticleWeight(bool local = false); std::array meanParticleVelocity(bool local = false); diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 912dd3a5a40..46101c29e0b 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1306,9 +1306,9 @@ WarpXParticleContainer::GetChargeDensity (int lev, bool local) return rho; } -amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { +amrex::ParticleReal WarpXParticleContainer::sumParticleWeight(bool local) { - amrex::ParticleReal total_charge = 0.0; + amrex::ParticleReal total_weight = 0.0; ReduceOps reduce_op; ReduceData reduce_data(reduce_op); @@ -1328,11 +1328,15 @@ amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { } } - total_charge = get<0>(reduce_data.value()); + total_weight = get<0>(reduce_data.value()); + + if (!local) { ParallelDescriptor::ReduceRealSum(total_weight); } + return total_weight; +} + +amrex::ParticleReal WarpXParticleContainer::sumParticleCharge(bool local) { - if (!local) { ParallelDescriptor::ReduceRealSum(total_charge); } - total_charge *= this->charge; - return total_charge; + return this->sumParticleWeight(local) * this->charge; } std::array WarpXParticleContainer::meanParticleVelocity(bool local) { diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index 07793a373f3..3feae5d8c90 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -109,6 +109,10 @@ void init_WarpXParticleContainer (py::module& m) &WarpXParticleContainer::TotalNumberOfParticles, py::arg("valid_particles_only"), py::arg("local") ) + .def("sum_particle_weight", + &WarpXParticleContainer::sumParticleWeight, + py::arg("local") + ) .def("sum_particle_charge", &WarpXParticleContainer::sumParticleCharge, py::arg("local") From 32772aad57900823af552cb4fa8a7fc24af5fe07 Mon Sep 17 00:00:00 2001 From: Avigdor Veksler <124003120+aveksler1@users.noreply.github.com> Date: Wed, 20 Mar 2024 21:59:56 -0700 Subject: [PATCH 004/190] Electron current diagnostic for Hybrid solver (#4336) * JeFunctor to output electron current by subtracting ampere's current from total current in hybrid solver * add support for electron current diagnostic if hybrid solver is being used * add documentation for deposit_current parameter * add solver parameter for FieldDiagnostic passed in from Simulation object to help add in electron current diagnostics * Fix compilation errors * fix indentation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix include statement * always define a Je_fields_list in picmi rather than adding extra solver-dependent logic * fix typo * Unify signature in H/cpp * Add to Build Scripts * define m_deposit_current in header file * fix InterpolateMFForDiag function call with bool flag not to sum up modes as Hybrid-PIC only works for m=0 in RZ right now * calculate electron current for every dimension, bug fix * fix namespace issue, single dimension multifab (direction passed in when creating the Functor) * use member function to access private member var * semicolon * change m_dir to be an int from const int in declaration * fix attempted indexing on FAB * fix Je indexing in LinComb * added external current to electron current diag, code cleanup * output JeFunctor as a field_functor initialization loop * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix RZ openPMD functor checks for Je * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix quote issue * rename je to j_e to work with string parsing in converting from WarpX to OpenPMD * fix clang-tidy issue * remove comment * Reworked electron current to be a displacement current diagnostic, only implemented for HybridPICSolver right now. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update test to output new displacement current field * clang tidy fixes * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * renamed picmi field quantity to J_displacement, comment clean-up, add in convert2RZmodes functionality. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update docs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix bug where old naming convention was left behind, plus a missing comma * Apply suggestions from code review Reduce code duplication Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * remove leftover comment block * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestions from code review fix skeleton code and comment * temporarily initialize curlB as nullptr in non-hybrid solver case * fix Jext staggering for subtraction * prepare for checksum * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update var name for checksum * fix OMP bug where tiling was attempted over uninitialized data * commenting out external current functionality to debug failing CI test * update checksum values * corrected staggering IntVects passed into interp function --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl Co-authored-by: Avigdor Veksler Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- .../theory/kinetic_fluid_hybrid_model.rst | 4 + .../Tests/ohm_solver_EM_modes/PICMI_inputs.py | 2 +- Python/pywarpx/picmi.py | 7 ++ .../Python_ohms_law_solver_EM_modes_1d.json | 6 +- .../ComputeDiagFunctors/CMakeLists.txt | 1 + .../ComputeDiagFunctors/JFunctor.H | 2 + .../ComputeDiagFunctors/JdispFunctor.H | 49 ++++++++ .../ComputeDiagFunctors/JdispFunctor.cpp | 106 ++++++++++++++++++ .../ComputeDiagFunctors/Make.package | 1 + Source/Diagnostics/FullDiagnostics.cpp | 34 +++++- .../HybridPICModel/HybridPICModel.H | 6 + Source/WarpX.H | 1 + 12 files changed, 216 insertions(+), 3 deletions(-) create mode 100644 Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H create mode 100644 Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index 3fb8320c531..c517bd3b2f3 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -93,6 +93,10 @@ Implementation details Various verification tests of the hybrid model implementation can be found in the :ref:`examples section `. +.. note:: + WarpX's displacement current diagnostic is equivalent to electron current in + the kinetic-fluid hybrid model. + The kinetic-fluid hybrid extension mostly uses the same routines as the standard electromagnetic PIC algorithm with the only exception that the E-field is calculated from the above equation rather than it being updated from the full Maxwell-Ampere equation. The diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py index 5101e218129..6711331d538 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py @@ -268,7 +268,7 @@ def setup_run(self): name='field_diag', grid=self.grid, period=self.total_steps, - data_list=['B', 'E'], + data_list=['B', 'E', 'J_displacement'], write_dir='.', warpx_file_prefix='Python_ohms_law_solver_EM_modes_1d_plt', # warpx_format = 'openpmd', diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 7b1daab412a..3f7b139d4f2 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2247,11 +2247,13 @@ def diagnostic_initialize_inputs(self): E_fields_list = ['Er', 'Et', 'Ez'] B_fields_list = ['Br', 'Bt', 'Bz'] J_fields_list = ['Jr', 'Jt', 'Jz'] + J_displacement_fields_list = ['Jr_displacement', 'Jt_displacement', 'Jz_displacement'] A_fields_list = ['Ar', 'At', 'Az'] else: E_fields_list = ['Ex', 'Ey', 'Ez'] B_fields_list = ['Bx', 'By', 'Bz'] J_fields_list = ['Jx', 'Jy', 'Jz'] + J_displacement_fields_list = ['Jx_displacement', 'Jy_displacement', 'Jz_displacement'] A_fields_list = ['Ax', 'Ay', 'Az'] if self.data_list is not None: for dataname in self.data_list: @@ -2264,6 +2266,9 @@ def diagnostic_initialize_inputs(self): elif dataname == 'J': for field_name in J_fields_list: fields_to_plot.add(field_name.lower()) + elif dataname == 'J_displacement': + for field_name in J_displacement_fields_list: + fields_to_plot.add(field_name.lower()) elif dataname == 'A': for field_name in A_fields_list: fields_to_plot.add(field_name) @@ -2277,6 +2282,8 @@ def diagnostic_initialize_inputs(self): fields_to_plot.add(dataname) elif dataname in J_fields_list: fields_to_plot.add(dataname.lower()) + elif dataname in J_displacement_fields_list: + fields_to_plot.add(dataname.lower()) elif dataname.startswith('rho_'): # Adds rho_species diagnostic fields_to_plot.add(dataname) diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json index f5606cfee2f..5e42d7ec14b 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_1d.json @@ -5,7 +5,10 @@ "Bz": 256.0, "Ex": 1954.3267519602405, "Ey": 2363.4281756166347, - "Ez": 4873.508158589938 + "Ez": 4873.508158589938, + "jx_displacement": 914396602.5344071, + "jy_displacement": 762623912.4748704, + "jz_displacement": 2938492698.375407 }, "ions": { "particle_momentum_x": 1.6151135948675135e-19, @@ -15,3 +18,4 @@ "particle_weight": 4.220251350277737e+21 } } + diff --git a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt index 2ef6af16bfe..5e0eeaab73a 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt +++ b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt @@ -6,6 +6,7 @@ foreach(D IN LISTS WarpX_DIMS) DivBFunctor.cpp DivEFunctor.cpp JFunctor.cpp + JdispFunctor.cpp RhoFunctor.cpp PartPerCellFunctor.cpp PartPerGridFunctor.cpp diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H index 3080b500712..d9f9a1e82e0 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.H @@ -19,6 +19,8 @@ public: to the output diagnostic MultiFab, mf_dst. * \param[in] convertRZmodes2cartesian (in cylindrical) whether to * sum all modes in mf_src before cell-centering into dst multifab. + * \param[in] deposit_current whether to deposit current. Used to output current in cases + where no field solve is used. * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. */ JFunctor (int dir, int lev, diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H new file mode 100644 index 00000000000..aa364f6da3b --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.H @@ -0,0 +1,49 @@ +#ifndef WARPX_JDISPFUNCTOR_H_ +#define WARPX_JDISPFUNCTOR_H_ + +#include "ComputeDiagFunctor.H" + +#include + +/** + * \brief Functor to cell-center MF for displacement current density and store + * result in mf_out. + */ +class +JdispFunctor final: public ComputeDiagFunctor +{ +public: + /** + * + * \param[in] dir direction of vector field to operate on + * \param[in] lev level of multifab. Used for averaging in rz. + * \param[in] crse_ratio for interpolating field values from the simulation Multifab, src_mf, + * to the output diagnostic MultiFab, mf_dst. + * \param[in] convertRZmodes2cartesian (in cylindrical) whether to + * sum all modes in mf_src before cell-centering into dst multifab. + * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. + */ + JdispFunctor (int dir, int lev, + amrex::IntVect crse_ratio, + bool convertRZmodes2cartesian=true, + int ncomp=1); + /** \brief Cell-center m_mf_src and write the result in mf_dst. + * + * In cylindrical geometry, by default this functor average all components + * of a MultiFab and writes into one single component. + * + * \param[out] mf_dst output MultiFab where the result is written + * \param[in] dcomp first component of mf_dst in which cell-centered + * data is stored + */ + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; +private: + /** direction of the electron current density to save */ + int m_dir; + /** level on which mf_src is defined */ + int m_lev; + /** (for cylindrical) whether to average all modes into 1 comp */ + bool m_convertRZmodes2cartesian; +}; + +#endif // WARPX_JDISPFUNCTOR_H_ diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp new file mode 100644 index 00000000000..a8051b33747 --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -0,0 +1,106 @@ +/* This file is part of Warpx. + * + * Authors: Avigdor Veksler + * License: BSD-3-Clause-LBNL +*/ +#include "JdispFunctor.H" + +#include "WarpX.H" +#include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" +#include "Particles/MultiParticleContainer.H" + +#include +#include +#include + +using namespace amrex; + +JdispFunctor::JdispFunctor (int dir, int lev, + amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, int ncomp) + : ComputeDiagFunctor(ncomp, crse_ratio), m_dir(dir), m_lev(lev), + m_convertRZmodes2cartesian(convertRZmodes2cartesian) +{ } + +void +JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/) const +{ + auto& warpx = WarpX::GetInstance(); + auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); + + /** pointer to source1 (total simulation current J) multifab */ + amrex::MultiFab* m_mf_j = warpx.get_pointer_current_fp(m_lev, m_dir); + amrex::MultiFab* m_mf_curlB; + [[maybe_unused]] amrex::MultiFab* m_mf_j_external; + if (hybrid_pic_model) { + /** pointer to source2 (current calculated from Ampere's Law, Jamp) multifab */ + m_mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir); + /** pointer to source3 (external currents, Jext) multifab */ + m_mf_j_external = hybrid_pic_model->get_pointer_current_fp_external(m_lev, m_dir); + } else { + // To finish this implementation, we need to implement a method to + // calculate (∇ x B). + m_mf_curlB = nullptr; // Remove when curlB implemented + WARPX_ABORT_WITH_MESSAGE( + "Displacement current diagnostic is only implemented for the HybridPICModel."); + + // Skeleton for future implementation for solvers other than HybridPIC. + // Get curlB multifab + + // Divide curlB multifab by mu0 to get units of current + // m_mf_curlB->mult(1.0/PhysConsts::mu0) + } + // A Jdisp multifab is generated to hold displacement current. + amrex::MultiFab Jdisp( m_mf_j->boxArray(), m_mf_j->DistributionMap(), 1, m_mf_j->nGrowVect() ); + Jdisp.setVal(0); + + // J_displacement = curl x B / mu0 - J + amrex::MultiFab::LinComb( + Jdisp, 1, *m_mf_curlB, 0, + -1, *m_mf_j, 0, 0, 1, Jdisp.nGrowVect() + ); + + // Subtract the interpolated j_external value from j_displacement. + if (hybrid_pic_model) { + // Index type required for interpolating Jext from their respective + // staggering (nodal) to the Jx_displacement, Jy_displacement, Jz_displacement + // locations. The staggering of J_displacement is the same as the + // staggering for J, so we use J_stag as the interpolation map. + // For interp to work below, the indices of the undefined dimensions + // must match. We set them as (1,1,1). + amrex::GpuArray Jext_IndexType = {1, 1, 1}; + amrex::GpuArray J_IndexType = {1, 1, 1}; + amrex::IntVect Jext_stag = m_mf_j_external->ixType().toIntVect(); + amrex::IntVect J_stag = m_mf_j->ixType().toIntVect(); + + // Index types for the dimensions simulated are overwritten. + for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + Jext_IndexType[idim] = Jext_stag[idim]; + J_IndexType[idim] = J_stag[idim]; + } + + // Parameters for `interp` that maps from Jext to J. + // The "coarsening is just 1 i.e. no coarsening" + amrex::GpuArray const& coarsen = {1, 1, 1}; + + // Loop through the grids, and over the tiles within each grid to + // subtract the interpolated Jext from J_displacement. +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( MFIter mfi(Jdisp, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { + + Array4 const& Jdisp_arr = Jdisp.array(mfi); + Array4 const& Jext = m_mf_j_external->const_array(mfi); + + // Loop over cells and update the Jdisp MultiFab + amrex::ParallelFor(mfi.tilebox(), [=] AMREX_GPU_DEVICE (int i, int j, int k){ + // Interpolate Jext to the staggering of J + auto const jext_interp = ablastr::coarsen::sample::Interp(Jext, Jext_IndexType, J_IndexType, coarsen, i, j, k, 0); + Jdisp_arr(i, j, k, 0) -= jext_interp; + }); + } + } + + InterpolateMFForDiag(mf_dst, Jdisp, dcomp, warpx.DistributionMap(m_lev), + m_convertRZmodes2cartesian); +} diff --git a/Source/Diagnostics/ComputeDiagFunctors/Make.package b/Source/Diagnostics/ComputeDiagFunctors/Make.package index 9c329263b0f..fd1624b8708 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/Make.package +++ b/Source/Diagnostics/ComputeDiagFunctors/Make.package @@ -4,6 +4,7 @@ CEXE_sources += PartPerGridFunctor.cpp CEXE_sources += DivBFunctor.cpp CEXE_sources += DivEFunctor.cpp CEXE_sources += JFunctor.cpp +CEXE_sources += JdispFunctor.cpp CEXE_sources += RhoFunctor.cpp CEXE_sources += BackTransformFunctor.cpp CEXE_sources += BackTransformParticleFunctor.cpp diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index d93c67fe893..27fe7e48c66 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -3,6 +3,7 @@ #include "ComputeDiagFunctors/DivBFunctor.H" #include "ComputeDiagFunctors/DivEFunctor.H" #include "ComputeDiagFunctors/JFunctor.H" +#include "ComputeDiagFunctors/JdispFunctor.H" #include "ComputeDiagFunctors/PartPerCellFunctor.H" #include "ComputeDiagFunctors/PartPerGridFunctor.H" #include "ComputeDiagFunctors/ParticleReductionFunctor.H" @@ -266,6 +267,24 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) if (update_varnames) { AddRZModesToOutputNames(std::string("jz"), ncomp); } + } else if ( m_varnames_fields[comp] == "jr_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("jr_displacement"), ncomp); + } + } else if ( m_varnames_fields[comp] == "jt_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("jt_displacement"), ncomp); + } + } else if ( m_varnames_fields[comp] == "jz_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, + false, ncomp); + if (update_varnames) { + AddRZModesToOutputNames(std::string("jz_displacement"), ncomp); + } } else if ( m_varnames_fields[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, -1, @@ -636,6 +655,8 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jz" ){ m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true, deposit_current); deposit_current = false; + } else if ( m_varnames[comp] == "jz_displacement" ) { + m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Az" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "rho" ){ @@ -677,11 +698,17 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jt" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); deposit_current = false; + } else if (m_varnames[comp] == "jr_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true); + } else if (m_varnames[comp] == "jt_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Ar" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "At" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 1), lev, m_crse_ratio); - } else { + } + else { + std::cout << "Error on component " << m_varnames[comp] << std::endl; WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); } #else @@ -700,11 +727,16 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jy" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true, deposit_current); deposit_current = false; + } else if ( m_varnames[comp] == "jx_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio); + } else if ( m_varnames[comp] == "jy_displacement" ){ + m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ax" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ay" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 1), lev, m_crse_ratio); } else { + std::cout << "Error on component " << m_varnames[comp] << std::endl; WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); } #endif diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 5e4c6382bfc..7d477b4e5fd 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -200,6 +200,12 @@ public: return current_fp_ampere[lev][direction].get(); } + [[nodiscard]] amrex::MultiFab* + get_pointer_current_fp_external (int lev, int direction) const + { + return current_fp_external[lev][direction].get(); + } + [[nodiscard]] amrex::MultiFab* get_pointer_electron_pressure_fp (int lev) const { diff --git a/Source/WarpX.H b/Source/WarpX.H index 6526171ff67..d7b7266d7b0 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -131,6 +131,7 @@ public: MultiFluidContainer& GetFluidContainer () { return *myfl; } MacroscopicProperties& GetMacroscopicProperties () { return *m_macroscopic_properties; } HybridPICModel& GetHybridPICModel () { return *m_hybrid_pic_model; } + [[nodiscard]] HybridPICModel * get_pointer_HybridPICModel () const { return m_hybrid_pic_model.get(); } MultiDiagnostics& GetMultiDiags () {return *multi_diags;} #ifdef AMREX_USE_EB amrex::Vector >& GetDistanceToEB () {return m_distance_to_eb;} From daa8b3b238ff436331d6587b85b27337c15ce46d Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 21 Mar 2024 07:48:40 -0700 Subject: [PATCH 005/190] Update spacecraft charging test (#4792) * Update spacecraft charging test * Apply suggestions from code review * Apply suggestions from code review * Make calculation of `rho_integral` faster * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py * Update checksum --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../spacecraft_charging/PICMI_inputs_rz.py | 20 ++++----- .../spacecraft_charging/analysis.py | 4 +- .../benchmarks_json/spacecraft_charging.json | 42 +++++++++---------- 3 files changed, 31 insertions(+), 35 deletions(-) diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index 3de56a93710..6eefdca942d 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -25,7 +25,8 @@ class SpaceChargeFieldCorrector(object): """ Class used by the callback functions to calculate the - correct charge on the spacecraft at each initialisation. + correct field around the spacecraft, at each timestep + (taking into account the charge that has been collected on the spacecraft) """ def __init__(self): self.saved_first_iteration_fields = False @@ -46,12 +47,12 @@ def correct_space_charge_fields(self, q=None): q = compute_actual_charge_on_spacecraft() # Correct fields so as to recover the actual charge - Er = ExWrapper(include_ghosts=True)[:,:] - Er[...] = Er[...]+(q - q_v)*self.normalized_Er[...] - Ez = EzWrapper(include_ghosts=True)[:,:] - Ez[...] += (q - q_v)*self.normalized_Ez[...] - phi = PhiFPWrapper(include_ghosts=True)[:,:] - phi[...] += (q - q_v)*self.normalized_phi[...] + Er = ExWrapper(include_ghosts=True) + Er[...] += (q - q_v)*self.normalized_Er + Ez = EzWrapper(include_ghosts=True) + Ez[...] += (q - q_v)*self.normalized_Ez + phi = PhiFPWrapper(include_ghosts=True) + phi[...] += (q - q_v)*self.normalized_phi self.spacecraft_potential += (q - q_v)*self.spacecraft_capacitance sim.extension.warpx.set_potential_on_eb( "%f" %self.spacecraft_potential ) print('Setting potential to %f' %self.spacecraft_potential) @@ -111,10 +112,7 @@ def compute_virtual_charge_on_spacecraft(): # Compute integral of rho over volume of the domain # (i.e. total charge of the plasma particles) - rho_integral = 0.0 - for k in range(1, nz-1): - for i in range(1, nr-1): - rho_integral += rho[i,k] * r[i] * dr * dz + rho_integral = (rho[1:nr-1,1:nz-1] * r[1:nr-1,np.newaxis]).sum()*dr*dz # Due to an oddity in WarpX (which will probably be solved later) # we need to multiply `rho` by `-epsilon_0` to get the correct charge diff --git a/Examples/Physics_applications/spacecraft_charging/analysis.py b/Examples/Physics_applications/spacecraft_charging/analysis.py index 10786baab61..ef75fd1a10a 100755 --- a/Examples/Physics_applications/spacecraft_charging/analysis.py +++ b/Examples/Physics_applications/spacecraft_charging/analysis.py @@ -61,8 +61,8 @@ def func(x, v0, tau): print('v0=%5.3f, tau=%5.9f' % (popt[0], popt[1])) -tolerance_v0=0.01 -tolerance_tau=0.01 +tolerance_v0=0.04 +tolerance_tau=0.04 print("tolerance for v0 = "+ str(tolerance_v0 *100) + '%') print("tolerance for tau = "+ str(tolerance_tau*100) + '%') diff --git a/Regression/Checksum/benchmarks_json/spacecraft_charging.json b/Regression/Checksum/benchmarks_json/spacecraft_charging.json index d9f753e1df2..2994d4feeae 100644 --- a/Regression/Checksum/benchmarks_json/spacecraft_charging.json +++ b/Regression/Checksum/benchmarks_json/spacecraft_charging.json @@ -1,30 +1,28 @@ { "lev=0": { - "Er": 75713.05000099652, - "Ez": 75260.78239853957, - "phi": 55650.30604185804, - "rho": 1.4793075271598396e-06, - "rho_electrons": 6.506538129003745e-06, - "rho_protons": 6.98347902659172e-06 + "Er": 72926.92774219153, + "Ez": 71877.43945667242, + "phi": 58270.32861572235, + "rho": 1.460951374255109e-06, + "rho_electrons": 6.501141119196457e-06, + "rho_protons": 6.992817056027598e-06 }, "electrons": { - "particle_position_x": 38158.7364935527, - "particle_position_y": 37779.25499255196, - "particle_position_z": 45010.371467374425, - "particle_momentum_x": 8.27307207197173e-20, - "particle_momentum_y": 8.264475255806164e-20, - "particle_momentum_z": 8.271327169054914e-20, - "particle_weight": 1140673608016.2212 + "particle_position_x": 38094.07563382932, + "particle_position_y": 37781.48683071032, + "particle_position_z": 44952.3745721341, + "particle_momentum_x": 8.278426203619687e-20, + "particle_momentum_y": 8.272126720129096e-20, + "particle_momentum_z": 8.268559239607837e-20, + "particle_weight": 1140843527312.664 }, "protons": { - "particle_position_x": 751407.372588289, - "particle_position_y": 751687.788498272, - "particle_position_z": 644420.0485148785, - "particle_momentum_x": 1.468116154656724e-17, - "particle_momentum_y": 1.4650318746367807e-17, - "particle_momentum_z": 1.1638654342620123e-17, - "particle_weight": 1175692137613.312 + "particle_position_x": 751412.1033107714, + "particle_position_y": 751687.9554423956, + "particle_position_z": 644516.6066265699, + "particle_momentum_x": 1.4698270572291615e-17, + "particle_momentum_y": 1.466699162579312e-17, + "particle_momentum_z": 1.162881798230096e-17, + "particle_weight": 1176011216106.0706 } } - - From 1381da3682cba2bc01a3499a0bdfdfd1138dad1a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 22 Mar 2024 09:26:18 -0700 Subject: [PATCH 006/190] Document the function `add_applied_field` in PICMI (#4794) --- Docs/source/usage/python.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 639b57d219e..3aba5f165de 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -35,7 +35,7 @@ Simulation and Grid Setup ------------------------- .. autoclass:: pywarpx.picmi.Simulation - :members: step, add_species, add_laser, write_input_file + :members: step, add_species, add_laser, add_applied_field, write_input_file .. autoclass:: pywarpx.picmi.Cartesian3DGrid @@ -69,6 +69,8 @@ which can be used directly inside any PICMI script. The values are in SI units. Applied fields -------------- +Instances of the classes below need to be passed to the method `add_applied_field` of the `Simulation` class. + .. autoclass:: pywarpx.picmi.AnalyticInitialField .. autoclass:: pywarpx.picmi.ConstantAppliedField From e39ead78d0416268fcba3365ad9657cf37192ce1 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Fri, 22 Mar 2024 19:12:11 -0700 Subject: [PATCH 007/190] Docs: update hybrid-pic description and parameters (#4796) * Docs: update hybrid-pic description and parameters * also update docstring in picmi --- .../theory/kinetic_fluid_hybrid_model.rst | 18 +++++++++++------- Docs/source/usage/parameters.rst | 3 +++ Python/pywarpx/picmi.py | 3 +++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index c517bd3b2f3..b4f494d8382 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -73,13 +73,13 @@ If we now further assume electrons are inertialess (i.e. :math:`m=0`), the above en_e\vec{E} = -\vec{J}_e\times\vec{B}-\nabla\cdot{\overleftrightarrow P}_e+\vec{R}_e. Making the further simplifying assumptions that the electron pressure is isotropic and that -the electron drag term can be written as a simple resistance -i.e. :math:`\vec{R}_e = en_e\vec{\eta}\cdot\vec{J}`, brings us to the implemented form of +the electron drag term can be written using a simple resistivity (:math:`\eta`) and hyper-resistivity (:math:`\eta_h`) +i.e. :math:`\vec{R}_e = en_e(\eta-\eta_h \nabla^2)\vec{J}`, brings us to the implemented form of Ohm's law: .. math:: - \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla P_e \right)+\vec{\eta}\cdot\vec{J}. + \vec{E} = -\frac{1}{en_e}\left( \vec{J}_e\times\vec{B} + \nabla P_e \right)+\eta\vec{J}-\eta_h \nabla^2\vec{J}. Lastly, if an electron temperature is given from which the electron pressure can be calculated, the model is fully constrained and can be evolved given initial @@ -93,10 +93,6 @@ Implementation details Various verification tests of the hybrid model implementation can be found in the :ref:`examples section `. -.. note:: - WarpX's displacement current diagnostic is equivalent to electron current in - the kinetic-fluid hybrid model. - The kinetic-fluid hybrid extension mostly uses the same routines as the standard electromagnetic PIC algorithm with the only exception that the E-field is calculated from the above equation rather than it being updated from the full Maxwell-Ampere equation. The @@ -171,5 +167,13 @@ input parameters, :math:`T_{e0}`, :math:`n_0` and :math:`\gamma` using The isothermal limit is given by :math:`\gamma = 1` while :math:`\gamma = 5/3` (default) produces the adiabatic limit. +Electron current +^^^^^^^^^^^^^^^^ + +WarpX's displacement current diagnostic can be used to output the electron current in +the kinetic-fluid hybrid model since in the absence of kinetic electrons, and under +the assumption of zero displacement current, that diagnostic simply calculates the +hybrid model's electron current. + .. bibliography:: :keyprefix: kfhm- diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index bf66c24ccb9..1a3e9e63064 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2263,6 +2263,9 @@ Maxwell solver: kinetic-fluid hybrid * ``hybrid_pic_model.plasma_resistivity(rho,J)`` (`float` or `str`) optional (default ``0``) If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma resistivity in :math:`\Omega m`. +* ``hybrid_pic_model.plasma_hyper_resistivity`` (`float` or `str`) optional (default ``0``) + If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the plasma hyper-resistivity in :math:`\Omega m^3`. + * ``hybrid_pic_model.J[x/y/z]_external_grid_function(x, y, z, t)`` (`float` or `str`) optional (default ``0``) If ``algo.maxwell_solver`` is set to ``hybrid``, this sets the external current (on the grid) in :math:`A/m^2`. diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 3f7b139d4f2..9b64c6e8ecf 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1134,6 +1134,9 @@ class HybridPICSolver(picmistandard.base._ClassWithInit): plasma_resistivity: float or str Value or expression to use for the plasma resistivity. + plasma_hyper_resistivity: float or str + Value or expression to use for the plasma hyper-resistivity. + substeps: int, default=100 Number of substeps to take when updating the B-field. From 964c77e193bfc18a9d9f9ffd4be5e2434723a4ec Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 23 Mar 2024 00:41:53 -0700 Subject: [PATCH 008/190] AMReX/pyAMReX/PICSAR: Weekly Update (#4800) * AMReX: Weekly Update * pyAMReX: Weekly Update * AMReX: Weekly Update --------- Co-authored-by: roelof-groenewald --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 1c384183fbd..feb779d6ad4 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 155f1f4c3f4fa03528d38d2769beb4b35b38042c && cd - + cd ../amrex && git checkout --detach a8633589c1fae5b3fda6deca86fba79878a8cb54 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 138cf90534f..d5ddbe2614d 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 155f1f4c3f4fa03528d38d2769beb4b35b38042c +branch = a8633589c1fae5b3fda6deca86fba79878a8cb54 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 56a7fbc8f10..69b9f0d4b4d 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 155f1f4c3f4fa03528d38d2769beb4b35b38042c +branch = a8633589c1fae5b3fda6deca86fba79878a8cb54 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 54b1cc45322..4c9a7bcb906 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "155f1f4c3f4fa03528d38d2769beb4b35b38042c" +set(WarpX_amrex_branch "a8633589c1fae5b3fda6deca86fba79878a8cb54" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index eaa0fc016cd..3242747b6f5 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "7cd4a4220bc3fc523cd0d78f7320909aa2feb47a" +set(WarpX_pyamrex_branch "e028359d5679e0775b4a8a36332c5d1d37fc7506" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 1bf6e96d236..081efe21626 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 155f1f4c3f4fa03528d38d2769beb4b35b38042c && cd - +cd amrex && git checkout --detach a8633589c1fae5b3fda6deca86fba79878a8cb54 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From aa8df1cf1aa576d4e10e68cdedd3e66e132cc1be Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Sat, 23 Mar 2024 03:44:24 -0700 Subject: [PATCH 009/190] Fix a number of compilation warnings (#4801) * Fix a number of compilation warnings * remove `m_` from non-member variables --- .../ComputeDiagFunctors/JdispFunctor.cpp | 34 +++++++++---------- Source/EmbeddedBoundary/ParticleScraper.H | 2 +- .../Particles/PhysicalParticleContainer.cpp | 4 ++- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index a8051b33747..373004d4633 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -27,19 +27,16 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff auto& warpx = WarpX::GetInstance(); auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); - /** pointer to source1 (total simulation current J) multifab */ - amrex::MultiFab* m_mf_j = warpx.get_pointer_current_fp(m_lev, m_dir); - amrex::MultiFab* m_mf_curlB; - [[maybe_unused]] amrex::MultiFab* m_mf_j_external; + /** pointer to total simulation current (J) multifab */ + amrex::MultiFab* mf_j = warpx.get_pointer_current_fp(m_lev, m_dir); + amrex::MultiFab* mf_curlB; if (hybrid_pic_model) { - /** pointer to source2 (current calculated from Ampere's Law, Jamp) multifab */ - m_mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir); - /** pointer to source3 (external currents, Jext) multifab */ - m_mf_j_external = hybrid_pic_model->get_pointer_current_fp_external(m_lev, m_dir); + /** pointer to current calculated from Ampere's Law (Jamp) multifab */ + mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir); } else { // To finish this implementation, we need to implement a method to // calculate (∇ x B). - m_mf_curlB = nullptr; // Remove when curlB implemented + mf_curlB = nullptr; // Remove when curlB implemented WARPX_ABORT_WITH_MESSAGE( "Displacement current diagnostic is only implemented for the HybridPICModel."); @@ -47,20 +44,23 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff // Get curlB multifab // Divide curlB multifab by mu0 to get units of current - // m_mf_curlB->mult(1.0/PhysConsts::mu0) + // mf_curlB->mult(1.0/PhysConsts::mu0) } // A Jdisp multifab is generated to hold displacement current. - amrex::MultiFab Jdisp( m_mf_j->boxArray(), m_mf_j->DistributionMap(), 1, m_mf_j->nGrowVect() ); + amrex::MultiFab Jdisp( mf_j->boxArray(), mf_j->DistributionMap(), 1, mf_j->nGrowVect() ); Jdisp.setVal(0); // J_displacement = curl x B / mu0 - J amrex::MultiFab::LinComb( - Jdisp, 1, *m_mf_curlB, 0, - -1, *m_mf_j, 0, 0, 1, Jdisp.nGrowVect() + Jdisp, 1, *mf_curlB, 0, + -1, *mf_j, 0, 0, 1, Jdisp.nGrowVect() ); - // Subtract the interpolated j_external value from j_displacement. if (hybrid_pic_model) { + // Subtract the interpolated j_external value from j_displacement. + /** pointer to external currents (Jext) multifab */ + amrex::MultiFab* mf_j_external = hybrid_pic_model->get_pointer_current_fp_external(m_lev, m_dir); + // Index type required for interpolating Jext from their respective // staggering (nodal) to the Jx_displacement, Jy_displacement, Jz_displacement // locations. The staggering of J_displacement is the same as the @@ -69,8 +69,8 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff // must match. We set them as (1,1,1). amrex::GpuArray Jext_IndexType = {1, 1, 1}; amrex::GpuArray J_IndexType = {1, 1, 1}; - amrex::IntVect Jext_stag = m_mf_j_external->ixType().toIntVect(); - amrex::IntVect J_stag = m_mf_j->ixType().toIntVect(); + amrex::IntVect Jext_stag = mf_j_external->ixType().toIntVect(); + amrex::IntVect J_stag = mf_j->ixType().toIntVect(); // Index types for the dimensions simulated are overwritten. for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -90,7 +90,7 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff for ( MFIter mfi(Jdisp, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { Array4 const& Jdisp_arr = Jdisp.array(mfi); - Array4 const& Jext = m_mf_j_external->const_array(mfi); + Array4 const& Jext = mf_j_external->const_array(mfi); // Loop over cells and update the Jdisp MultiFab amrex::ParallelFor(mfi.tilebox(), [=] AMREX_GPU_DEVICE (int i, int j, int k){ diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index 7d6934cbb9c..d058de7e48f 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -189,7 +189,7 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t if (phi_value < 0.0) { int ic, jc, kc; // Cell-centered indices - int nodal; + [[maybe_unused]] int nodal; amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weights ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, ic, jc, kc, Wc, nodal=0); amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phi, dxi); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 160eac0d19c..06154958dde 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -148,7 +148,9 @@ namespace AMREX_GPU_HOST_DEVICE PDim3(const amrex::XDim3& a): - x{a.x}, y{a.y}, z{a.z} + x{static_cast(a.x)}, + y{static_cast(a.y)}, + z{static_cast(a.z)} {} AMREX_GPU_HOST_DEVICE From 91f3ff1be27f1761b069c78399e8cac3fe5316be Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Sun, 24 Mar 2024 18:16:54 -0700 Subject: [PATCH 010/190] Clean up BeamRelevant with TypeMultiplier (#4798) * Clean up BeamRelevant with TypeMultiplier The BeamRelevant reduced diagnostics had a lot of code where `amrex::ReduceOps` were constructed from lists of types. It is hard to see immediately how many types are in that list. The `TypeMultiplier` utility can reduce this to make it more legible and this was done here. * Change length variable to constexpr size_t The variables `num_red_ops_*` are used for the TypeMultiplier and also for the length of the output vectors for the reduced diagnostics. This commit changes their type from `const int` to `constexpr size_t` because this is more appropriate for a vector length, and with constexpr we indicate that the value is constant and can be evaluated at compile time already. --- .../Diagnostics/ReducedDiags/BeamRelevant.cpp | 78 +++++++++---------- 1 file changed, 38 insertions(+), 40 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index fbe67ba6fc5..b78a7713abc 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -198,14 +199,14 @@ void BeamRelevant::ComputeDiags (int step) using PType = typename WarpXParticleContainer::SuperParticleType; - amrex::ReduceOps reduce_ops; - auto r = amrex::ParticleReduce>( + // number of reduction operations in first concurrent batch + constexpr size_t num_red_ops_1 = 8; + TypeMultiplier reduce_ops_1; + using ReducedDataT1 = TypeMultiplier; + + auto r1 = amrex::ParticleReduce( myspc, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple - + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> ReducedDataT1::Type { const ParticleReal p_ux = p.rdata(PIdx::ux); const ParticleReal p_uy = p.rdata(PIdx::uy); @@ -242,18 +243,17 @@ void BeamRelevant::ComputeDiags (int step) p_ux_mean, p_uy_mean, p_uz_mean, p_gm_mean}; }, - reduce_ops); - - std::vector values_per_rank_1st = { - amrex::get<0>(r), // w - amrex::get<1>(r), // x_mean - amrex::get<2>(r), // y_mean - amrex::get<3>(r), // z_mean - amrex::get<4>(r), // ux_mean - amrex::get<5>(r), // uy_mean - amrex::get<6>(r), // uz_mean - amrex::get<7>(r), // gm_mean - }; + reduce_ops_1); + + std::vector values_per_rank_1st(num_red_ops_1); + + /* contains in this order: + * w, x_mean, y_mean, z_mean + * ux_mean, uy_mean, uz_mean, gm_mean + */ + amrex::constexpr_for<0, num_red_ops_1> ([&](auto i) { + values_per_rank_1st[i] = amrex::get(r1); + }); // reduced sum over mpi ranks (allreduce) amrex::ParallelAllReduce::Sum @@ -275,16 +275,15 @@ void BeamRelevant::ComputeDiags (int step) return; } - amrex::ReduceOps reduce_ops2; + // number of reduction operations in second concurrent batch + constexpr size_t num_red_ops_2 = 11; + + TypeMultiplier reduce_ops2; + using ReducedDataT2 = TypeMultiplier; - auto r2 = amrex::ParticleReduce>( + auto r2 = amrex::ParticleReduce( myspc, - [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple - + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> ReducedDataT2::Type { const ParticleReal p_ux = p.rdata(PIdx::ux); const ParticleReal p_uy = p.rdata(PIdx::uy); @@ -336,19 +335,18 @@ void BeamRelevant::ComputeDiags (int step) }, reduce_ops2); - std::vector values_per_rank_2nd = { - amrex::get<0>(r2), // x_ms - amrex::get<1>(r2), // y_ms - amrex::get<2>(r2), // z_ms - amrex::get<3>(r2), // ux_ms - amrex::get<4>(r2), // uy_ms - amrex::get<5>(r2), // uz_ms - amrex::get<6>(r2), // gm_ms - amrex::get<7>(r2), // xux - amrex::get<8>(r2), // yuy - amrex::get<9>(r2), // zuz - amrex::get<10>(r2) // charge - }; + std::vector values_per_rank_2nd(num_red_ops_2); + + /* contains in this order: + * x_ms, y_ms, z_ms + * ux_ms, uy_ms, uz_ms, + * gm_ms + * xux, yuy, zuz, + * charge + */ + amrex::constexpr_for<0, num_red_ops_2> ([&](auto i) { + values_per_rank_2nd[i] = amrex::get(r2); + }); // reduced sum over mpi ranks (reduce to IO rank) ParallelDescriptor::ReduceRealSum From d7b1abac8afa924efd012af6752fac22475752f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Ordyna?= Date: Mon, 25 Mar 2024 02:19:13 +0100 Subject: [PATCH 011/190] Fix ParticleHistogram2D (#4779) * fix unfreed host buffer and missing iteration and series close in ParticleHistogram2D * More efficient resize Co-authored-by: Weiqun Zhang --------- Co-authored-by: Axel Huebl Co-authored-by: Weiqun Zhang --- Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index bcc22eac598..0d696f8b7bc 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -138,7 +138,7 @@ void ParticleHistogram2D::ComputeDiags (int step) Array tlo{0,0}; // lower bounds Array thi{m_bin_num_abs-1, m_bin_num_ord-1}; // inclusive upper bounds amrex::TableData d_data_2D(tlo, thi); - m_h_data_2D = amrex::TableData (tlo, thi, The_Pinned_Arena()); + m_h_data_2D.resize(tlo, thi, The_Pinned_Arena()); auto const& h_table_data = m_h_data_2D.table(); // Initialize data on the host @@ -316,6 +316,8 @@ void ParticleHistogram2D::WriteToFile (int step) const {static_cast(m_bin_num_ord), static_cast(m_bin_num_abs)}); series.flush(); + i.close(); + series.close(); #else amrex::ignore_unused(step); WARPX_ABORT_WITH_MESSAGE("ParticleHistogram2D: Needs openPMD-api compiled into WarpX, but was not found!"); From f49a63f038cf081ca8ea93420c0ddffb3279b918 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 25 Mar 2024 16:03:05 -0700 Subject: [PATCH 012/190] Remove GPUClock Cost Function (#4802) Removing the GPUClock cost function due to the following reasons. Incomplete Implementation: The implementation is only added to selected kernels. The implementation is not generalized to work with varying occupancy of different kernels, even if it were used in all kernels. The implementation is verbose. Unused: Our host-side timer implementation was in the last years extended to synchronize kernels at minimal overhead cost. This and heuristic is actually used. Research scope shifted: In the last years, we realized that we do not need more precise scalar cost functions, but instead vector cost functions to build better load balance performance models from. Costly when used: The implementation uses an atomic add of each kernel, instead of, e.g., just using one per warp. This adds severe memory bandwidth strain. Costly, even if not used: The implementation adds about 4 registers unnecessary to all instrumented GPU kernels once compiled in (by default). --- CMakeLists.txt | 8 - Docs/source/developers/testing.rst | 2 +- Docs/source/install/cmake.rst | 1 - Docs/source/usage/parameters.rst | 6 +- GNUmakefile | 1 - Python/pywarpx/picmi.py | 2 +- Source/Make.WarpX | 5 - .../Particles/Deposition/ChargeDeposition.H | 59 +---- .../Particles/Deposition/CurrentDeposition.H | 210 +----------------- Source/Particles/WarpXParticleContainer.cpp | 114 ++++------ Source/Utils/WarpXAlgorithmSelection.H | 5 +- Source/Utils/WarpXAlgorithmSelection.cpp | 1 - Source/WarpX.H | 2 +- Source/WarpX.cpp | 4 - Source/ablastr/particles/DepositCharge.H | 17 +- Tools/PerformanceTests/summit.py | 2 +- WarpXConfig.cmake | 1 - cmake/WarpXFunctions.cmake | 1 - 18 files changed, 65 insertions(+), 376 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e1d5dca6e3a..55fae3db78d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,10 +69,6 @@ include(CMakeDependentOption) option(WarpX_APP "Build the WarpX executable application" ON) option(WarpX_ASCENT "Ascent in situ diagnostics" OFF) option(WarpX_EB "Embedded boundary support" OFF) -cmake_dependent_option(WarpX_GPUCLOCK - "Add GPU kernel timers (cost function)" ON - "WarpX_COMPUTE STREQUAL CUDA OR WarpX_COMPUTE STREQUAL HIP" - OFF) option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) @@ -510,10 +506,6 @@ foreach(D IN LISTS WarpX_DIMS) target_compile_definitions(ablastr_${SD} PUBLIC WARPX_DIM_RZ WARPX_ZINDEX=1) endif() - if(WarpX_GPUCLOCK) - target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_GPUCLOCK) - endif() - if(WarpX_OPENPMD) target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_OPENPMD) endif() diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst index 6dcc1886e12..c6a09d970df 100644 --- a/Docs/source/developers/testing.rst +++ b/Docs/source/developers/testing.rst @@ -90,7 +90,7 @@ The content of this directory will look like the following (possibly including b $ ls ./test_dir/rt-WarpX/WarpX-tests/2021-04-30/pml_x_yee/ analysis_pml_yee.py # Python analysis script inputs_2d # input file - main2d.gnu.TEST.TPROF.MTMPI.OMP.QED.GPUCLOCK.ex # executable + main2d.gnu.TEST.TPROF.MTMPI.OMP.QED.ex # executable pml_x_yee.analysis.out # Python analysis output pml_x_yee.err.out # error output pml_x_yee.make.out # build output diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 7ff8986cf36..2e66c95a4d6 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -89,7 +89,6 @@ CMake Option Default & Values Descr ``WarpX_COMPUTE`` NOACC/**OMP**/CUDA/SYCL/HIP On-node, accelerated computing backend ``WarpX_DIMS`` **3**/2/1/RZ Simulation dimensionality. Use ``"1;2;RZ;3"`` for all. ``WarpX_EB`` ON/**OFF** Embedded boundary support (not supported in RZ yet) -``WarpX_GPUCLOCK`` **ON**/OFF Add GPU kernel timers (cost function, +4 registers/kernel) ``WarpX_IPO`` ON/**OFF** Compile WarpX with interprocedural optimization (aka LTO) ``WarpX_LIB`` ON/**OFF** Build WarpX as a library, e.g., for PICMI Python ``WarpX_MPI`` **ON**/OFF Multi-node support (message-passing) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 1a3e9e63064..fc6d338fc1c 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -557,7 +557,7 @@ Distribution across MPI ranks and parallelization For example, if there are 4 boxes per rank and `load_balance_knapsack_factor=2`, no more than 8 boxes can be assigned to any rank. -* ``algo.load_balance_costs_update`` (`heuristic` or `timers` or `gpuclock`) optional (default `timers`) +* ``algo.load_balance_costs_update`` (``heuristic`` or ``timers``) optional (default ``timers``) If this is `heuristic`: load balance costs are updated according to a measure of particles and cells assigned to each box of the domain. The cost :math:`c` is computed as @@ -574,10 +574,6 @@ Distribution across MPI ranks and parallelization If this is `timers`: costs are updated according to in-code timers. - If this is `gpuclock`: [**requires to compile with option** ``-DWarpX_GPUCLOCK=ON``] - costs are measured as (max-over-threads) time spent in current deposition - routine (only applies when running on GPUs). - * ``algo.costs_heuristic_particles_wt`` (`float`) optional Particle weight factor used in `Heuristic` strategy for costs update; if running on GPU, the particle weight is set to a value determined from single-GPU tests on Summit, diff --git a/GNUmakefile b/GNUmakefile index 7ddded50483..863d37dd56f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -29,7 +29,6 @@ USE_GPU = FALSE EBASE = main -USE_GPUCLOCK = TRUE USE_PYTHON_MAIN = FALSE USE_SENSEI_INSITU = FALSE diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 9b64c6e8ecf..5902b958bc9 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1825,7 +1825,7 @@ class Simulation(picmistandard.PICMI_Simulation): warpx_load_balance_knapsack_factor: float, default=1.24 (See documentation) - warpx_load_balance_costs_update: {'heuristic' or 'timers' or 'gpuclock'}, optional + warpx_load_balance_costs_update: {'heuristic' or 'timers'}, optional (See documentation) warpx_costs_heuristic_particles_wt: float, optional diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 6c55372d199..4662dc18abd 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -214,11 +214,6 @@ ifeq ($(USE_HDF5),TRUE) DEFINES += -DWARPX_USE_HDF5 endif -ifeq ($(USE_GPUCLOCK),TRUE) - USERSuffix := $(USERSuffix).GPUCLOCK - DEFINES += -DWARPX_USE_GPUCLOCK -endif - # job_info support CEXE_sources += AMReX_buildInfo.cpp INCLUDE_LOCATIONS += $(AMREX_HOME)/Tools/C_scripts diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index d0822789015..8ff712eefa2 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -33,8 +33,6 @@ * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost: Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, @@ -42,20 +40,14 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, const int* ion_lev, amrex::FArrayBox& rho_fab, long np_to_deposit, - const std::array& dx, + const std::array& dx, const std::array xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex; -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; @@ -87,21 +79,9 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, constexpr int CELL = amrex::IndexType::CELL; // Loop over particles and deposit into rho_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0.; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif // --- Get particle quantities amrex::Real wq = q*wp[ip]*invvol; if (do_ionization){ @@ -202,13 +182,6 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if (cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif #ifndef WARPX_DIM_RZ amrex::ignore_unused(n_rz_azimuthal_modes); @@ -230,8 +203,6 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. * \param a_bins * \param box * \param geom @@ -245,13 +216,11 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio amrex::FArrayBox& rho_fab, const amrex::IntVect& ix_type, const long np_to_deposit, - const std::array& dx, + const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, const amrex::Real q, const int n_rz_azimuthal_modes, - amrex::Real* cost, - const long load_balance_costs_update_algo, const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, @@ -264,7 +233,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio const auto *permutation = a_bins.permutationPtr(); #if !defined(AMREX_USE_GPU) - amrex::ignore_unused(ix_type, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size, bin_size); + amrex::ignore_unused(ix_type, a_bins, box, geom, a_tbox_max_size, bin_size); #endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer @@ -299,14 +268,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio constexpr int CELL = amrex::IndexType::CELL; // Loop over particles and deposit into rho_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0.; - } -#endif - #if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) const auto dxiarr = geom.InvCellSizeArray(); const auto plo = geom.ProbLoArray(); @@ -394,11 +355,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio { const unsigned int ip = permutation[ip_orig]; -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif // --- Get particle quantities amrex::Real wq = q*wp[ip]*invvol; if (do_ionization){ @@ -506,13 +462,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif // defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) } ); -#if defined(WARPX_USE_GPUCLOCK) - if(cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif #ifndef WARPX_DIM_RZ amrex::ignore_unused(n_rz_azimuthal_modes); diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 8d4c5b6a5f4..e722839c9c6 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -287,8 +287,6 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doDepositionShapeN (const GetParticlePosition& GetPosition, @@ -306,9 +304,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, const std::array& xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex::literals; @@ -316,10 +312,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; @@ -354,22 +346,9 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::IntVect const jz_type = jz_fab.box().type(); // Loop over particles and deposit into jx_fab, jy_fab and jz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - amrex::ParticleReal xp, yp, zp; GetPosition(ip, xp, yp, zp); @@ -395,13 +374,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, } ); -#if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -424,8 +396,6 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, @@ -445,19 +415,13 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, const std::array& xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, - amrex::Real* cost, - const long load_balance_costs_update_algo) + const int n_rz_azimuthal_modes) { using namespace amrex::literals; #if !defined(WARPX_DIM_RZ) amrex::ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; @@ -490,22 +454,9 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, amrex::IntVect const jz_type = jz_fab.box().type(); // Loop over particles and deposit into jx_fab, jy_fab and jz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - amrex::ParticleReal xp, yp, zp; GetPosition(ip, xp, yp, zp); @@ -539,13 +490,6 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, } ); -#if defined(WARPX_USE_GPUCLOCK) - if(cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -570,8 +514,6 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, @@ -590,8 +532,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::Dim3 lo, amrex::Real q, int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo, const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, @@ -616,13 +556,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, constexpr int CELL = amrex::IndexType::CELL; // Loop over particles and deposit into jx_fab, jy_fab and jz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if (cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif const auto dxiarr = geom.InvCellSizeArray(); const auto plo = geom.ProbLoArray(); const auto domain = geom.Domain(); @@ -651,11 +584,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::launch( nblocks, threads_per_block, shared_mem_bytes, amrex::Gpu::gpuStream(), [=] AMREX_GPU_DEVICE () noexcept { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif const int bin_id = blockIdx.x; const unsigned int bin_start = offsets_ptr[bin_id]; const unsigned int bin_stop = offsets_ptr[bin_id+1]; @@ -743,18 +671,11 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, __syncthreads(); addLocalToGlobal(tbox_z, jz_arr, jz_buff); }); -#if defined(WARPX_USE_GPUCLOCK) - if (cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif #else // not using hip/cuda // Note, you should never reach this part of the code. This funcion cannot be called unless // using HIP/CUDA, and those things are checked prior //don't use any args - ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, cost, load_balance_costs_update_algo, a_bins, box, geom, a_tbox_max_size); + ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, a_bins, box, geom, a_tbox_max_size); WARPX_ABORT_WITH_MESSAGE("Shared memory only implemented for HIP/CUDA"); #endif } @@ -782,8 +703,6 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, @@ -802,9 +721,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, std::array xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real * const cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex; using namespace amrex::literals; @@ -813,10 +730,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; @@ -857,22 +770,9 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - // --- Get particle quantities Real const gaminv = 1.0_rt/std::sqrt(1.0_rt + uxp[ip]*uxp[ip]*clightsq + uyp[ip]*uyp[ip]*clightsq @@ -1127,13 +1027,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if(cost && load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -1159,8 +1052,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, @@ -1180,23 +1071,17 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, + const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, - amrex::Real * const cost, - const long load_balance_costs_update_algo) + const int n_rz_azimuthal_modes) { using namespace amrex; #if !defined(WARPX_DIM_RZ) ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; @@ -1236,22 +1121,9 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - #if !defined(WARPX_DIM_3D) constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); @@ -1526,13 +1398,6 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -1560,8 +1425,6 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. - * \param cost Pointer to (load balancing) cost corresponding to box where present particles deposit current. - * \param load_balance_costs_update_algo Selected method for updating load balance costs. */ template void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, @@ -1581,23 +1444,17 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, + const std::array& dx, const std::array xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, - amrex::Real * const cost, - const long load_balance_costs_update_algo) + const int n_rz_azimuthal_modes) { using namespace amrex; #if !defined(WARPX_DIM_RZ) ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(AMREX_USE_GPU) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; @@ -1628,21 +1485,9 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ #endif // Loop over particles and deposit into Jx_arr, Jy_arr and Jz_arr -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor( np_to_deposit, [=] AMREX_GPU_DEVICE (long const ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif #if !defined(WARPX_DIM_3D) constexpr amrex::ParticleReal inv_c2 = 1._prt/(PhysConst::c*PhysConst::c); @@ -2190,13 +2035,6 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ #endif } ); -#if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -#endif } /** @@ -2224,9 +2062,6 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ * \param[in] lo Dimension-agnostic lower bounds of index domain * \param[in] q Species charge * \param[in] n_rz_azimuthal_modes Number of azimuthal modes in RZ geometry - * \param[in,out] cost Pointer to (load balancing) cost corresponding to box where - present particles deposit current - * \param[in] load_balance_costs_update_algo Selected method for updating load balance costs */ template void doVayDepositionShapeN (const GetParticlePosition& GetPosition, @@ -2245,9 +2080,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, const std::array& xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes, - amrex::Real* cost, - long load_balance_costs_update_algo) + int n_rz_azimuthal_modes) { using namespace amrex::literals; @@ -2265,10 +2098,6 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in cartesian 1D geometry"); #endif -#if !defined(WARPX_USE_GPUCLOCK) - amrex::ignore_unused(cost, load_balance_costs_update_algo); -#endif - #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) amrex::ignore_unused(n_rz_azimuthal_modes); @@ -2319,21 +2148,8 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, amrex::Array4 const& Dz_arr = Dz_fab.array(); // Loop over particles and deposit (Dx,Dy,Dz) into Dx_fab, Dy_fab and Dz_fab -#if defined(WARPX_USE_GPUCLOCK) - amrex::Real* cost_real = nullptr; - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - cost_real = (amrex::Real *) amrex::The_Managed_Arena()->alloc(sizeof(amrex::Real)); - *cost_real = 0._rt; - } -#endif amrex::ParallelFor(np_to_deposit, [=] AMREX_GPU_DEVICE (long ip) { -#if defined(WARPX_USE_GPUCLOCK) - const auto KernelTimer = ablastr::parallelization::KernelTimer( - cost && (load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock), - cost_real); -#endif - // Inverse of Lorentz factor gamma const amrex::Real invgam = 1._rt / std::sqrt(1._rt + uxp[ip] * uxp[ip] * invcsq + uyp[ip] * uyp[ip] * invcsq @@ -2560,14 +2376,6 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, // Synchronize so that temp_fab can be safely deallocated in its destructor amrex::Gpu::streamSynchronize(); - -# if defined(WARPX_USE_GPUCLOCK) - if( load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::GpuClock) { - amrex::Gpu::streamSynchronize(); - *cost += *cost_real; - amrex::The_Managed_Arena()->free(cost_real); - } -# endif #endif // #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) } #endif // CURRENTDEPOSITION_H_ diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 46101c29e0b..fd1bc7917ab 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -461,8 +461,6 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, } WARPX_PROFILE_VAR_START(blp_deposit); - amrex::LayoutData * const costs = WarpX::getCosts(lev); - amrex::Real * const cost = costs ? &((*costs)[pti.index()]) : nullptr; // If doing shared mem current deposition, get tile info if (WarpX::do_shared_mem_current_deposition) { @@ -531,29 +529,29 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } else if (WarpX::nox == 2){ doDepositionSharedShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } else if (WarpX::nox == 3){ doDepositionSharedShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } else if (WarpX::nox == 4){ doDepositionSharedShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size); } WARPX_PROFILE_VAR_STOP(direct_current_dep_kernel); } @@ -567,29 +565,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { #if (AMREX_SPACEDIM >= 2) @@ -616,8 +610,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doChargeConservingDepositionShapeNImplicit<2>( xp_n_data, yp_n_data, zp_n_data, @@ -625,8 +618,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doChargeConservingDepositionShapeNImplicit<3>( xp_n_data, yp_n_data, zp_n_data, @@ -634,8 +626,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doChargeConservingDepositionShapeNImplicit<4>( xp_n_data, yp_n_data, zp_n_data, @@ -643,8 +634,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } } else if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { @@ -673,8 +663,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVillasenorDepositionShapeNImplicit<2>( xp_n_data, yp_n_data, zp_n_data, @@ -682,8 +671,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVillasenorDepositionShapeNImplicit<3>( xp_n_data, yp_n_data, zp_n_data, @@ -691,8 +679,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVillasenorDepositionShapeNImplicit<4>( xp_n_data, yp_n_data, zp_n_data, @@ -700,8 +687,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } else { @@ -716,29 +702,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVayDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVayDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVayDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + WarpX::n_rz_azimuthal_modes); } } else { // Direct deposition if (push_type == PushType::Explicit) { @@ -747,29 +729,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { auto& uxp_n = pti.GetAttribs(particle_comps["ux_n"]); @@ -782,8 +760,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeNImplicit<2>( GetPosition, wp.dataPtr() + offset, @@ -791,8 +768,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeNImplicit<3>( GetPosition, wp.dataPtr() + offset, @@ -800,8 +776,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeNImplicit<4>( GetPosition, wp.dataPtr() + offset, @@ -809,8 +784,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, dx, - xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo); + xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } } @@ -1084,8 +1058,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, } WARPX_PROFILE_VAR_START(blp_ppc_chd); - amrex::LayoutData* costs = WarpX::getCosts(lev); - amrex::Real* cost = costs ? &((*costs)[pti.index()]) : nullptr; const auto GetPosition = GetParticlePosition(pti, offset); const Geometry& geom = Geom(lev); @@ -1095,26 +1067,26 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, if (WarpX::nox == 1){ doChargeDepositionSharedShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 2){ doChargeDepositionSharedShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 3){ doChargeDepositionSharedShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 4){ doChargeDepositionSharedShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, - WarpX::n_rz_azimuthal_modes, cost, - WarpX::load_balance_costs_update_algo, bins, box, geom, max_tbox_size, + WarpX::n_rz_azimuthal_modes, + bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } #ifndef AMREX_USE_GPU @@ -1158,10 +1130,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const amrex::Real time_shift_delta = (icomp == 0 ? 0.0_rt : dt); const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); - // pointer to costs data - amrex::LayoutData* costs = WarpX::getCosts(lev); - amrex::Real* cost = costs ? &((*costs)[pti.index()]) : nullptr; - AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noy); AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noz); @@ -1172,7 +1140,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, ng_rho, depos_lev, ref_ratio, offset, np_to_deposit, icomp, nc, - cost, WarpX::load_balance_costs_update_algo, WarpX::do_device_synchronize + WarpX::do_device_synchronize ); } } diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index e94b7bb7719..b7af3121814 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -132,9 +132,8 @@ struct RhoInTime { struct LoadBalanceCostsUpdateAlgo { enum { Timers = 0, //!< load balance according to in-code timer-based weights (i.e., with `costs`) - Heuristic = 1, /**< load balance according to weights computed from number of cells - and number of particles per box (i.e., with `costs_heuristic`)*/ - GpuClock = 2 + Heuristic = 1 /**< load balance according to weights computed from number of cells + and number of particles per box (i.e., with `costs_heuristic`) */ }; }; diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index c58318bbd53..3092e1984d6 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -101,7 +101,6 @@ const std::map rho_in_time_to_int = { const std::map load_balance_costs_update_algo_to_int = { {"timers", LoadBalanceCostsUpdateAlgo::Timers }, - {"gpuclock", LoadBalanceCostsUpdateAlgo::GpuClock }, {"heuristic", LoadBalanceCostsUpdateAlgo::Heuristic }, {"default", LoadBalanceCostsUpdateAlgo::Timers } }; diff --git a/Source/WarpX.H b/Source/WarpX.H index d7b7266d7b0..76b9b99e9a1 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -176,7 +176,7 @@ public: //! Flags whether the Picard iterations are required to converge static bool require_picard_convergence; /** Records a number corresponding to the load balance cost update strategy - * being used (0, 1, 2 corresponding to timers, heuristic, or gpuclock). + * being used (0 or 1 corresponding to timers or heuristic). */ static short load_balance_costs_update_algo; //! Integer that corresponds to electromagnetic Maxwell solver (vacuum - 0, macroscopic - 1) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index c929718b043..5caeb9ffb2e 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1320,10 +1320,6 @@ WarpX::ReadParameters () utils::parser::queryWithParser( pp_algo, "costs_heuristic_particles_wt", costs_heuristic_particles_wt); } -# ifndef WARPX_USE_GPUCLOCK - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(WarpX::load_balance_costs_update_algo!=LoadBalanceCostsUpdateAlgo::GpuClock, - "`algo.load_balance_costs_update = gpuclock` requires to compile with `-DWarpX_GPUCLOCK=ON`."); -# endif // !WARPX_USE_GPUCLOCK // Parse algo.particle_shape and check that input is acceptable // (do this only if there is at least one particle or laser species) diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index f43e35c6b0b..4d78e309f85 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -44,9 +44,6 @@ namespace ablastr::particles * \param np_to_deposit number of particles to deposit (default: pti.numParticles()) * \param icomp component in MultiFab to start depositing to * \param nc number of components to deposit - * \param cost pointer to (load balancing) cost corresponding to box where present - particles deposit current. If nullptr, costs are not updated. (default: nullptr) - * \param load_balance_costs_update_algo selected method for updating load balance costs (default: 0) * \param do_device_synchronize call amrex::Gpu::synchronize() for tiny profiler regions (default: true) */ template< typename T_PC > @@ -67,8 +64,6 @@ deposit_charge (typename T_PC::ParIterType& pti, long const offset = 0, std::optional np_to_deposit = std::nullopt, int const icomp = 0, int const nc = 1, - amrex::Real * const AMREX_RESTRICT cost = nullptr, - long const load_balance_costs_update_algo = 0, bool const do_device_synchronize = true) { // deposition guards @@ -183,23 +178,19 @@ deposit_charge (typename T_PC::ParIterType& pti, if (nox == 1){ doChargeDepositionShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } else if (nox == 2){ doChargeDepositionShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } else if (nox == 3){ doChargeDepositionShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } else if (nox == 4){ doChargeDepositionShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, - n_rz_azimuthal_modes, cost, - load_balance_costs_update_algo); + n_rz_azimuthal_modes); } ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd, do_device_synchronize); diff --git a/Tools/PerformanceTests/summit.py b/Tools/PerformanceTests/summit.py index cb0200e197c..c2ba6c70a2e 100644 --- a/Tools/PerformanceTests/summit.py +++ b/Tools/PerformanceTests/summit.py @@ -15,7 +15,7 @@ def executable_name(compiler,architecture): - return 'perf_tests3d.' + compiler + '.TPROF.MTMPI.CUDA.QED.GPUCLOCK.ex' + return 'perf_tests3d.' + compiler + '.TPROF.MTMPI.CUDA.QED.ex' def get_config_command(compiler, architecture): config_command = '' diff --git a/WarpXConfig.cmake b/WarpXConfig.cmake index 03225878700..5a2c0916432 100644 --- a/WarpXConfig.cmake +++ b/WarpXConfig.cmake @@ -17,7 +17,6 @@ set(WarpX_EB @WarpX_EB@) set(WarpX_EB_FOUND ${WarpX_EB}) set(WarpX_LIB @WarpX_LIB@) set(WarpX_LIB_FOUND ${WarpX_LIB}) -WarpX_GPUCLOCK # dependencies set(WarpX_MPI @WarpX_MPI@) diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index 6e0e2ca5d1d..9b9feb64437 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -444,7 +444,6 @@ function(warpx_print_summary) message(" COMPUTE: ${WarpX_COMPUTE}") message(" DIMS: ${WarpX_DIMS}") message(" Embedded Boundary: ${WarpX_EB}") - message(" GPU clock timers: ${WarpX_GPUCLOCK}") message(" IPO/LTO: ${WarpX_IPO}") message(" LIB: ${WarpX_LIB}${LIB_TYPE}") message(" MPI: ${WarpX_MPI}") From 5857c0462238b03b76bb8ab5126bda689d1a9b2d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 26 Mar 2024 03:30:31 -0700 Subject: [PATCH 013/190] AMReX/pyAMReX/PICSAR: Weekly Update (#4803) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index feb779d6ad4..739684f6c74 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach a8633589c1fae5b3fda6deca86fba79878a8cb54 && cd - + cd ../amrex && git checkout --detach 9e8f8d60eca337e3c14396f06f0a77500289d45c && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index d5ddbe2614d..8f7da493f24 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = a8633589c1fae5b3fda6deca86fba79878a8cb54 +branch = 9e8f8d60eca337e3c14396f06f0a77500289d45c [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 69b9f0d4b4d..4ac74d14310 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = a8633589c1fae5b3fda6deca86fba79878a8cb54 +branch = 9e8f8d60eca337e3c14396f06f0a77500289d45c [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 4c9a7bcb906..fdc628f3f06 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "a8633589c1fae5b3fda6deca86fba79878a8cb54" +set(WarpX_amrex_branch "9e8f8d60eca337e3c14396f06f0a77500289d45c" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 3242747b6f5..e2b83469010 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "e028359d5679e0775b4a8a36332c5d1d37fc7506" +set(WarpX_pyamrex_branch "bde190184effaf095f14e173df8f61f31ed67308" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 081efe21626..593d7bf07ca 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach a8633589c1fae5b3fda6deca86fba79878a8cb54 && cd - +cd amrex && git checkout --detach 9e8f8d60eca337e3c14396f06f0a77500289d45c && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 6b0a040d1289f31899c67c2182e4f558a8a8d9ba Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 26 Mar 2024 11:35:50 -0700 Subject: [PATCH 014/190] Cleaning: `.rst` Ending in TOC (#4804) Follow-up to #3137 --- Docs/source/usage/workflows.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index b68810a4345..039be199257 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -12,7 +12,7 @@ This section collects typical user workflows and best practices for WarpX. workflows/domain_decomposition workflows/plot_distribution_mapping workflows/debugging - workflows/generate_lookup_tables_with_tools.rst + workflows/generate_lookup_tables_with_tools workflows/libensemble workflows/plot_timestep_duration workflows/psatd_stencil From fdbf455f8a47ce0f160a9848551d2ab8f96ab5b3 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 28 Mar 2024 18:32:31 +0100 Subject: [PATCH 015/190] update fugaku_warpx.profile (#4808) --- Tools/machines/fugaku-riken/fugaku_warpx.profile.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/machines/fugaku-riken/fugaku_warpx.profile.example b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example index a430579a7d5..caf85983b58 100644 --- a/Tools/machines/fugaku-riken/fugaku_warpx.profile.example +++ b/Tools/machines/fugaku-riken/fugaku_warpx.profile.example @@ -10,7 +10,7 @@ export LD_LIBRARY_PATH=/lib64:$LD_LIBRARY_PATH spack load ninja@1.11.1%fj@4.10.0 # optional: for PSATD -spack load fujitsu-fftw +spack load fujitsu-fftw@1.1.0%fj@4.10.0 # optional: for QED lookup table generation support spack load boost@1.80.0%fj@4.8.1/zc5pwgc From 697fc5b37ee65d00483f968899e57a3049e442e6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 28 Mar 2024 13:09:06 -0700 Subject: [PATCH 016/190] Evolve Loop: Simplify (#4791) * Evolve Loop: Simplify Introduce structure functions to simplify the number of calls in our main evolve loop. * Code Review (Dave) Thank you! :) * Update Timer Cost Rescaling * simplify return logic Co-authored-by: Luca Fedeli * Rename: `HandleParticlesAtBoundaries` from `InjectRedistributeScapeParticles` * Fix Typo: scrape --------- Co-authored-by: Luca Fedeli Co-authored-by: David Grote --- Source/Evolve/WarpXEvolve.cpp | 278 +++++++++++++------------ Source/Parallelization/WarpXRegrid.cpp | 47 +++++ Source/WarpX.H | 46 ++++ 3 files changed, 237 insertions(+), 134 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index dd6acfd2694..a8aa89355ed 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -70,8 +70,8 @@ WarpX::Evolve (int numsteps) // Note that the default argument is numsteps = -1 const int numsteps_max = (numsteps < 0)?(max_step):(istep[0] + numsteps); - bool early_params_checked = false; // check typos in inputs after step 1 - bool exit_loop_due_to_interrupt_signal = false; + // check typos in inputs after step 1 + bool early_params_checked = false; static Real evolve_time = 0; @@ -81,7 +81,7 @@ WarpX::Evolve (int numsteps) WARPX_PROFILE("WarpX::Evolve::step"); const auto evolve_time_beg_step = static_cast(amrex::second()); - //Check and clear signal flags and asynchronously broadcast them from process 0 + // Check and clear signal flags and asynchronously broadcast them from process 0 SignalHandling::CheckSignals(); multi_diags->NewIteration(); @@ -92,90 +92,29 @@ WarpX::Evolve (int numsteps) } ExecutePythonCallback("beforestep"); - amrex::LayoutData* cost = WarpX::getCosts(0); - if (cost) { - if (step > 0 && load_balance_intervals.contains(step+1)) - { - LoadBalance(); + CheckLoadBalance(step); - // Reset the costs to 0 - ResetCosts(); - } - for (int lev = 0; lev <= finest_level; ++lev) - { - cost = WarpX::getCosts(lev); - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - // Perform running average of the costs - // (Giving more importance to most recent costs; only needed - // for timers update, heuristic load balance considers the - // instantaneous costs) - for (const auto& i : cost->IndexArray()) - { - (*cost)[i] *= (1._rt - 2._rt/load_balance_intervals.localPeriod(step+1)); - } - } - } - } - - if (evolve_scheme == EvolveScheme::Explicit) { - // At the beginning, we have B^{n} and E^{n}. - // Particles have p^{n} and x^{n}. - // is_synchronized is true. - - if (is_synchronized) { - // Not called at each iteration, so exchange all guard cells - FillBoundaryE(guard_cells.ng_alloc_EB); - FillBoundaryB(guard_cells.ng_alloc_EB); - - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - // on first step, push p by -0.5*dt - for (int lev = 0; lev <= finest_level; ++lev) - { - mypc->PushP(lev, -0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); - } - is_synchronized = false; - - } else { - // Beyond one step, we have E^{n} and B^{n}. - // Particles have p^{n-1/2} and x^{n}. - // E and B: enough guard cells to update Aux or call Field Gather in fp and cp - // Need to update Aux on lower levels, to interpolate to higher levels. - - // E and B are up-to-date inside the domain only - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { - if (fft_do_time_averaging) - { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); - } - // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? - if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - FillBoundaryAux(guard_cells.ng_UpdateAux); - } - } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - } + if (evolve_scheme == EvolveScheme::Explicit) + { + ExplicitFillBoundaryEBUpdateAux(); } // If needed, deposit the initial ion charge and current densities that // will be used to update the E-field in Ohm's law. if (step == step_begin && electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC - ) { HybridPICDepositInitialRhoAndJ(); } + ) { + HybridPICDepositInitialRhoAndJ(); + } // Run multi-physics modules: // ionization, Coulomb collisions, QED doFieldIonization(); + ExecutePythonCallback("beforecollisions"); mypc->doCollisions( cur_time, dt[0] ); ExecutePythonCallback("aftercollisions"); + #ifdef WARPX_QED doQEDEvents(); mypc->doQEDSchwinger(); @@ -186,6 +125,7 @@ WarpX::Evolve (int numsteps) ExecutePythonCallback("particleinjection"); + // TODO: move out if (evolve_scheme == EvolveScheme::ImplicitPicard || evolve_scheme == EvolveScheme::SemiImplicitPicard) { OneStep_ImplicitPicard(cur_time); @@ -229,12 +169,13 @@ WarpX::Evolve (int numsteps) mypc->doResampling(istep[0]+1, verbose); if (evolve_scheme == EvolveScheme::Explicit) { - if (num_mirrors>0){ - applyMirrors(cur_time); - // E : guard cells are NOT up-to-date - // B : guard cells are NOT up-to-date - } + applyMirrors(cur_time); + // E : guard cells are NOT up-to-date + // B : guard cells are NOT up-to-date + } + // TODO: move out + if (evolve_scheme == EvolveScheme::Explicit) { if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { // At the end of last step, push p by 0.5*dt to synchronize FillBoundaryE(guard_cells.ng_FieldGather); @@ -284,49 +225,7 @@ WarpX::Evolve (int numsteps) } } - mypc->ContinuousFluxInjection(cur_time, dt[0]); - - mypc->ApplyBoundaryConditions(); - - // interact the particles with EB walls (if present) -#ifdef AMREX_USE_EB - mypc->ScrapeParticles(amrex::GetVecOfConstPtrs(m_distance_to_eb)); -#endif - - m_particle_boundary_buffer->gatherParticles(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); - - // Non-Maxwell solver: particles can move by an arbitrary number of cells - if( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || - electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) - { - mypc->Redistribute(); - } - else - { - // Electromagnetic solver: due to CFL condition, particles can - // only move by one or two cells per time step - if (max_level == 0) { - int num_redistribute_ghost = num_moved; - if ((m_v_galilean[0]!=0) or (m_v_galilean[1]!=0) or (m_v_galilean[2]!=0)) { - // Galilean algorithm ; particles can move by up to 2 cells - num_redistribute_ghost += 2; - } else { - // Standard algorithm ; particles can move by up to 1 cell - num_redistribute_ghost += 1; - } - mypc->RedistributeLocal(num_redistribute_ghost); - } - else { - mypc->Redistribute(); - } - } - - if (sort_intervals.contains(step+1)) { - if (verbose) { - amrex::Print() << Utils::TextMsg::Info("re-sorting particles"); - } - mypc->SortParticlesByBin(sort_bin_size); - } + HandleParticlesAtBoundaries(step, cur_time, num_moved); // Field solve step for electrostatic or hybrid-PIC solvers if( electrostatic_solver_id != ElectrostaticSolverAlgo::None || @@ -378,12 +277,7 @@ WarpX::Evolve (int numsteps) // inputs: unused parameters (e.g. typos) check after step 1 has finished if (!early_params_checked) { - amrex::Print() << "\n"; // better: conditional \n based on return value - amrex::ParmParse::QueryUnusedInputs(); - - //Print the warning list right after the first step. - amrex::Print() << - ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("FIRST STEP"); + checkEarlyUnusedParams(); early_params_checked = true; } @@ -401,20 +295,18 @@ WarpX::Evolve (int numsteps) << " s; Avg. per step = " << evolve_time/(step-step_begin+1) << " s\n\n"; } - exit_loop_due_to_interrupt_signal = SignalHandling::TestAndResetActionRequestFlag(SignalHandling::SIGNAL_REQUESTS_BREAK); - if (cur_time >= stop_time - 1.e-3*dt[0] || exit_loop_due_to_interrupt_signal) { + if (checkStopSimulation(cur_time)) { break; } + } // End loop on time steps - // End loop on time steps - } // This if statement is needed for PICMI, which allows the Evolve routine to be // called multiple times, otherwise diagnostics will be done at every call, // regardless of the diagnostic period parameter provided in the inputs. if (istep[0] == max_step || (stop_time - 1.e-3*dt[0] <= cur_time && cur_time < stop_time + dt[0]) - || exit_loop_due_to_interrupt_signal) { + || m_exit_loop_due_to_interrupt_signal) { multi_diags->FilterComputePackFlushLastTimestep( istep[0] ); - if (exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } + if (m_exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } } } @@ -526,6 +418,119 @@ WarpX::OneStep_nosub (Real cur_time) ExecutePythonCallback("afterEsolve"); } +bool WarpX::checkStopSimulation (amrex::Real cur_time) +{ + m_exit_loop_due_to_interrupt_signal = SignalHandling::TestAndResetActionRequestFlag(SignalHandling::SIGNAL_REQUESTS_BREAK); + return (cur_time >= stop_time - 1.e-3*dt[0]) || + m_exit_loop_due_to_interrupt_signal; +} + +void WarpX::checkEarlyUnusedParams () +{ + amrex::Print() << "\n"; // better: conditional \n based on return value + amrex::ParmParse::QueryUnusedInputs(); + + // Print the warning list right after the first step. + amrex::Print() << ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("FIRST STEP"); +} + +void WarpX::ExplicitFillBoundaryEBUpdateAux () +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(evolve_scheme == EvolveScheme::Explicit, + "Cannot call WarpX::ExplicitFillBoundaryEBUpdateAux wihtout Explicit evolve scheme set!"); + + // At the beginning, we have B^{n} and E^{n}. + // Particles have p^{n} and x^{n}. + // is_synchronized is true. + + if (is_synchronized) { + // Not called at each iteration, so exchange all guard cells + FillBoundaryE(guard_cells.ng_alloc_EB); + FillBoundaryB(guard_cells.ng_alloc_EB); + + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + // on first step, push p by -0.5*dt + for (int lev = 0; lev <= finest_level; ++lev) + { + mypc->PushP(lev, -0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1],*Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1],*Bfield_aux[lev][2]); + } + is_synchronized = false; + + } else { + // Beyond one step, we have E^{n} and B^{n}. + // Particles have p^{n-1/2} and x^{n}. + // E and B: enough guard cells to update Aux or call Field Gather in fp and cp + // Need to update Aux on lower levels, to interpolate to higher levels. + + // E and B are up-to-date inside the domain only + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + if (electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + // TODO Remove call to FillBoundaryAux before UpdateAuxilaryData? + if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { + FillBoundaryAux(guard_cells.ng_UpdateAux); + } + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + } +} + +void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num_moved) +{ + mypc->ContinuousFluxInjection(cur_time, dt[0]); + + mypc->ApplyBoundaryConditions(); + + // interact the particles with EB walls (if present) +#ifdef AMREX_USE_EB + mypc->ScrapeParticles(amrex::GetVecOfConstPtrs(m_distance_to_eb)); +#endif + + m_particle_boundary_buffer->gatherParticles(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); + + // Non-Maxwell solver: particles can move by an arbitrary number of cells + if( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || + electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) + { + mypc->Redistribute(); + } + else + { + // Electromagnetic solver: due to CFL condition, particles can + // only move by one or two cells per time step + if (max_level == 0) { + int num_redistribute_ghost = num_moved; + if ((m_v_galilean[0]!=0) or (m_v_galilean[1]!=0) or (m_v_galilean[2]!=0)) { + // Galilean algorithm ; particles can move by up to 2 cells + num_redistribute_ghost += 2; + } else { + // Standard algorithm ; particles can move by up to 1 cell + num_redistribute_ghost += 1; + } + mypc->RedistributeLocal(num_redistribute_ghost); + } + else { + mypc->Redistribute(); + } + } + + if (sort_intervals.contains(step+1)) { + if (verbose) { + amrex::Print() << Utils::TextMsg::Info("re-sorting particles"); + } + mypc->SortParticlesByBin(sort_bin_size); + } +} + void WarpX::SyncCurrentAndRho () { if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) @@ -1054,8 +1059,13 @@ WarpX::PushParticlesandDeposit (int lev, amrex::Real cur_time, DtType a_dt_type, * The mirror normal direction has to be parallel to the z axis. */ void -WarpX::applyMirrors(Real time) +WarpX::applyMirrors (Real time) { + // something to do? + if (num_mirrors == 0) { + return; + } + // Loop over the mirrors for(int i_mirror=0; i_mirror 0 && load_balance_intervals.contains(step+1)) + { + LoadBalance(); + + // Reset the costs to 0 + ResetCosts(); + } + if (!costs.empty()) + { + RescaleCosts(step); + } +} + void WarpX::LoadBalance () { WARPX_PROFILE_REGION("LoadBalance"); WARPX_PROFILE("WarpX::LoadBalance()"); + AMREX_ALWAYS_ASSERT(!costs.empty()); AMREX_ALWAYS_ASSERT(costs[0] != nullptr); #ifdef AMREX_USE_MPI @@ -401,6 +418,9 @@ WarpX::ComputeCostsHeuristic (amrex::VectorIndexArray(); @@ -411,3 +431,30 @@ WarpX::ResetCosts () } } } + +void +WarpX::RescaleCosts (int step) +{ + // rescale is only used for timers + if (WarpX::load_balance_costs_update_algo != LoadBalanceCostsUpdateAlgo::Timers) + { + return; + } + + AMREX_ALWAYS_ASSERT(costs.size() == finest_level + 1); + + for (int lev = 0; lev <= finest_level; ++lev) + { + if (costs[lev]) + { + // Perform running average of the costs + // (Giving more importance to most recent costs; only needed + // for timers update, heuristic load balance considers the + // instantaneous costs) + for (const auto& i : costs[lev]->IndexArray()) + { + (*costs[lev])[i] *= (1._rt - 2._rt/load_balance_intervals.localPeriod(step+1)); + } + } + } +} diff --git a/Source/WarpX.H b/Source/WarpX.H index 76b9b99e9a1..d9ca494e72b 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -664,13 +664,26 @@ public: static amrex::Real quantum_xi_c2; + /** Check and potentially compute load balancing + */ + void CheckLoadBalance (int step); + /** \brief perform load balance; compute and communicate new `amrex::DistributionMapping` */ void LoadBalance (); + /** \brief resets costs to zero */ void ResetCosts (); + /** Perform running average of the LB costs + * + * Only needed for timers cost update, heuristic load balance considers the + * instantaneous costs. + * This gives more importance to most recent costs. + */ + void RescaleCosts (int step); + /** \brief returns the load balance interval */ [[nodiscard]] utils::parser::IntervalsParser get_load_balance_intervals () const @@ -1723,8 +1736,41 @@ private: return *m_field_factory[lev]; } + /** Stop the simulation at the end of the current step due to a received Unix signal? + */ + bool m_exit_loop_due_to_interrupt_signal = false; + + /** Stop the simulation at the end of the current step? + */ + [[nodiscard]] + bool checkStopSimulation (amrex::Real cur_time); + + /** Print Unused Parameter Warnings after Step 1 + * + * Instead of waiting for a simulation to end, we already do an early "unused parameter check" + * after step 1 to inform users early of potential issues with their simulation setup. + */ + void checkEarlyUnusedParams (); + + /** Perform essential particle house keeping at boundaries + * + * Inject, communicate, scrape and sort particles. + * + * @param step current step + * @param cur_time current time + * @param num_moved number of cells the moving window moved + */ + void HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num_moved); + void ScrapeParticles (); + /** Update the E and B fields in the explicit em PIC scheme. + * + * At the beginning, we have B^{n} and E^{n}. + * Particles have p^{n} and x^{n}. + */ + void ExplicitFillBoundaryEBUpdateAux (); + void PushPSATD (); #ifdef WARPX_USE_PSATD From 5cfa8f13f32c2a3c678e33ec230395777af49df0 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Fri, 29 Mar 2024 19:34:16 +0100 Subject: [PATCH 017/190] fix typos in scripts to run WarpX on Frontier (#4811) --- Tools/machines/frontier-olcf/install_dependencies.sh | 2 +- Tools/machines/frontier-olcf/submit.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 9460f9a5175..98c30cfca8f 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -14,7 +14,7 @@ set -eu -o pipefail # Check: ###################################################################### # -# Was perlmutter_gpu_warpx.profile sourced and configured correctly? +# Was frontier_warpx.profile sourced and configured correctly? if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your frontier_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi diff --git a/Tools/machines/frontier-olcf/submit.sh b/Tools/machines/frontier-olcf/submit.sh index 13d981238a3..b340d56546e 100644 --- a/Tools/machines/frontier-olcf/submit.sh +++ b/Tools/machines/frontier-olcf/submit.sh @@ -32,7 +32,7 @@ export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor # Seen since August 2023 -# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors # https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors export MPICH_SMP_SINGLE_COPY_MODE=NONE export FI_CXI_RX_MATCH_MODE=software From 72800eab2b889bb67b5c90f14929b679c715c34c Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Sat, 30 Mar 2024 19:26:03 -0700 Subject: [PATCH 018/190] Add thermalBC for particles (#4790) * add thermal as particle boundary type * separate file for gaussian distribution function * parse and apply thermal boundary * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix typo * fixing unused var * isAnyThermal as static fctn is warpx class, instead of particle boundaries struct * unused var for 1D, 2D, RZ * change API for u_th * add thermal for PEC * file name change * Update Source/Particles/ParticleBoundaries_K.H Co-authored-by: David Grote * restructure rethermalize within ifdef for apply boundary * add CI test * doc * fix py file name * condition for 0 uth * typo * fix path for reduceddiags * no chksum too noisy * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * addchksum * import os * Update Examples/Tests/particle_thermal_boundary/analysis_2d.py * Update Source/Particles/ParticleBoundaries_K.H --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: David Grote Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 6 ++ .../particle_thermal_boundary/analysis_2d.py | 37 +++++++++ .../Tests/particle_thermal_boundary/inputs_2d | 77 ++++++++++++++++++ .../particle_thermal_boundary.json | 12 +++ Regression/WarpX-tests.ini | 17 ++++ .../WarpXFieldBoundaries.cpp | 6 +- Source/BoundaryConditions/WarpX_PEC.cpp | 22 +++-- Source/Initialization/InjectorMomentum.H | 70 +--------------- .../SampleGaussianFluxDistribution.H | 81 +++++++++++++++++++ Source/Particles/ParticleBoundaries.H | 3 + Source/Particles/ParticleBoundaries.cpp | 7 ++ Source/Particles/ParticleBoundaries_K.H | 53 ++++++++++-- .../Particles/PhysicalParticleContainer.cpp | 8 ++ Source/Utils/WarpXAlgorithmSelection.H | 3 +- Source/Utils/WarpXAlgorithmSelection.cpp | 1 + Source/WarpX.H | 6 +- Source/WarpX.cpp | 10 +++ 17 files changed, 332 insertions(+), 87 deletions(-) create mode 100755 Examples/Tests/particle_thermal_boundary/analysis_2d.py create mode 100644 Examples/Tests/particle_thermal_boundary/inputs_2d create mode 100644 Regression/Checksum/benchmarks_json/particle_thermal_boundary.json create mode 100644 Source/Initialization/SampleGaussianFluxDistribution.H diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index fc6d338fc1c..fa1d5e99670 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -417,6 +417,12 @@ Domain Boundary Conditions * ``Reflecting``: Particles leaving the boundary are reflected from the boundary back into the domain. When ``boundary.reflect_all_velocities`` is false, the sign of only the normal velocity is changed, otherwise the sign of all velocities are changed. + * ``Thermal``: Particles leaving the boundary are reflected from the boundary back into the domain + and their velocities are thermalized. The tangential velocity components are sampled from ``gaussian`` distribution + and the component normal to the boundary is sampled from ``gaussian flux`` distribution. + The standard deviation for these distributions should be provided for each species using + ``boundary..u_th``. The same standard deviation is used to sample all components. + * ``boundary.reflect_all_velocities`` (`bool`) optional (default `false`) For a reflecting boundary condition, this flags whether the sign of only the normal velocity is changed or all velocities. diff --git a/Examples/Tests/particle_thermal_boundary/analysis_2d.py b/Examples/Tests/particle_thermal_boundary/analysis_2d.py new file mode 100755 index 00000000000..db14479af2c --- /dev/null +++ b/Examples/Tests/particle_thermal_boundary/analysis_2d.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# +# Copyright 2023-2024 Revathi Jambunathan +# +# This file is part of WarpX +# +# License: BSD-3-Clause-LBNL + +""" +The script checks to see the growth in total field energy and total particle energy + +The input file in 2D initializes uniform plasma (electrons,ions) with +thermal boundary condition. We do not expect the particle energy to increase +beyond 2% in the time that it takes all particles to cross the domain boundary +""" + +import os +import sys + +import numpy as np + +sys.path.insert(1,'../../../../warpx/Regression/Checksum/') +import checksumAPI + +FE_rdiag = './diags/reducedfiles/EF.txt' +init_Fenergy = np.loadtxt(FE_rdiag)[1,2] +final_Fenergy = np.loadtxt(FE_rdiag)[-1,2] +assert(final_Fenergy/init_Fenergy < 40) +assert(final_Fenergy < 5.e-5) + +PE_rdiag = './diags/reducedfiles/EN.txt' +init_Penergy = np.loadtxt(PE_rdiag)[0,2] +final_Penergy = np.loadtxt(PE_rdiag)[-1,2] +assert( abs(final_Penergy - init_Penergy)/init_Penergy < 0.02) +filename = sys.argv[1] +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/particle_thermal_boundary/inputs_2d b/Examples/Tests/particle_thermal_boundary/inputs_2d new file mode 100644 index 00000000000..9c3e18cd35d --- /dev/null +++ b/Examples/Tests/particle_thermal_boundary/inputs_2d @@ -0,0 +1,77 @@ +max_step = 2000 + +# number of grid points +amr.n_cell = 16 16 + +# Maximum allowable size of each subdomain in the problem domain; +# this is used to decompose the domain for parallel calculations. +amr.max_grid_size = 128 + +# Maximum level in hierarchy (for now must be 0, i.e., one level in total) +amr.max_level = 0 + +# Geometry +geometry.dims = 2 +geometry.prob_lo = 0.e-6 0.e-6 # physical domain +geometry.prob_hi = 2.5e-7 2.5e-7 + +# Boundary condition +boundary.field_lo = pml pml +boundary.field_hi = pml pml +boundary.particle_lo = thermal thermal +boundary.particle_hi = thermal thermal +boundary.electrons.u_th = uth_e +boundary.C.u_th = uth_C +warpx.do_dive_cleaning = 0 + +# Verbosity +warpx.verbose = 1 + +# Order of particle shape factors +algo.particle_shape = 2 + +# CFL +warpx.cfl = 0.98 + +# Density +my_constants.n0 = 4e26 +my_constants.uth_e = 0.06256112470898544 +my_constants.uth_C = 0.00042148059678527106 + +# Particles +particles.species_names = electrons C + +electrons.charge = -q_e +electrons.mass = m_e +electrons.injection_style = "NUniformPerCell" +electrons.num_particles_per_cell_each_dim = 8 8 +electrons.profile = constant +electrons.density = n0 +electrons.momentum_distribution_type = gaussian +electrons.ux_th = uth_e +electrons.uy_th = uth_e +electrons.uz_th = uth_e + +C.charge = 6*q_e +C.mass = 12*m_p +C.injection_style = "NUniformPerCell" +C.num_particles_per_cell_each_dim = 8 8 +C.profile = constant +C.density = n0/6 +C.momentum_distribution_type = gaussian +C.ux_th = uth_C +C.uy_th = uth_C +C.uz_th = uth_C + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 3000 +diag1.write_species = 0 +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE +diag1.diag_type = Full + +warpx.reduced_diags_names = EN EF +EN.intervals = 10 +EN.type = ParticleEnergy +EF.intervals = 10 +EF.type = FieldEnergy diff --git a/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json b/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json new file mode 100644 index 00000000000..5c302d4c5b3 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json @@ -0,0 +1,12 @@ +{ + "lev=0": { + "Bx": 167.49421217759414, + "By": 269.6793406439766, + "Bz": 258.3282607081799, + "Ex": 864657516595.9229, + "Ey": 24072965021.088013, + "Ez": 702786324880.7338, + "divE": 5.137699003981513e+19, + "rho": 357057125.6699786 + } +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 4ac74d14310..42274fa30ae 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4720,3 +4720,20 @@ compareParticles = 1 particleTypes = electrons outputFile = particle_boundary_interaction_plt analysisRoutine = Examples/Tests/particle_boundary_interaction/analysis.py + +[particle_thermal_boundary] +buildDir = . +inputFile = Examples/Tests/particle_thermal_boundary/inputs_2d +runtime_params = +dim = 2 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index a9e2cf29fe9..4844585cc90 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -106,7 +106,8 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, PatchType patch_type) { - if (PEC::isAnyParticleBoundaryReflecting() || PEC::isAnyBoundaryPEC()) { + if (PEC::isAnyParticleBoundaryReflecting() || WarpX::isAnyParticleBoundaryThermal() || PEC::isAnyBoundaryPEC()) + { PEC::ApplyReflectiveBoundarytoRhofield(rho, lev, patch_type); } } @@ -115,7 +116,8 @@ void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type) { - if (PEC::isAnyParticleBoundaryReflecting() || PEC::isAnyBoundaryPEC()) { + if (PEC::isAnyParticleBoundaryReflecting() || WarpX::isAnyParticleBoundaryThermal() || PEC::isAnyBoundaryPEC()) + { PEC::ApplyReflectiveBoundarytoJfield(Jx, Jy, Jz, lev, patch_type); } } diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index 0b4b4dae30a..a3d92fa507d 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -258,8 +258,10 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat amrex::GpuArray, AMREX_SPACEDIM> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_reflective[idim][0] = ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) || ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC); is_reflective[idim][1] = ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) || ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC); if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -268,9 +270,11 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat // components of the current density is_tangent_to_bndy[idim] = true; - psign[idim][0] = (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + psign[idim][0] = ( ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) ? 1._rt : -1._rt; - psign[idim][1] = (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + psign[idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) ? 1._rt : -1._rt; mirrorfac[idim][0] = 2*domain_lo[idim] - (1 - rho_nodal[idim]); mirrorfac[idim][1] = 2*domain_hi[idim] + (1 - rho_nodal[idim]); @@ -352,8 +356,10 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::GpuArray, AMREX_SPACEDIM>, 3> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { is_reflective[idim][0] = ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) || ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC); is_reflective[idim][1] = ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) || ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC); if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -375,15 +381,19 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, #endif if (is_tangent_to_bndy[icomp][idim]){ - psign[icomp][idim][0] = (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][0] = ( (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) ? 1._rt : -1._rt; - psign[icomp][idim][1] = (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) ? 1._rt : -1._rt; } else { - psign[icomp][idim][0] = (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][0] = ( (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) ? -1._rt : 1._rt; - psign[icomp][idim][1] = (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + psign[icomp][idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) ? -1._rt : 1._rt; } } diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index f7ee6cff138..2b8b9065a36 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -12,6 +12,7 @@ #include "GetVelocity.H" #include "TemperatureProperties.H" #include "VelocityProperties.H" +#include "SampleGaussianFluxDistribution.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" @@ -90,75 +91,6 @@ private: amrex::Real m_ux_th, m_uy_th, m_uz_th; }; -namespace { - /** Return u sampled according to the probability distribution: - * p(u) \propto u \exp(-(u-u_m)^2/2u_th^2) - * - * @param u_m Central momentum - * @param u_th Momentum spread - * @param engine Object used to generate random numbers - */ - [[nodiscard]] - AMREX_FORCE_INLINE - AMREX_GPU_HOST_DEVICE - amrex::Real - generateGaussianFluxDist( amrex::Real u_m, amrex::Real u_th, amrex::RandomEngine const& engine ) { - - using namespace amrex::literals; - - // Momentum to be returned at the end of this function - amrex::Real u = 0._rt; - - const amrex::Real abs_u_m = std::abs(u_m); - - if (u_th == 0._rt) { - u = u_m; // Trivial case ; avoids division by 0 in the rest of the code below - } else if (abs_u_m < 0.6*u_th) { - // Mean velocity magnitude is less than thermal velocity - // Use the distribution u*exp(-u**2*(1-abs(u_m)/u_th)/(2*u_th**2)) as an approximation - // and then use the rejection method to correct it - // ( stop rejecting with probability exp(-abs(u_m)/(2*u_th**3)*(u-sign(u_m)*u_th)**2) ) - // Note that this is the method that is used in the common case u_m=0 - const amrex::Real umsign = std::copysign(1._rt, u_m); - const amrex::Real approx_u_th = u_th/std::sqrt( 1._rt - abs_u_m/u_th ); - const amrex::Real reject_prefactor = (abs_u_m/u_th)/(2._rt*u_th*u_th); // To save computation - bool reject = true; - while (reject) { - // Generates u according to u*exp(-u**2/(2*approx_u_th**2)), - // using the method of the inverse cumulative function - amrex::Real xrand = 1._rt - amrex::Random(engine); // ensures urand > 0 - u = approx_u_th * std::sqrt(2._rt*std::log(1._rt/xrand)); - // Rejection method - xrand = amrex::Random(engine); - if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) { reject = false; } - } - } else { - // Mean velocity magnitude is greater than thermal velocity - // Use the distribution exp(-(u-u_m-u_th**2/abs(u_m))**2/(2*u_th**2)) as an approximation - // and then use the rejection method to correct it - // ( stop rejecting with probability (u/abs(u_m))*exp(1-(u/abs(u_m))) ; note - // that this number is always between 0 and 1 ) - // Note that in the common case `u_m = 0`, this rejection method - // is not used, and the above rejection method is used instead. - bool reject = true; - const amrex::Real approx_u_m = u_m + u_th*u_th/abs_u_m; - const amrex::Real inv_um = 1._rt/abs_u_m; // To save computation - while (reject) { - // Approximate distribution: normal distribution, where we only retain positive u - u = -1._rt; - while (u < 0) { - u = amrex::RandomNormal(approx_u_m, u_th, engine); - } - // Rejection method - const amrex::Real xrand = amrex::Random(engine); - if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) { reject = false; } - } - } - - return u; - } -} - // struct whose getMomentum returns momentum for 1 particle, from random // gaussian flux distribution in the specified direction. // Along the normal axis, the distribution is v*Gaussian, diff --git a/Source/Initialization/SampleGaussianFluxDistribution.H b/Source/Initialization/SampleGaussianFluxDistribution.H new file mode 100644 index 00000000000..407c9df632a --- /dev/null +++ b/Source/Initialization/SampleGaussianFluxDistribution.H @@ -0,0 +1,81 @@ +/* Copyright 2024 Remi Lehe, Revathi Jambunathan + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H +#define SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H + +#include + +namespace { + /** This function returns u sampled according to the probability distribution: + * p(u) \propto u \exp(-(u-u_m)^2/2u_th^2) + * + * @param u_m Central momentum + * @param u_th Momentum spread + * @param engine Object used to generate random numbers + */ + [[nodiscard]] + AMREX_FORCE_INLINE + AMREX_GPU_HOST_DEVICE + amrex::Real + generateGaussianFluxDist( amrex::Real u_m, amrex::Real u_th, amrex::RandomEngine const& engine ) { + + using namespace amrex::literals; + + // Momentum to be returned at the end of this function + amrex::Real u = 0._rt; + + const amrex::Real abs_u_m = std::abs(u_m); + + if (u_th == 0._rt) { + u = u_m; // Trivial case ; avoids division by 0 in the rest of the code below + } else if (abs_u_m < 0.6*u_th) { + // Mean velocity magnitude is less than thermal velocity + // Use the distribution u*exp(-u**2*(1-abs(u_m)/u_th)/(2*u_th**2)) as an approximation + // and then use the rejection method to correct it + // ( stop rejecting with probability exp(-abs(u_m)/(2*u_th**3)*(u-sign(u_m)*u_th)**2) ) + // Note that this is the method that is used in the common case u_m=0 + const amrex::Real umsign = std::copysign(1._rt, u_m); + const amrex::Real approx_u_th = u_th/std::sqrt( 1._rt - abs_u_m/u_th ); + const amrex::Real reject_prefactor = (abs_u_m/u_th)/(2._rt*u_th*u_th); // To save computation + bool reject = true; + while (reject) { + // Generates u according to u*exp(-u**2/(2*approx_u_th**2)), + // using the method of the inverse cumulative function + amrex::Real xrand = 1._rt - amrex::Random(engine); // ensures urand > 0 + u = approx_u_th * std::sqrt(2._rt*std::log(1._rt/xrand)); + // Rejection method + xrand = amrex::Random(engine); + if (xrand < std::exp(-reject_prefactor*(u - umsign*u_th)*(u - umsign*u_th))) { reject = false; } + } + } else { + // Mean velocity magnitude is greater than thermal velocity + // Use the distribution exp(-(u-u_m-u_th**2/abs(u_m))**2/(2*u_th**2)) as an approximation + // and then use the rejection method to correct it + // ( stop rejecting with probability (u/abs(u_m))*exp(1-(u/abs(u_m))) ; note + // that this number is always between 0 and 1 ) + // Note that in the common case `u_m = 0`, this rejection method + // is not used, and the above rejection method is used instead. + bool reject = true; + const amrex::Real approx_u_m = u_m + u_th*u_th/abs_u_m; + const amrex::Real inv_um = 1._rt/abs_u_m; // To save computation + while (reject) { + // Approximate distribution: normal distribution, where we only retain positive u + u = -1._rt; + while (u < 0) { + u = amrex::RandomNormal(approx_u_m, u_th, engine); + } + // Rejection method + const amrex::Real xrand = amrex::Random(engine); + if (xrand < u*inv_um* std::exp(1._rt - u*inv_um)) { reject = false; } + } + } + return u; + } +} + +#endif diff --git a/Source/Particles/ParticleBoundaries.H b/Source/Particles/ParticleBoundaries.H index 78b090056b1..c056436770d 100644 --- a/Source/Particles/ParticleBoundaries.H +++ b/Source/Particles/ParticleBoundaries.H @@ -22,6 +22,8 @@ struct ParticleBoundaries void Set_reflect_all_velocities (bool flag); void SetAll (ParticleBoundaryType bc); + /** Sets thermal velocity in ParticleBoundariesData 'data.m_uth' to u_th */ + void SetThermalVelocity (amrex::Real u_th); void SetBoundsX (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); void SetBoundsY (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi); @@ -54,6 +56,7 @@ struct ParticleBoundaries ParticleBoundaryType ymax_bc; ParticleBoundaryType zmin_bc; ParticleBoundaryType zmax_bc; + amrex::Real m_uth = 0.; amrex::ParserExecutor<1> reflection_model_xlo; amrex::ParserExecutor<1> reflection_model_xhi; diff --git a/Source/Particles/ParticleBoundaries.cpp b/Source/Particles/ParticleBoundaries.cpp index 5b51fa3fd25..10268a9c01f 100644 --- a/Source/Particles/ParticleBoundaries.cpp +++ b/Source/Particles/ParticleBoundaries.cpp @@ -6,6 +6,7 @@ */ #include "ParticleBoundaries.H" +#include "WarpX.H" #include "Utils/Parser/ParserUtils.H" @@ -32,6 +33,12 @@ ParticleBoundaries::SetAll (ParticleBoundaryType bc) data.zmax_bc = bc; } +void +ParticleBoundaries::SetThermalVelocity (amrex::Real u_th) +{ + data.m_uth = u_th; +} + void ParticleBoundaries::SetBoundsX (ParticleBoundaryType bc_lo, ParticleBoundaryType bc_hi) { diff --git a/Source/Particles/ParticleBoundaries_K.H b/Source/Particles/ParticleBoundaries_K.H index bbec1f54e01..1481dca7684 100644 --- a/Source/Particles/ParticleBoundaries_K.H +++ b/Source/Particles/ParticleBoundaries_K.H @@ -1,4 +1,4 @@ -/* Copyright 2021 David Grote +/* Copyright 2021 David Grote, Remi Lehe, Revathi Jambunathan * * This file is part of WarpX. * @@ -8,18 +8,19 @@ #define PARTICLEBOUNDARIES_K_H_ #include "ParticleBoundaries.H" +#include "Initialization/SampleGaussianFluxDistribution.H" #include namespace ApplyParticleBoundaries { - + using namespace amrex::literals; /* \brief Applies the boundary condition on a specific axis * This is called by apply_boundaries. */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void apply_boundary (amrex::ParticleReal& x, amrex::Real xmin, amrex::Real xmax, - bool& change_sign_ux, bool& particle_lost, + bool& change_sign_ux, bool& rethermalize_x, bool& particle_lost, ParticleBoundaryType xmin_bc, ParticleBoundaryType xmax_bc, amrex::Real refl_probability_xmin, amrex::Real refl_probability_xmax, amrex::RandomEngine const& engine ) @@ -42,6 +43,10 @@ namespace ApplyParticleBoundaries { x = 2*xmin - x; change_sign_ux = true; } + else if (xmin_bc == ParticleBoundaryType::Thermal) { + x = 2*xmin - x; + rethermalize_x = true; + } } else if (x > xmax) { if (xmax_bc == ParticleBoundaryType::Open) { @@ -61,16 +66,38 @@ namespace ApplyParticleBoundaries { x = 2*xmax - x; change_sign_ux = true; } + else if (xmax_bc == ParticleBoundaryType::Thermal) { + x = 2*xmax - x; + rethermalize_x = true; + } } } - /* \brief Applies absorbing or reflecting boundary condition to the input particles, along all axis. + /* \brief Thermalize particles that have been identified to cross the boundary. + * The normal component samples from a half-Maxwellian, + * and the two tangential components will sample from full Maxwellian disbutions + * with thermal velocity uth + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void thermalize_boundary_particle (amrex::ParticleReal& u_norm, amrex::ParticleReal& u_tang1, + amrex::ParticleReal& u_tang2, amrex::Real uth, + amrex::RandomEngine const& engine) + { + u_tang1 = (uth > 0._rt) ? PhysConst::c * amrex::RandomNormal(0._rt, uth, engine) : 0._rt; + u_tang2 = (uth > 0._rt) ? PhysConst::c * amrex::RandomNormal(0._rt, uth, engine) : 0._rt; + u_norm = (uth > 0._rt) ? std::copysign(1._prt, -u_norm) * PhysConst::c * generateGaussianFluxDist(0._rt, uth, engine) : 0._rt; + } + + + /* \brief Applies absorbing, reflecting or thermal boundary condition to the input particles, along all axis. * For reflecting boundaries, the position of the particle is changed appropriately and * the sign of the velocity is changed (depending on the reflect_all_velocities flag). * For absorbing, a flag is set whether the particle has been lost (it is up to the calling * code to take appropriate action to remove any lost particles). Absorbing boundaries can * be given a reflection coefficient for stochastic reflection of particles, this * coefficient is zero by default. + * For thermal boundaries, the particle is first reflected and the position of the particle + * is changed appropriately. * Note that periodic boundaries are handled in AMReX code. * * \param x, xmin, xmax: particle x position, location of x boundary @@ -103,21 +130,33 @@ namespace ApplyParticleBoundaries { bool change_sign_uz = false; #ifndef WARPX_DIM_1D_Z - apply_boundary(x, xmin, xmax, change_sign_ux, particle_lost, + bool rethermalize_x = false; // stores if particle crosses x boundary and needs to be thermalized + apply_boundary(x, xmin, xmax, change_sign_ux, rethermalize_x, particle_lost, boundaries.xmin_bc, boundaries.xmax_bc, boundaries.reflection_model_xlo(-ux), boundaries.reflection_model_xhi(ux), engine); + if (rethermalize_x) { + thermalize_boundary_particle(ux, uy, uz, boundaries.m_uth, engine); + } #endif #ifdef WARPX_DIM_3D - apply_boundary(y, ymin, ymax, change_sign_uy, particle_lost, + bool rethermalize_y = false; // stores if particle crosses y boundary and needs to be thermalized + apply_boundary(y, ymin, ymax, change_sign_uy, rethermalize_y, particle_lost, boundaries.ymin_bc, boundaries.ymax_bc, boundaries.reflection_model_ylo(-uy), boundaries.reflection_model_yhi(uy), engine); + if (rethermalize_y) { + thermalize_boundary_particle(uy, uz, ux, boundaries.m_uth, engine); + } #endif - apply_boundary(z, zmin, zmax, change_sign_uz, particle_lost, + bool rethermalize_z = false; // stores if particle crosses z boundary and needs to be thermalized + apply_boundary(z, zmin, zmax, change_sign_uz, rethermalize_z, particle_lost, boundaries.zmin_bc, boundaries.zmax_bc, boundaries.reflection_model_zlo(-uz), boundaries.reflection_model_zhi(uz), engine); + if (rethermalize_z) { + thermalize_boundary_particle(uz, ux, uy, boundaries.m_uth, engine); + } if (boundaries.reflect_all_velocities && (change_sign_ux | change_sign_uy | change_sign_uz)) { change_sign_ux = true; diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 06154958dde..4adb5386a2a 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -446,6 +446,14 @@ PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core, int isp pp_boundary.query("reflect_all_velocities", flag); m_boundary_conditions.Set_reflect_all_velocities(flag); + // currently supports only isotropic thermal distribution + // same distribution is applied to all boundaries + const amrex::ParmParse pp_species_boundary("boundary." + species_name); + if (WarpX::isAnyParticleBoundaryThermal()) { + amrex::Real boundary_uth; + utils::parser::getWithParser(pp_species_boundary,"u_th",boundary_uth); + m_boundary_conditions.SetThermalVelocity(boundary_uth); + } } PhysicalParticleContainer::PhysicalParticleContainer (AmrCore* amr_core) diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index b7af3121814..dd1ec66e051 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -160,7 +160,8 @@ enum struct ParticleBoundaryType { Absorbing = 0, //!< particles crossing domain boundary are removed Open = 1, //!< particles cross domain boundary leave with damped j Reflecting = 2, //!< particles are reflected - Periodic = 3 + Periodic = 3, //!< particles are introduced from the periodic boundary + Thermal = 4 }; /** MPI reductions diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 3092e1984d6..54c91e5eb15 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -134,6 +134,7 @@ const std::map ParticleBCType_algo_to_enum = {"open", ParticleBoundaryType::Open}, {"reflecting", ParticleBoundaryType::Reflecting}, {"periodic", ParticleBoundaryType::Periodic}, + {"thermal", ParticleBoundaryType::Thermal}, {"default", ParticleBoundaryType::Absorbing} }; diff --git a/Source/WarpX.H b/Source/WarpX.H index d9ca494e72b..1a56445f8dd 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -197,12 +197,12 @@ public: static amrex::Vector field_boundary_hi; /** Integers that correspond to boundary condition applied to particles at the * lower domain boundaries - * (0 to 3 correspond to Absorbing, Open, Reflecting, Periodic) + * (0 to 4 correspond to Absorbing, Open, Reflecting, Periodic, Thermal) */ static amrex::Vector particle_boundary_lo; /** Integers that correspond to boundary condition applied to particles at the * upper domain boundaries - * (0 to 3 correspond to Absorbing, Open, Reflecting, Periodic) + * (0 to 4 correspond to Absorbing, Open, Reflecting, Periodic, Thermal) */ static amrex::Vector particle_boundary_hi; @@ -767,6 +767,8 @@ public: void CopyJPML (); bool isAnyBoundaryPML(); + /** True if any of the particle boundary condition type is Thermal */ + static bool isAnyParticleBoundaryThermal(); PML* GetPML (int lev); #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 5caeb9ffb2e..83f48394400 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3277,6 +3277,16 @@ WarpX::isAnyBoundaryPML() return false; } +bool +WarpX::isAnyParticleBoundaryThermal () +{ + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) {return true;} + if (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) {return true;} + } + return false; +} + std::string TagWithLevelSuffix (std::string name, int const level) { From a5d5432ddbd9a3e53389cc6c47c02c2459ca9192 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 1 Apr 2024 22:20:56 +0200 Subject: [PATCH 019/190] Clang tidy CI test: add performance-unnecessary-* checks to clang-tidy CI test (#4136) * clang-tidy CI test: add performance-unnecessary-* checks * fixed issues * address issues found with clang-tidy * address issues found with clang-tidy * change name for clarity * fixed bug * address issues found with clang-tidy * fix issue found with clang-tidy * fix bug * remove file apparently added due to merge conflict * fix bug * fix bug * addressed all issues found with clang-tidy * fix issues found with clang-tidy * revert space deletion * fix variable name * fix issues found with clang-tidy * fix some residual issues found with clang-tidy * Update Source/Diagnostics/BTDiagnostics.H Co-authored-by: David Grote --------- Co-authored-by: David Grote --- .clang-tidy | 2 -- Source/Diagnostics/BTD_Plotfile_Header_Impl.H | 6 ++-- .../Diagnostics/BTD_Plotfile_Header_Impl.cpp | 6 ++-- Source/Diagnostics/BTDiagnostics.H | 22 +++++++-------- Source/Diagnostics/BTDiagnostics.cpp | 24 ++++++++-------- .../Diagnostics/BoundaryScrapingDiagnostics.H | 2 +- .../BoundaryScrapingDiagnostics.cpp | 4 +-- .../ParticleReductionFunctor.H | 4 +-- .../ParticleReductionFunctor.cpp | 4 +-- .../FlushFormats/FlushFormatPlotfile.cpp | 4 +-- .../FlushFormats/FlushFormatSensei.H | 2 +- .../FlushFormats/FlushFormatSensei.cpp | 2 +- Source/Diagnostics/FullDiagnostics.H | 2 +- Source/Diagnostics/FullDiagnostics.cpp | 2 +- .../Diagnostics/ParticleDiag/ParticleDiag.H | 3 +- .../Diagnostics/ParticleDiag/ParticleDiag.cpp | 6 ++-- .../Diagnostics/ReducedDiags/BeamRelevant.H | 2 +- .../Diagnostics/ReducedDiags/BeamRelevant.cpp | 2 +- Source/Diagnostics/ReducedDiags/ChargeOnEB.H | 2 +- .../Diagnostics/ReducedDiags/ChargeOnEB.cpp | 2 +- .../ReducedDiags/ColliderRelevant.H | 2 +- .../ReducedDiags/ColliderRelevant.cpp | 4 +-- Source/Diagnostics/ReducedDiags/FieldEnergy.H | 2 +- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldMaximum.H | 2 +- .../Diagnostics/ReducedDiags/FieldMaximum.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldMomentum.H | 2 +- .../ReducedDiags/FieldMomentum.cpp | 2 +- Source/Diagnostics/ReducedDiags/FieldProbe.H | 2 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 2 +- .../Diagnostics/ReducedDiags/FieldReduction.H | 2 +- .../ReducedDiags/FieldReduction.cpp | 2 +- .../ReducedDiags/LoadBalanceCosts.H | 2 +- .../ReducedDiags/LoadBalanceCosts.cpp | 2 +- .../ReducedDiags/LoadBalanceEfficiency.H | 2 +- .../ReducedDiags/LoadBalanceEfficiency.cpp | 2 +- .../Diagnostics/ReducedDiags/ParticleEnergy.H | 2 +- .../ReducedDiags/ParticleEnergy.cpp | 2 +- .../ReducedDiags/ParticleExtrema.H | 2 +- .../ReducedDiags/ParticleExtrema.cpp | 2 +- .../ReducedDiags/ParticleHistogram.H | 2 +- .../ReducedDiags/ParticleHistogram.cpp | 4 +-- .../ReducedDiags/ParticleHistogram2D.H | 2 +- .../ReducedDiags/ParticleHistogram2D.cpp | 2 +- .../ReducedDiags/ParticleMomentum.H | 2 +- .../ReducedDiags/ParticleMomentum.cpp | 2 +- .../Diagnostics/ReducedDiags/ParticleNumber.H | 2 +- .../ReducedDiags/ParticleNumber.cpp | 2 +- .../Diagnostics/ReducedDiags/ReducedDiags.H | 2 +- .../Diagnostics/ReducedDiags/ReducedDiags.cpp | 4 +-- Source/Diagnostics/ReducedDiags/RhoMaximum.H | 2 +- .../Diagnostics/ReducedDiags/RhoMaximum.cpp | 2 +- Source/Diagnostics/SliceDiagnostic.H | 8 +++--- Source/Diagnostics/SliceDiagnostic.cpp | 2 +- Source/Diagnostics/WarpXOpenPMD.H | 18 ++++++------ Source/Diagnostics/WarpXOpenPMD.cpp | 28 +++++++++---------- Source/FieldSolver/ElectrostaticSolver.H | 2 +- Source/Initialization/WarpXInitData.cpp | 6 ++-- Source/Laser/LaserProfiles.H | 4 +-- .../LaserProfileFromFile.cpp | 4 +-- Source/Parallelization/GuardCellManager.H | 4 +-- Source/Parallelization/GuardCellManager.cpp | 4 +-- .../BackgroundMCC/BackgroundMCCCollision.H | 2 +- .../BackgroundMCC/BackgroundMCCCollision.cpp | 2 +- .../BackgroundStopping/BackgroundStopping.H | 2 +- .../BackgroundStopping/BackgroundStopping.cpp | 2 +- .../BinaryCollision/BinaryCollisionUtils.H | 4 +-- .../BinaryCollision/BinaryCollisionUtils.cpp | 4 +-- .../Coulomb/PairWiseCoulombCollisionFunc.H | 2 +- .../Collision/BinaryCollision/DSMC/DSMC.H | 2 +- .../Collision/BinaryCollision/DSMC/DSMC.cpp | 4 +-- .../NuclearFusion/NuclearFusionFunc.H | 5 ++-- .../BinaryCollision/ParticleCreationFunc.H | 4 +-- .../BinaryCollision/ParticleCreationFunc.cpp | 7 ++--- Source/Particles/Collision/CollisionBase.H | 2 +- Source/Particles/Collision/CollisionBase.cpp | 2 +- .../Particles/Collision/ScatteringProcess.H | 2 +- .../Particles/Collision/ScatteringProcess.cpp | 2 +- Source/Particles/MultiParticleContainer.H | 4 +-- Source/Particles/MultiParticleContainer.cpp | 4 +-- Source/Particles/ParticleBoundaryBuffer.H | 6 ++-- Source/Particles/ParticleBoundaryBuffer.cpp | 6 ++-- Source/Particles/PhysicalParticleContainer.H | 4 +-- .../Particles/PhysicalParticleContainer.cpp | 4 +-- .../Particles/Resampling/LevelingThinning.H | 2 +- .../Particles/Resampling/LevelingThinning.cpp | 2 +- Source/Particles/Resampling/Resampling.H | 2 +- Source/Particles/Resampling/Resampling.cpp | 2 +- .../Particles/Resampling/ResamplingTrigger.H | 2 +- .../Resampling/ResamplingTrigger.cpp | 2 +- Source/Particles/WarpXParticleContainer.H | 4 +-- Source/Python/callbacks.H | 8 +++--- Source/Python/callbacks.cpp | 10 +++---- Source/Utils/WarpXUtil.H | 4 +-- Source/Utils/WarpXUtil.cpp | 8 +++--- Source/WarpX.H | 4 +-- Source/ablastr/fields/PoissonSolver.H | 6 ++-- Source/ablastr/fields/VectorPoissonSolver.H | 6 ++-- Source/ablastr/utils/msg_logger/MsgLogger.H | 2 +- Source/ablastr/utils/msg_logger/MsgLogger.cpp | 2 +- Source/ablastr/warn_manager/WarnManager.H | 12 ++++---- Source/ablastr/warn_manager/WarnManager.cpp | 12 ++++---- 102 files changed, 209 insertions(+), 210 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 21f0343519c..5d2422af214 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -30,8 +30,6 @@ Checks: ' -modernize-use-trailing-return-type, mpi-*, performance-*, - -performance-unnecessary-copy-initialization, - -performance-unnecessary-value-param, portability-*, readability-*, -readability-convert-member-functions-to-static, diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H index 0a22774bb8d..9d3250fa52f 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H @@ -200,12 +200,12 @@ class BTDMultiFabHeaderImpl */ void SetBox (int ibox, amrex::Box ba_box) { m_ba.set(ibox, ba_box); } /** Set Fab name of the ith fab to be written in the multifab Header file.*/ - void SetFabName (int ifab, std::string fodPrefix, std::string FabName, + void SetFabName (int ifab, const std::string& fodPrefix, const std::string& FabName, int FabHead); /** Set minimum value of all the components for the ith fab. */ - void SetMinVal (int ifab, amrex::Vector minval); + void SetMinVal (int ifab, const amrex::Vector& minval); /** Set maximum value of all the components for the ith fab. */ - void SetMaxVal (int ifab, amrex::Vector maxval); + void SetMaxVal (int ifab, const amrex::Vector& maxval); private: /** Header file path */ std::string m_Header_path; diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp index cd3eb962405..4ec1c974f8e 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.cpp @@ -323,7 +323,7 @@ BTDMultiFabHeaderImpl::ResizeFabData () } void -BTDMultiFabHeaderImpl::SetFabName (int ifab, std::string fodPrefix, std::string FabName, +BTDMultiFabHeaderImpl::SetFabName (int ifab, const std::string& fodPrefix, const std::string& FabName, int FabHead) { m_FabOnDiskPrefix[ifab] = fodPrefix; @@ -333,13 +333,13 @@ BTDMultiFabHeaderImpl::SetFabName (int ifab, std::string fodPrefix, std::string } void -BTDMultiFabHeaderImpl::SetMinVal (int ifab, amrex::Vector minval) +BTDMultiFabHeaderImpl::SetMinVal (int ifab, const amrex::Vector& minval) { CopyVec(m_minval[ifab], minval); } void -BTDMultiFabHeaderImpl::SetMaxVal (int ifab, amrex::Vector maxval) +BTDMultiFabHeaderImpl::SetMaxVal (int ifab, const amrex::Vector& maxval) { CopyVec(m_maxval[ifab], maxval); } diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index f6c44c777ea..679dd6b8fa4 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -28,7 +28,7 @@ class BTDiagnostics final : public Diagnostics { public: - BTDiagnostics (int i, std::string name); + BTDiagnostics (int i, const std::string& name); private: /** Whether to plot raw (i.e., NOT cell-centered) fields */ @@ -375,26 +375,26 @@ private: /** Interleave lab-frame meta-data of the buffers to be consistent * with the merged plotfile lab-frame data. */ - void InterleaveBufferAndSnapshotHeader ( std::string buffer_Header, - std::string snapshot_Header); + void InterleaveBufferAndSnapshotHeader (const std::string& buffer_Header, + const std::string& snapshot_Header); /** Interleave meta-data of the buffer multifabs to be consistent * with the merged plotfile lab-frame data. */ - void InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, - std::string snapshot_FabHeader_path, - std::string newsnapshot_FabFilename); + void InterleaveFabArrayHeader (const std::string& Buffer_FabHeader_path, + const std::string& snapshot_FabHeader_path, + const std::string& newsnapshot_FabFilename); /** Interleave lab-frame metadata of the species header file in the buffers to * be consistent with the merged plotfile lab-frame data */ - void InterleaveSpeciesHeader(std::string buffer_species_Header_path, - std::string snapshot_species_Header_path, - std::string species_name, int new_data_index); + void InterleaveSpeciesHeader(const std::string& buffer_species_Header_path, + const std::string& snapshot_species_Header_path, + const std::string& species_name, int new_data_index); /** Interleave lab-frame metadata of the particle header file in the buffers to * be consistent with the merged plotfile lab-frame data */ - void InterleaveParticleDataHeader( std::string buffer_ParticleHdrFilename, - std::string snapshot_ParticleHdrFilename); + void InterleaveParticleDataHeader(const std::string& buffer_ParticleHdrFilename, + const std::string& snapshot_ParticleHdrFilename); /** Initialize particle functors for each species to compute the back-transformed lab-frame data. */ void InitializeParticleFunctors () override; diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index fd9c5c783c1..2422d1324af 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -52,8 +52,8 @@ namespace constexpr int permission_flag_rwxrxrx = 0755; } -BTDiagnostics::BTDiagnostics (int i, std::string name) - : Diagnostics(i, name) +BTDiagnostics::BTDiagnostics (int i, const std::string& name) + : Diagnostics{i, name} { ReadParameters(); } @@ -1290,8 +1290,8 @@ void BTDiagnostics::MergeBuffersForPlotfile (int i_snapshot) } void -BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_path, - std::string snapshot_Header_path) +BTDiagnostics::InterleaveBufferAndSnapshotHeader ( const std::string& buffer_Header_path, + const std::string& snapshot_Header_path) { BTDPlotfileHeaderImpl snapshot_HeaderImpl(snapshot_Header_path); snapshot_HeaderImpl.ReadHeaderData(); @@ -1335,9 +1335,9 @@ BTDiagnostics::InterleaveBufferAndSnapshotHeader ( std::string buffer_Header_pat void -BTDiagnostics::InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, - std::string snapshot_FabHeader_path, - std::string newsnapshot_FabFilename) +BTDiagnostics::InterleaveFabArrayHeader(const std::string& Buffer_FabHeader_path, + const std::string& snapshot_FabHeader_path, + const std::string& newsnapshot_FabFilename) { BTDMultiFabHeaderImpl snapshot_FabHeader(snapshot_FabHeader_path); snapshot_FabHeader.ReadMultiFabHeader(); @@ -1365,9 +1365,9 @@ BTDiagnostics::InterleaveFabArrayHeader (std::string Buffer_FabHeader_path, } void -BTDiagnostics::InterleaveSpeciesHeader(std::string buffer_species_Header_path, - std::string snapshot_species_Header_path, - std::string species_name, const int new_data_index) +BTDiagnostics::InterleaveSpeciesHeader(const std::string& buffer_species_Header_path, + const std::string& snapshot_species_Header_path, + const std::string& species_name, const int new_data_index) { BTDSpeciesHeaderImpl BufferSpeciesHeader(buffer_species_Header_path, species_name); @@ -1389,8 +1389,8 @@ BTDiagnostics::InterleaveSpeciesHeader(std::string buffer_species_Header_path, } void -BTDiagnostics::InterleaveParticleDataHeader(std::string buffer_ParticleHdrFilename, - std::string snapshot_ParticleHdrFilename) +BTDiagnostics::InterleaveParticleDataHeader(const std::string& buffer_ParticleHdrFilename, + const std::string& snapshot_ParticleHdrFilename) { BTDParticleDataHeaderImpl BufferParticleHeader(buffer_ParticleHdrFilename); BufferParticleHeader.ReadHeader(); diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.H b/Source/Diagnostics/BoundaryScrapingDiagnostics.H index 60d184c30e2..3e5fc1f19eb 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.H +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.H @@ -23,7 +23,7 @@ public: * @param i index of diagnostics in MultiDiagnostics::alldiags * @param name diagnostics name in the inputs file */ - BoundaryScrapingDiagnostics (int i, std::string name); + BoundaryScrapingDiagnostics (int i, const std::string& name); private: /** Read relevant parameters for BoundaryScraping */ diff --git a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp index 11ffce02f09..da1e5fdcc00 100644 --- a/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp +++ b/Source/Diagnostics/BoundaryScrapingDiagnostics.cpp @@ -21,8 +21,8 @@ using namespace amrex::literals; -BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, std::string name) - : Diagnostics(i, name) +BoundaryScrapingDiagnostics::BoundaryScrapingDiagnostics (int i, const std::string& name) + : Diagnostics{i, name} { ReadParameters(); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H index 56c4372fad8..7de9844a99e 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.H @@ -30,9 +30,9 @@ public: * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab. */ ParticleReductionFunctor(const amrex::MultiFab * mf_src, int lev, - amrex::IntVect crse_ratio, std::string fn_str, + amrex::IntVect crse_ratio, const std::string& fn_str, int ispec, bool do_average, - bool do_filter, std::string filter_str, + bool do_filter, const std::string& filter_str, int ncomp=1); /** \brief Compute the average of the function m_map_fn over each grid cell. diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index feb25c11183..15473890233 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -18,9 +18,9 @@ using namespace amrex::literals; ParticleReductionFunctor::ParticleReductionFunctor (const amrex::MultiFab* mf_src, const int lev, - const amrex::IntVect crse_ratio, const std::string fn_str, + const amrex::IntVect crse_ratio, const std::string& fn_str, const int ispec, const bool do_average, - const bool do_filter, const std::string filter_str, const int ncomp) + const bool do_filter, const std::string& filter_str, const int ncomp) : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev), m_ispec(ispec), m_do_average(do_average), m_do_filter(do_filter) { // mf_src will not be used, let's make sure it's null. diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 48361d90fc0..555ffb123e7 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -479,7 +479,7 @@ WriteZeroRawMF( const MultiFab& F, const DistributionMapping& dm, * coarse and fine patch to have the same shape. */ void -WriteCoarseVector( const std::string field_name, +WriteCoarseVector( const std::string& field_name, const MultiFab* Fx_cp, const MultiFab* Fy_cp, const MultiFab* Fz_cp, @@ -519,7 +519,7 @@ WriteCoarseVector( const std::string field_name, * coarse and fine patch to have the same shape. */ void -WriteCoarseScalar( const std::string field_name, +WriteCoarseScalar( const std::string& field_name, const MultiFab* F_cp, const MultiFab* F_fp, const DistributionMapping& dm, diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H index d2ec9a5a4e0..45ea40077e4 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.H +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.H @@ -45,7 +45,7 @@ public: * \param[in] amr_mesh an AmrMesh instance * \param[in] diag_name ParmParse scope string. */ - FlushFormatSensei (amrex::AmrMesh *amr_mesh, std::string diag_name); + FlushFormatSensei (amrex::AmrMesh *amr_mesh, const std::string& diag_name); /** Do in-situ visualization for field and particle data */ void WriteToFile ( diff --git a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp index 348e1da4a00..468ed81ce18 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatSensei.cpp @@ -8,7 +8,7 @@ #endif FlushFormatSensei::FlushFormatSensei (amrex::AmrMesh *amr_mesh, - std::string diag_name) : + const std::string& diag_name) : m_amr_mesh(amr_mesh) { #ifndef AMREX_USE_SENSEI_INSITU diff --git a/Source/Diagnostics/FullDiagnostics.H b/Source/Diagnostics/FullDiagnostics.H index 42b60c213f2..1b999a9b361 100644 --- a/Source/Diagnostics/FullDiagnostics.H +++ b/Source/Diagnostics/FullDiagnostics.H @@ -9,7 +9,7 @@ class FullDiagnostics final : public Diagnostics { public: - FullDiagnostics (int i, std::string name); + FullDiagnostics (int i, const std::string& name); private: /** Read user-requested parameters for full diagnostics */ void ReadParameters (); diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 27fe7e48c66..bf4cd3f95fc 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -41,7 +41,7 @@ using namespace amrex::literals; -FullDiagnostics::FullDiagnostics (int i, std::string name): +FullDiagnostics::FullDiagnostics (int i, const std::string& name): Diagnostics{i, name}, m_solver_deposits_current{ (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::None) || diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.H b/Source/Diagnostics/ParticleDiag/ParticleDiag.H index 7d8b5819f5f..e9f54d2e5e1 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.H +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.H @@ -17,7 +17,8 @@ class ParticleDiag { public: - ParticleDiag(std::string diag_name, std::string name, WarpXParticleContainer* pc, PinnedMemoryParticleContainer *pinned_pc = nullptr); + ParticleDiag(const std::string& diag_name, const std::string& name, + WarpXParticleContainer* pc, PinnedMemoryParticleContainer *pinned_pc = nullptr); [[nodiscard]] WarpXParticleContainer* getParticleContainer() const { return m_pc; } [[nodiscard]] PinnedMemoryParticleContainer* getPinnedParticleContainer() const { return m_pinned_pc; } [[nodiscard]] std::string getSpeciesName() const { return m_name; } diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp index 18fbb4f593b..9954fa24285 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp @@ -13,8 +13,10 @@ using namespace amrex; -ParticleDiag::ParticleDiag(std::string diag_name, std::string name, WarpXParticleContainer* pc, PinnedMemoryParticleContainer* pinned_pc) - : m_diag_name(diag_name), m_name(name), m_pc(pc), m_pinned_pc(pinned_pc) +ParticleDiag::ParticleDiag( + const std::string& diag_name, const std::string& name, + WarpXParticleContainer* pc, PinnedMemoryParticleContainer* pinned_pc): + m_diag_name(diag_name), m_name(name), m_pc(pc), m_pinned_pc(pinned_pc) { //variable to set m_plot_flags size const int plot_flag_size = pc->NumRealComps(); diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.H b/Source/Diagnostics/ReducedDiags/BeamRelevant.H index 018ddef5463..ed410c72fc9 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.H +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.H @@ -23,7 +23,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - BeamRelevant(std::string rd_name); + BeamRelevant(const std::string& rd_name); /// name of beam species std::string m_beam_name; diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index b78a7713abc..ae7d5230b4c 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -32,7 +32,7 @@ using namespace amrex; // constructor -BeamRelevant::BeamRelevant (std::string rd_name) +BeamRelevant::BeamRelevant (const std::string& rd_name) : ReducedDiags{rd_name} { // read beam name diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.H b/Source/Diagnostics/ReducedDiags/ChargeOnEB.H index ed69ce01767..3cea5b0b30e 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.H +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.H @@ -33,7 +33,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ChargeOnEB (std::string rd_name); + ChargeOnEB (const std::string& rd_name); /** * This function computes the charge at the surface of the EB: diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index c6f75e4a712..1398072a84d 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -28,7 +28,7 @@ using namespace amrex; // constructor -ChargeOnEB::ChargeOnEB (std::string rd_name) +ChargeOnEB::ChargeOnEB (const std::string& rd_name) : ReducedDiags{rd_name} { // Only 3D is working for now diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.H b/Source/Diagnostics/ReducedDiags/ColliderRelevant.H index fe00dd2e7e3..3b93eb1ed25 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.H +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.H @@ -26,7 +26,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ColliderRelevant(std::string rd_name); + ColliderRelevant(const std::string& rd_name); /// name of the two colliding species std::vector m_beam_name; diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 4cd1a66f3f2..3229f10c89e 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -59,8 +59,8 @@ using namespace amrex; -ColliderRelevant::ColliderRelevant (std::string rd_name) -: ReducedDiags{std::move(rd_name)} +ColliderRelevant::ColliderRelevant (const std::string& rd_name) +: ReducedDiags{rd_name} { // read colliding species names - must be 2 const amrex::ParmParse pp_rd_name(m_rd_name); diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.H b/Source/Diagnostics/ReducedDiags/FieldEnergy.H index 4f5d1656841..40de174526e 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.H +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.H @@ -26,7 +26,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldEnergy(std::string rd_name); + FieldEnergy(const std::string& rd_name); /** * This function computes the field energy (EF): diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index a88690f863f..1271e285c6c 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -30,7 +30,7 @@ using namespace amrex; // constructor -FieldEnergy::FieldEnergy (std::string rd_name) +FieldEnergy::FieldEnergy (const std::string& rd_name) : ReducedDiags{rd_name} { diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.H b/Source/Diagnostics/ReducedDiags/FieldMaximum.H index 95ae6ff16da..1d7cf4d236c 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.H +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.H @@ -24,7 +24,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldMaximum(std::string rd_name); + FieldMaximum(const std::string& rd_name); /** * This function computes the maximum value of Ex, Ey, Ez, |E|, Bx, By, Bz and |B| diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index 6465ee2be1b..8c66404056b 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -40,7 +40,7 @@ using namespace amrex; // constructor -FieldMaximum::FieldMaximum (std::string rd_name) +FieldMaximum::FieldMaximum (const std::string& rd_name) : ReducedDiags{rd_name} { // RZ coordinate is not working diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.H b/Source/Diagnostics/ReducedDiags/FieldMomentum.H index 0f04b6413d1..99eabf3772f 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.H +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.H @@ -24,7 +24,7 @@ public: * * \param[in] rd_name reduced diags names */ - FieldMomentum(std::string rd_name); + FieldMomentum(const std::string& rd_name); /** * \brief This function computes the electromagnetic momentum, diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index fe2040e50d2..3b3b7463593 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -38,7 +38,7 @@ using namespace amrex; -FieldMomentum::FieldMomentum (std::string rd_name) +FieldMomentum::FieldMomentum (const std::string& rd_name) : ReducedDiags{rd_name} { // RZ coordinate is not working diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.H b/Source/Diagnostics/ReducedDiags/FieldProbe.H index 9f318c4f8a0..a7c04649999 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.H +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.H @@ -41,7 +41,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldProbe (std::string rd_name); + FieldProbe (const std::string& rd_name); /** * This function assins test/data particles to constructed environemnt diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 24ad0e64ea8..f0780622d54 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -47,7 +47,7 @@ using namespace amrex; // constructor -FieldProbe::FieldProbe (std::string rd_name) +FieldProbe::FieldProbe (const std::string& rd_name) : ReducedDiags{rd_name}, m_probe(&WarpX::GetInstance()) { diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index e99c87ad14f..940222867b1 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -50,7 +50,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - FieldReduction(std::string rd_name); + FieldReduction(const std::string& rd_name); /** * This function is called at every time step, and if necessary calls the templated function diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.cpp b/Source/Diagnostics/ReducedDiags/FieldReduction.cpp index 85e4d4028e6..683ae1921d6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.cpp @@ -22,7 +22,7 @@ #include // constructor -FieldReduction::FieldReduction (std::string rd_name) +FieldReduction::FieldReduction (const std::string& rd_name) : ReducedDiags{rd_name} { using namespace amrex::literals; diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H index 0ecb74d4bf5..cfba804d0f0 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.H @@ -54,7 +54,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - LoadBalanceCosts(std::string rd_name); + LoadBalanceCosts(const std::string& rd_name); /** * This function updates the costs, given the current distribution mapping, diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index b4e07b51982..8af2c66711c 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -65,7 +65,7 @@ namespace } // constructor -LoadBalanceCosts::LoadBalanceCosts (std::string rd_name) +LoadBalanceCosts::LoadBalanceCosts (const std::string& rd_name) : ReducedDiags{rd_name} { } diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H index a43868ccc5f..c58f78deedb 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.H @@ -24,7 +24,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - LoadBalanceEfficiency(std::string rd_name); + LoadBalanceEfficiency(const std::string& rd_name); /** * This function gets the current load balance efficiency diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp index 6dd066d5324..b6f3a58e670 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceEfficiency.cpp @@ -20,7 +20,7 @@ using namespace amrex; // constructor -LoadBalanceEfficiency::LoadBalanceEfficiency (std::string rd_name) +LoadBalanceEfficiency::LoadBalanceEfficiency (const std::string& rd_name) : ReducedDiags{rd_name} { // read number of levels diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.H b/Source/Diagnostics/ReducedDiags/ParticleEnergy.H index b0df66709f3..cb3fb8c4f08 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.H +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.H @@ -25,7 +25,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleEnergy(std::string rd_name); + ParticleEnergy(const std::string& rd_name); /** * This function computes the particle relativistic kinetic energy (EP). diff --git a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp index 157b4bfb368..9446af5e20c 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleEnergy.cpp @@ -34,7 +34,7 @@ using namespace amrex; // constructor -ParticleEnergy::ParticleEnergy (std::string rd_name) +ParticleEnergy::ParticleEnergy (const std::string& rd_name) : ReducedDiags{rd_name} { // get a reference to WarpX instance diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H index ed5cce1d7e0..d3505b5ccf7 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.H +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.H @@ -25,7 +25,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleExtrema(std::string rd_name); + ParticleExtrema(const std::string& rd_name); /// name of species std::string m_species_name; diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index ba8e7e373de..0414d9ce869 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -53,7 +53,7 @@ using namespace amrex; // constructor -ParticleExtrema::ParticleExtrema (std::string rd_name) +ParticleExtrema::ParticleExtrema (const std::string& rd_name) : ReducedDiags{rd_name} { // read species name diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.H b/Source/Diagnostics/ReducedDiags/ParticleHistogram.H index 758f4a399a9..39ccd8f1ace 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.H +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.H @@ -28,7 +28,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleHistogram(std::string rd_name); + ParticleHistogram(const std::string& rd_name); /// normalization type int m_norm; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp index 512e6f1394a..4aeb2af526f 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram.cpp @@ -50,8 +50,8 @@ struct NormalizationType { }; // constructor -ParticleHistogram::ParticleHistogram (std::string rd_name): - ReducedDiags{rd_name} +ParticleHistogram::ParticleHistogram (const std::string& rd_name) +: ReducedDiags{rd_name} { const ParmParse pp_rd_name(rd_name); diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H index 6ba85502819..0c648528920 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.H @@ -30,7 +30,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleHistogram2D(std::string rd_name); + ParticleHistogram2D(const std::string& rd_name); /// File type std::string m_openpmd_backend {"default"}; diff --git a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp index 0d696f8b7bc..8dd19186b25 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleHistogram2D.cpp @@ -54,7 +54,7 @@ using namespace amrex; // constructor -ParticleHistogram2D::ParticleHistogram2D (std::string rd_name) +ParticleHistogram2D::ParticleHistogram2D (const std::string& rd_name) : ReducedDiags{rd_name} { ParmParse pp_rd_name(rd_name); diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.H b/Source/Diagnostics/ReducedDiags/ParticleMomentum.H index feb8957e516..ea079f92e62 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.H +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.H @@ -25,7 +25,7 @@ public: * * \param[in] rd_name reduced diags names */ - ParticleMomentum(std::string rd_name); + ParticleMomentum(const std::string& rd_name); /** * \brief This function computes the particle relativistic momentum, diff --git a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp index 5f278d48f14..47d1ec2cdd4 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleMomentum.cpp @@ -32,7 +32,7 @@ using namespace amrex; -ParticleMomentum::ParticleMomentum (std::string rd_name) +ParticleMomentum::ParticleMomentum (const std::string& rd_name) : ReducedDiags{rd_name} { // Get a reference to WarpX instance diff --git a/Source/Diagnostics/ReducedDiags/ParticleNumber.H b/Source/Diagnostics/ReducedDiags/ParticleNumber.H index 3143a387a74..894048806ca 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleNumber.H +++ b/Source/Diagnostics/ReducedDiags/ParticleNumber.H @@ -24,7 +24,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ParticleNumber(std::string rd_name); + ParticleNumber(const std::string& rd_name); /** * This function computes the total number of macroparticles and physical particles of each diff --git a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp index 7fe3f75d4ce..bfdf1daffe3 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleNumber.cpp @@ -27,7 +27,7 @@ using namespace amrex::literals; // constructor -ParticleNumber::ParticleNumber (std::string rd_name) +ParticleNumber::ParticleNumber (const std::string& rd_name) : ReducedDiags{rd_name} { // get a reference to WarpX instance diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.H b/Source/Diagnostics/ReducedDiags/ReducedDiags.H index 09aa85586be..2c942e1df6d 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.H +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.H @@ -52,7 +52,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - ReducedDiags (std::string rd_name); + ReducedDiags (const std::string& rd_name); /** * Virtual destructor for polymorphism diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp index 483f4c68ece..e3835dde416 100644 --- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp @@ -23,8 +23,8 @@ using namespace amrex; // constructor -ReducedDiags::ReducedDiags (std::string rd_name) - : m_rd_name(std::move(rd_name)) +ReducedDiags::ReducedDiags (const std::string& rd_name): +m_rd_name{rd_name} { BackwardCompatibility(); diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.H b/Source/Diagnostics/ReducedDiags/RhoMaximum.H index 6f306a5e7ad..ceff0f666e0 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.H +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.H @@ -28,7 +28,7 @@ public: * constructor * @param[in] rd_name reduced diags names */ - RhoMaximum(std::string rd_name); + RhoMaximum(const std::string& rd_name); /** * This function computes the maximum and minimum values of rho (summed over all species) and diff --git a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp index 4613a70b3b0..9902415d9a9 100644 --- a/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/RhoMaximum.cpp @@ -29,7 +29,7 @@ using namespace amrex::literals; // constructor -RhoMaximum::RhoMaximum (std::string rd_name) +RhoMaximum::RhoMaximum (const std::string& rd_name) : ReducedDiags{rd_name} { // RZ coordinate is not working diff --git a/Source/Diagnostics/SliceDiagnostic.H b/Source/Diagnostics/SliceDiagnostic.H index 000b01d470f..570f86d5384 100644 --- a/Source/Diagnostics/SliceDiagnostic.H +++ b/Source/Diagnostics/SliceDiagnostic.H @@ -25,10 +25,10 @@ void CheckSliceInput( amrex::RealBox real_box, amrex::IntVect &slice_hi, amrex::IntVect &interp_lo); void InterpolateSliceValues( amrex::MultiFab& smf, - amrex::IntVect interp_lo, amrex::RealBox slice_realbox, - amrex::Vector geom, int ncomp, int nghost, - amrex::IntVect slice_lo, amrex::IntVect slice_hi, - amrex::IntVect SliceType, amrex::RealBox real_box); + amrex::IntVect interp_lo, amrex::RealBox slice_realbox, + const amrex::Vector& geom, int ncomp, int nghost, + amrex::IntVect slice_lo, amrex::IntVect slice_hi, + amrex::IntVect SliceType, amrex::RealBox real_box); void InterpolateLo( const amrex::Box& bx, amrex::FArrayBox &fabox, amrex::IntVect slice_lo, amrex::Vector geom, diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 71a585aea35..f373e78c9b2 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -419,7 +419,7 @@ CheckSliceInput( const RealBox real_box, RealBox &slice_cc_nd_box, */ void InterpolateSliceValues(MultiFab& smf, IntVect interp_lo, RealBox slice_realbox, - Vector geom, int ncomp, int nghost, + const Vector& geom, int ncomp, int nghost, IntVect slice_lo, IntVect /*slice_hi*/, IntVect SliceType, const RealBox real_box) { diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 6c904790e15..12aa46de058 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -91,12 +91,12 @@ public: * @param authors a string specifying the authors of the simulation (can be empty) */ WarpXOpenPMDPlot (openPMD::IterationEncoding ie, - std::string filetype, - std::string operator_type, - std::map< std::string, std::string > operator_parameters, - std::string engine_type, - std::map< std::string, std::string > engine_parameters, - std::vector fieldPMLdirections, + const std::string& filetype, + const std::string& operator_type, + const std::map< std::string, std::string >& operator_parameters, + const std::string& engine_type, + const std::map< std::string, std::string >& engine_parameters, + const std::vector& fieldPMLdirections, const std::string& authors); ~WarpXOpenPMDPlot (); @@ -188,9 +188,9 @@ private: void SetupMeshComp ( openPMD::Mesh& mesh, - amrex::Geometry& full_geom, - std::string comp_name, - std::string field_name, + amrex::Geometry const& full_geom, + std::string const& comp_name, + std::string const& field_name, amrex::MultiFab const& mf, bool var_in_theta_mode ) const; diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index b55dbe79175..d90f3b45c96 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -343,7 +343,7 @@ namespace detail * set the metadata that indicates the physical unit. */ inline void - setOpenPMDUnit ( openPMD::Mesh mesh, const std::string field_name ) + setOpenPMDUnit ( openPMD::Mesh mesh, const std::string& field_name ) { if (field_name[0] == 'E'){ // Electric field mesh.setUnitDimension({ @@ -377,19 +377,19 @@ namespace detail #ifdef WARPX_USE_OPENPMD WarpXOpenPMDPlot::WarpXOpenPMDPlot ( openPMD::IterationEncoding ie, - std::string openPMDFileType, - std::string operator_type, - std::map< std::string, std::string > operator_parameters, - std::string engine_type, - std::map< std::string, std::string > engine_parameters, - std::vector fieldPMLdirections, + const std::string& openPMDFileType, + const std::string& operator_type, + const std::map< std::string, std::string >& operator_parameters, + const std::string& engine_type, + const std::map< std::string, std::string >& engine_parameters, + const std::vector& fieldPMLdirections, const std::string& authors) : m_Series(nullptr), m_MPIRank{amrex::ParallelDescriptor::MyProc()}, m_MPISize{amrex::ParallelDescriptor::NProcs()}, m_Encoding(ie), - m_OpenPMDFileType(std::move(openPMDFileType)), - m_fieldPMLdirections(std::move(fieldPMLdirections)), + m_OpenPMDFileType{openPMDFileType}, + m_fieldPMLdirections{fieldPMLdirections}, m_authors{authors} { m_OpenPMDoptions = detail::getSeriesOptions(operator_type, operator_parameters, @@ -816,7 +816,7 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc, // the beam/input3d showed write_real_comp.size() = 16 while only 10 real comp names // so using the min to be safe. // - auto const getComponentRecord = [&currSpecies](std::string const comp_name) { + auto const getComponentRecord = [&currSpecies](std::string const& comp_name) { // handle scalar and non-scalar records by name const auto [record_name, component_name] = detail::name2openPMD(comp_name); return currSpecies[record_name][component_name]; @@ -893,7 +893,7 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, auto const numParticleOnTile64 = static_cast(numParticleOnTile); auto const& soa = pti.GetStructOfArrays(); - auto const getComponentRecord = [&currSpecies](std::string const comp_name) { + auto const getComponentRecord = [&currSpecies](std::string const& comp_name) { // handle scalar and non-scalar records by name const auto [record_name, component_name] = detail::name2openPMD(comp_name); return currSpecies[record_name][component_name]; @@ -1194,9 +1194,9 @@ WarpXOpenPMDPlot::SetupFields ( openPMD::Container< openPMD::Mesh >& meshes, */ void WarpXOpenPMDPlot::SetupMeshComp (openPMD::Mesh& mesh, - amrex::Geometry& full_geom, - std::string comp_name, - std::string field_name, + amrex::Geometry const& full_geom, + std::string const& comp_name, + std::string const& field_name, amrex::MultiFab const& mf, bool var_in_theta_mode) const { diff --git a/Source/FieldSolver/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolver.H index 7fc6c9bf6da..0a2abe2d6b2 100644 --- a/Source/FieldSolver/ElectrostaticSolver.H +++ b/Source/FieldSolver/ElectrostaticSolver.H @@ -56,7 +56,7 @@ namespace ElectrostaticSolver { * * \param [in] potential The string value of the potential */ - void setPotentialEB (std::string potential) { + void setPotentialEB(const std::string& potential) { potential_eb_str = potential; buildParsersEB(); } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index e3ec624be8e..5309b878f08 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1337,8 +1337,8 @@ WarpX::LoadExternalFieldsFromFile (int const lev) #if defined(WARPX_USE_OPENPMD) && !defined(WARPX_DIM_1D_Z) && !defined(WARPX_DIM_XZ) void WarpX::ReadExternalFieldFromFile ( - std::string read_fields_from_path, amrex::MultiFab* mf, - std::string F_name, std::string F_component) + const std::string& read_fields_from_path, amrex::MultiFab* mf, + const std::string& F_name, const std::string& F_component) { // Get WarpX domain info auto& warpx = WarpX::GetInstance(); @@ -1514,7 +1514,7 @@ WarpX::ReadExternalFieldFromFile ( } // End function WarpX::ReadExternalFieldFromFile #else // WARPX_USE_OPENPMD && !WARPX_DIM_1D_Z && !defined(WARPX_DIM_XZ) void -WarpX::ReadExternalFieldFromFile (std::string , amrex::MultiFab* ,std::string, std::string) +WarpX::ReadExternalFieldFromFile (const std::string& , amrex::MultiFab* , const std::string& , const std::string& ) { #if defined(WARPX_DIM_1D_Z) WARPX_ABORT_WITH_MESSAGE("Reading fields from openPMD files is not supported in 1D"); diff --git a/Source/Laser/LaserProfiles.H b/Source/Laser/LaserProfiles.H index 3aea88ac956..c2a3f337fbf 100644 --- a/Source/Laser/LaserProfiles.H +++ b/Source/Laser/LaserProfiles.H @@ -286,7 +286,7 @@ private: /** \brief parse a field file in the HDF5 'lasy' format * \param lasy_file_name: name of the lasy file to parse */ - void parse_lasy_file(std::string lasy_file_name); + void parse_lasy_file(const std::string& lasy_file_name); /** \brief parse a field file in the binary 'binary' format (whose details are given below). * @@ -301,7 +301,7 @@ private: * The spatiotemporal grid must be rectangular and uniform. * \param binary_file_name: name of the binary file to parse */ - void parse_binary_file(std::string binary_file_name); + void parse_binary_file(const std::string& binary_file_name); /** \brief Finds left and right time indices corresponding to time t * diff --git a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp index 0fdb45c64f8..934a537be5a 100644 --- a/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp +++ b/Source/Laser/LaserProfilesImpl/LaserProfileFromFile.cpp @@ -162,7 +162,7 @@ WarpXLaserProfiles::FromFileLaserProfile::fill_amplitude ( } void -WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_name) +WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(const std::string& lasy_file_name) { #ifdef WARPX_USE_OPENPMD if(ParallelDescriptor::IOProcessor()){ @@ -237,7 +237,7 @@ WarpXLaserProfiles::FromFileLaserProfile::parse_lasy_file(std::string lasy_file_ } void -WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (std::string binary_file_name) +WarpXLaserProfiles::FromFileLaserProfile::parse_binary_file (const std::string& binary_file_name) { if(ParallelDescriptor::IOProcessor()){ std::ifstream inp(binary_file_name, std::ios::binary); diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 38cef54921b..5c530441fc9 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -63,8 +63,8 @@ public: int nci_corr_stencil, int electromagnetic_solver_id, int max_level, - amrex::Vector v_galilean, - amrex::Vector v_comoving, + const amrex::Vector& v_galilean, + const amrex::Vector& v_comoving, bool safe_guard_cells, int do_multi_J, bool fft_do_time_averaging, diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index ae5ab839d10..dc7800f490a 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -44,8 +44,8 @@ guardCellManager::Init ( const int nci_corr_stencil, const int electromagnetic_solver_id, const int max_level, - const amrex::Vector v_galilean, - const amrex::Vector v_comoving, + const amrex::Vector& v_galilean, + const amrex::Vector& v_comoving, const bool safe_guard_cells, const int do_multi_J, const bool fft_do_time_averaging, diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H index 1de6b999e0b..3fd8b4d6fc2 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H @@ -23,7 +23,7 @@ class BackgroundMCCCollision final : public CollisionBase { public: - BackgroundMCCCollision (std::string collision_name); + BackgroundMCCCollision (std::string const& collision_name); ~BackgroundMCCCollision () override = default; diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 4cb16f6fa50..1eb89ff4c0f 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -21,7 +21,7 @@ #include -BackgroundMCCCollision::BackgroundMCCCollision (std::string const collision_name) +BackgroundMCCCollision::BackgroundMCCCollision (std::string const& collision_name) : CollisionBase(collision_name) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_species_names.size() == 1, diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H index 55fa4b9e1e3..8ba9b6f1b31 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.H @@ -24,7 +24,7 @@ class BackgroundStopping final : public CollisionBase { public: - BackgroundStopping (std::string collision_name); + BackgroundStopping (const std::string& collision_name); ~BackgroundStopping () override = default; diff --git a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp index 8122d7225b4..517b1138295 100644 --- a/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp +++ b/Source/Particles/Collision/BackgroundStopping/BackgroundStopping.cpp @@ -16,7 +16,7 @@ #include -BackgroundStopping::BackgroundStopping (std::string const collision_name) +BackgroundStopping::BackgroundStopping (std::string const& collision_name) : CollisionBase(collision_name) { using namespace amrex::literals; diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index b8a390ddb93..f6aa8e6e303 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -31,10 +31,10 @@ enum struct NuclearFusionType { namespace BinaryCollisionUtils{ - NuclearFusionType get_nuclear_fusion_type (std::string collision_name, + NuclearFusionType get_nuclear_fusion_type (const std::string& collision_name, MultiParticleContainer const * mypc); - CollisionType get_collision_type (std::string collision_name, + CollisionType get_collision_type (const std::string& collision_name, MultiParticleContainer const * mypc); CollisionType nuclear_fusion_type_to_collision_type (NuclearFusionType fusion_type); diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp index 430bac5e548..812be089462 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp @@ -17,7 +17,7 @@ namespace BinaryCollisionUtils{ - NuclearFusionType get_nuclear_fusion_type (const std::string collision_name, + NuclearFusionType get_nuclear_fusion_type (const std::string& collision_name, MultiParticleContainer const * const mypc) { const amrex::ParmParse pp_collision_name(collision_name); @@ -102,7 +102,7 @@ namespace BinaryCollisionUtils{ return NuclearFusionType::Undefined; } - CollisionType get_collision_type (const std::string collision_name, + CollisionType get_collision_type (const std::string& collision_name, MultiParticleContainer const * const mypc) { const amrex::ParmParse pp_collision_name(collision_name); diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index cfdc36d3c50..d1a92ee9716 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -46,7 +46,7 @@ public: * @param[in] mypc the particle container (unused) * @param[in] isSameSpecies true if this is an intra-species colission */ - PairWiseCoulombCollisionFunc (const std::string collision_name, + PairWiseCoulombCollisionFunc (const std::string& collision_name, [[maybe_unused]] MultiParticleContainer const * const mypc, const bool isSameSpecies): m_isSameSpecies{isSameSpecies} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H index ab01eba2c81..aa656a3d92e 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H @@ -50,7 +50,7 @@ public: * * @param[in] collision_name the name of the collision */ - DSMC (std::string collision_name); + DSMC (const std::string& collision_name); /** Perform the collisions * diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp index 92ae3ddf257..8f309c3610a 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp @@ -11,8 +11,8 @@ #include "SplitAndScatterFunc.H" -DSMC::DSMC (const std::string collision_name) - : CollisionBase(collision_name) +DSMC::DSMC (const std::string& collision_name) + : CollisionBase{collision_name} { using namespace amrex::literals; const amrex::ParmParse pp_collision_name(collision_name); diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index b2a2112ca68..8f19364baec 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -56,11 +56,12 @@ public: * @param[in] mypc pointer to the MultiParticleContainer * @param[in] isSameSpecies whether the two colliding species are the same */ - NuclearFusionFunc (const std::string collision_name, MultiParticleContainer const * const mypc, + NuclearFusionFunc (const std::string& collision_name, MultiParticleContainer const * const mypc, const bool isSameSpecies): m_fusion_multiplier{amrex::ParticleReal{1.0}}, // default fusion multiplier m_probability_threshold{amrex::ParticleReal{0.02}}, // default fusion probability threshold m_probability_target_value{amrex::ParticleReal{0.002}}, // default fusion probability target_value + m_fusion_type{BinaryCollisionUtils::get_nuclear_fusion_type(collision_name, mypc)}, m_isSameSpecies{isSameSpecies} { @@ -68,8 +69,6 @@ public: WARPX_ABORT_WITH_MESSAGE("Nuclear fusion module does not currently work with single precision"); #endif - m_fusion_type = BinaryCollisionUtils::get_nuclear_fusion_type(collision_name, mypc); - const amrex::ParmParse pp_collision_name(collision_name); utils::parser::queryWithParser( pp_collision_name, "fusion_multiplier", m_fusion_multiplier); diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index d53ce4348fb..4a77006dd92 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -52,7 +52,7 @@ public: * @param[in] collision_name the name of the collision * @param[in] mypc pointer to the MultiParticleContainer */ - ParticleCreationFunc (std::string collision_name, MultiParticleContainer const * mypc); + ParticleCreationFunc (const std::string& collision_name, MultiParticleContainer const * mypc); /** * \brief operator() of the ParticleCreationFunc class. It creates new particles from binary @@ -318,7 +318,7 @@ class NoParticleCreationFunc public: NoParticleCreationFunc () = default; - NoParticleCreationFunc (const std::string /*collision_name*/, + NoParticleCreationFunc (const std::string& /*collision_name*/, MultiParticleContainer const * const /*mypc*/) {} AMREX_INLINE diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp index 33629cd530f..3690c5d7295 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.cpp @@ -17,13 +17,12 @@ #include -ParticleCreationFunc::ParticleCreationFunc (const std::string collision_name, - MultiParticleContainer const * const mypc) +ParticleCreationFunc::ParticleCreationFunc (const std::string& collision_name, + MultiParticleContainer const * const mypc): + m_collision_type{BinaryCollisionUtils::get_collision_type(collision_name, mypc)} { const amrex::ParmParse pp_collision_name(collision_name); - m_collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); - if (m_collision_type == CollisionType::ProtonBoronToAlphasFusion) { // Proton-Boron fusion only produces alpha particles diff --git a/Source/Particles/Collision/CollisionBase.H b/Source/Particles/Collision/CollisionBase.H index db79eaf1a01..21a5de6a38c 100644 --- a/Source/Particles/Collision/CollisionBase.H +++ b/Source/Particles/Collision/CollisionBase.H @@ -18,7 +18,7 @@ class CollisionBase { public: - CollisionBase (std::string collision_name); + CollisionBase (const std::string& collision_name); virtual void doCollisions (amrex::Real /*cur_time*/, amrex::Real /*dt*/, MultiParticleContainer* /*mypc*/ ){} diff --git a/Source/Particles/Collision/CollisionBase.cpp b/Source/Particles/Collision/CollisionBase.cpp index 8f799c83013..c8b4f76cbd2 100644 --- a/Source/Particles/Collision/CollisionBase.cpp +++ b/Source/Particles/Collision/CollisionBase.cpp @@ -10,7 +10,7 @@ #include -CollisionBase::CollisionBase (std::string collision_name) +CollisionBase::CollisionBase (const std::string& collision_name) { // read collision species diff --git a/Source/Particles/Collision/ScatteringProcess.H b/Source/Particles/Collision/ScatteringProcess.H index d05ea32e2fe..59ef7a02afb 100644 --- a/Source/Particles/Collision/ScatteringProcess.H +++ b/Source/Particles/Collision/ScatteringProcess.H @@ -57,7 +57,7 @@ public: */ static void readCrossSectionFile ( - std::string cross_section_file, + const std::string& cross_section_file, amrex::Vector& energies, amrex::Gpu::HostVector& sigmas ); diff --git a/Source/Particles/Collision/ScatteringProcess.cpp b/Source/Particles/Collision/ScatteringProcess.cpp index c15d11eea07..ea1b4b40f54 100644 --- a/Source/Particles/Collision/ScatteringProcess.cpp +++ b/Source/Particles/Collision/ScatteringProcess.cpp @@ -94,7 +94,7 @@ ScatteringProcess::parseProcessType(const std::string& scattering_process) void ScatteringProcess::readCrossSectionFile ( - const std::string cross_section_file, + const std::string& cross_section_file, amrex::Vector& energies, amrex::Gpu::HostVector& sigmas ) { diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 9946a680fcd..0421febc475 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -249,7 +249,7 @@ public: * \param[in] species_name The species for which back-transformed particles is set. * \param[in] do_back_transformed_particles The parameter to set if back-transformed particles are set to true/false */ - void SetDoBackTransformedParticles (std::string species_name, bool do_back_transformed_particles); + void SetDoBackTransformedParticles (const std::string& species_name, bool do_back_transformed_particles); [[nodiscard]] int nSpeciesDepositOnMainGrid () const { @@ -330,7 +330,7 @@ public: const amrex::MultiFab& Bz); #endif - [[nodiscard]] int getSpeciesID (std::string product_str) const; + [[nodiscard]] int getSpeciesID (const std::string& product_str) const; amrex::Vector>::iterator begin() {return allcontainers.begin();} amrex::Vector>::iterator end() {return allcontainers.end();} diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index a31f426a0e4..b70844a8d15 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -803,7 +803,7 @@ MultiParticleContainer::mapSpeciesProduct () /* \brief Given a species name, return its ID. */ int -MultiParticleContainer::getSpeciesID (std::string product_str) const +MultiParticleContainer::getSpeciesID (const std::string& product_str) const { auto species_and_lasers_names = GetSpeciesAndLasersNames(); int i_product = 0; @@ -832,7 +832,7 @@ MultiParticleContainer::SetDoBackTransformedParticles (const bool do_back_transf } void -MultiParticleContainer::SetDoBackTransformedParticles (std::string species_name, const bool do_back_transformed_particles) { +MultiParticleContainer::SetDoBackTransformedParticles (const std::string& species_name, const bool do_back_transformed_particles) { auto species_names_list = GetSpeciesNames(); bool found = false; // Loop over species diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index 1e9748b2ff5..20bc63f7a19 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -48,11 +48,11 @@ public: void printNumParticles () const; - int getNumParticlesInContainer(std::string species_name, int boundary, bool local); + int getNumParticlesInContainer(const std::string& species_name, int boundary, bool local); - PinnedMemoryParticleContainer& getParticleBuffer(std::string species_name, int boundary); + PinnedMemoryParticleContainer& getParticleBuffer(const std::string& species_name, int boundary); - PinnedMemoryParticleContainer* getParticleBufferPointer(std::string species_name, int boundary); + PinnedMemoryParticleContainer* getParticleBufferPointer(const std::string& species_name, int boundary); static constexpr int numBoundaries () { return AMREX_SPACEDIM*2 diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index d1f9f814bc5..25b77ddc496 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -531,7 +531,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, } int ParticleBoundaryBuffer::getNumParticlesInContainer( - const std::string species_name, int boundary, bool local) { + const std::string& species_name, int boundary, bool local) { auto& buffer = m_particle_containers[boundary]; auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); @@ -545,7 +545,7 @@ int ParticleBoundaryBuffer::getNumParticlesInContainer( } PinnedMemoryParticleContainer & -ParticleBoundaryBuffer::getParticleBuffer(const std::string species_name, int boundary) { +ParticleBoundaryBuffer::getParticleBuffer(const std::string& species_name, int boundary) { auto& buffer = m_particle_containers[boundary]; auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); @@ -560,7 +560,7 @@ ParticleBoundaryBuffer::getParticleBuffer(const std::string species_name, int bo } PinnedMemoryParticleContainer * -ParticleBoundaryBuffer::getParticleBufferPointer(const std::string species_name, int boundary) { +ParticleBoundaryBuffer::getParticleBufferPointer(const std::string& species_name, int boundary) { auto& buffer = m_particle_containers[boundary]; auto index = WarpX::GetInstance().GetPartContainer().getSpeciesID(species_name); diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H index 286d0675da6..5d9b41b8b75 100644 --- a/Source/Particles/PhysicalParticleContainer.H +++ b/Source/Particles/PhysicalParticleContainer.H @@ -352,14 +352,14 @@ public: * @param[in] ptr the pointer */ void set_breit_wheeler_engine_ptr - (std::shared_ptr ptr) override; + (const std::shared_ptr& ptr) override; /** * Acquires a shared smart pointer to a QuantumSynchrotronEngine * @param[in] ptr the pointer */ void set_quantum_sync_engine_ptr - (std::shared_ptr ptr) override; + (const std::shared_ptr& ptr) override; //__________ BreitWheelerEngine* get_breit_wheeler_engine_ptr () const override { diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 4adb5386a2a..9923d93a907 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -3392,14 +3392,14 @@ bool PhysicalParticleContainer::has_breit_wheeler () const void PhysicalParticleContainer:: -set_breit_wheeler_engine_ptr (std::shared_ptr ptr) +set_breit_wheeler_engine_ptr (const std::shared_ptr& ptr) { m_shr_p_bw_engine = ptr; } void PhysicalParticleContainer:: -set_quantum_sync_engine_ptr (std::shared_ptr ptr) +set_quantum_sync_engine_ptr (const std::shared_ptr& ptr) { m_shr_p_qs_engine = ptr; } diff --git a/Source/Particles/Resampling/LevelingThinning.H b/Source/Particles/Resampling/LevelingThinning.H index fa05525e270..9e3e63fc8be 100644 --- a/Source/Particles/Resampling/LevelingThinning.H +++ b/Source/Particles/Resampling/LevelingThinning.H @@ -36,7 +36,7 @@ public: * * @param[in] species_name the name of the resampled species */ - LevelingThinning (std::string species_name); + LevelingThinning (const std::string& species_name); /** * \brief A method that performs leveling thinning for the considered species. diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index 5dc6a458f97..71ebe735c78 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -29,7 +29,7 @@ #include -LevelingThinning::LevelingThinning (const std::string species_name) +LevelingThinning::LevelingThinning (const std::string& species_name) { using namespace amrex::literals; diff --git a/Source/Particles/Resampling/Resampling.H b/Source/Particles/Resampling/Resampling.H index 35439e905df..7c5c60d858d 100644 --- a/Source/Particles/Resampling/Resampling.H +++ b/Source/Particles/Resampling/Resampling.H @@ -62,7 +62,7 @@ public: * * @param[in] species_name the name of the resampled species */ - Resampling (std::string species_name); + Resampling (const std::string& species_name); /** * \brief A method that returns true if resampling should be done for the considered species diff --git a/Source/Particles/Resampling/Resampling.cpp b/Source/Particles/Resampling/Resampling.cpp index 63ac448880b..11f8706dc33 100644 --- a/Source/Particles/Resampling/Resampling.cpp +++ b/Source/Particles/Resampling/Resampling.cpp @@ -12,7 +12,7 @@ #include #include -Resampling::Resampling (const std::string species_name) +Resampling::Resampling (const std::string& species_name) { const amrex::ParmParse pp_species_name(species_name); std::string resampling_algorithm_string = "leveling_thinning"; // default resampling algorithm diff --git a/Source/Particles/Resampling/ResamplingTrigger.H b/Source/Particles/Resampling/ResamplingTrigger.H index 42c3902c247..538680475a0 100644 --- a/Source/Particles/Resampling/ResamplingTrigger.H +++ b/Source/Particles/Resampling/ResamplingTrigger.H @@ -33,7 +33,7 @@ public: * \brief Constructor of the ResamplingTrigger class. Reads the resampling trigger parameters * from the input file. */ - ResamplingTrigger (std::string species_name); + ResamplingTrigger (const std::string& species_name); /** * \brief A method that returns true if resampling should be done for the considered species diff --git a/Source/Particles/Resampling/ResamplingTrigger.cpp b/Source/Particles/Resampling/ResamplingTrigger.cpp index 16a32dd7d15..5efb473fe68 100644 --- a/Source/Particles/Resampling/ResamplingTrigger.cpp +++ b/Source/Particles/Resampling/ResamplingTrigger.cpp @@ -14,7 +14,7 @@ #include -ResamplingTrigger::ResamplingTrigger (const std::string species_name) +ResamplingTrigger::ResamplingTrigger (const std::string& species_name) { const amrex::ParmParse pp_species_name(species_name); diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index c100cc0bff4..4b01f09e27e 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -467,9 +467,9 @@ protected: //Species can receive a shared pointer to a QED engine (species for //which this is relevant should override these functions) virtual void - set_breit_wheeler_engine_ptr(std::shared_ptr){} + set_breit_wheeler_engine_ptr(const std::shared_ptr&){} virtual void - set_quantum_sync_engine_ptr(std::shared_ptr){} + set_quantum_sync_engine_ptr(const std::shared_ptr&){} int m_qed_breit_wheeler_ele_product; std::string m_qed_breit_wheeler_ele_product_name; diff --git a/Source/Python/callbacks.H b/Source/Python/callbacks.H index f0c98caf636..7ec78782bed 100644 --- a/Source/Python/callbacks.H +++ b/Source/Python/callbacks.H @@ -29,21 +29,21 @@ extern WARPX_EXPORT std::map< std::string, std::function > warpx_callbac /** * \brief Function to install the given name and function in warpx_callback_py_map */ -void InstallPythonCallback ( std::string name, std::function callback ); +void InstallPythonCallback ( const std::string& name, std::function callback ); /** * \brief Function to check if the given name is a key in warpx_callback_py_map */ -bool IsPythonCallbackInstalled ( std::string name ); +bool IsPythonCallbackInstalled ( const std::string& name ); /** * \brief Function to look for and execute Python callbacks */ -void ExecutePythonCallback ( std::string name ); +void ExecutePythonCallback ( const std::string& name ); /** * \brief Function to clear the given callback name from warpx_callback_py_map */ -void ClearPythonCallback ( std::string name ); +void ClearPythonCallback ( const std::string& name ); #endif // WARPX_PY_CALLBACKS_H_ diff --git a/Source/Python/callbacks.cpp b/Source/Python/callbacks.cpp index 930c88e65d1..79f15c62835 100644 --- a/Source/Python/callbacks.cpp +++ b/Source/Python/callbacks.cpp @@ -15,18 +15,18 @@ std::map< std::string, std::function > warpx_callback_py_map; -void InstallPythonCallback ( std::string name, std::function callback ) +void InstallPythonCallback ( const std::string& name, std::function callback ) { - warpx_callback_py_map[name] = callback; + warpx_callback_py_map[name] = std::move(callback); } -bool IsPythonCallbackInstalled ( std::string name ) +bool IsPythonCallbackInstalled ( const std::string& name ) { return (warpx_callback_py_map.count(name) == 1u); } // Execute Python callbacks of the type given by the input string -void ExecutePythonCallback ( std::string name ) +void ExecutePythonCallback ( const std::string& name ) { if ( IsPythonCallbackInstalled(name) ) { WARPX_PROFILE("warpx_py_" + name); @@ -47,7 +47,7 @@ void ExecutePythonCallback ( std::string name ) } } -void ClearPythonCallback ( std::string name ) +void ClearPythonCallback ( const std::string& name ) { warpx_callback_py_map.erase(name); } diff --git a/Source/Utils/WarpXUtil.H b/Source/Utils/WarpXUtil.H index ad3194edfc3..1de03eb61f0 100644 --- a/Source/Utils/WarpXUtil.H +++ b/Source/Utils/WarpXUtil.H @@ -64,7 +64,7 @@ namespace WarpXUtilIO{ * @param[in] data Vector containing binary data to write on disk * return true if it succeeds, false otherwise */ -bool WriteBinaryDataOnFile(std::string filename, const amrex::Vector& data); +bool WriteBinaryDataOnFile(const std::string& filename, const amrex::Vector& data); } @@ -116,7 +116,7 @@ namespace WarpXUtilLoadBalance * @param[in] dm the dmap to check * @return consistent whether the grids are consistent or not. */ - bool doCosts (const amrex::LayoutData* cost, amrex::BoxArray ba, + bool doCosts (const amrex::LayoutData* cost, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); } diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index 5b917a81fdf..e08a6304f09 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -266,7 +266,7 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) } namespace WarpXUtilIO{ - bool WriteBinaryDataOnFile(std::string filename, const amrex::Vector& data) + bool WriteBinaryDataOnFile(const std::string& filename, const amrex::Vector& data) { std::ofstream of{filename, std::ios::binary}; of.write(data.data(), data.size()); @@ -481,11 +481,11 @@ void ReadBCParams () namespace WarpXUtilLoadBalance { - bool doCosts (const amrex::LayoutData* costs, const amrex::BoxArray ba, + bool doCosts (const amrex::LayoutData* cost, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm) { - const bool consistent = costs && (dm == costs->DistributionMap()) && - (ba.CellEqual(costs->boxArray())) && + const bool consistent = cost && (dm == cost->DistributionMap()) && + (ba.CellEqual(cost->boxArray())) && (WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers); return consistent; } diff --git a/Source/WarpX.H b/Source/WarpX.H index 1a56445f8dd..0c86a1c9afe 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1050,8 +1050,8 @@ public: * for a specific field (specified by `F_name`) */ void ReadExternalFieldFromFile ( - std::string read_fields_from_path, amrex::MultiFab* mf, - std::string F_name, std::string F_component); + const std::string& read_fields_from_path, amrex::MultiFab* mf, + const std::string& F_name, const std::string& F_component); /** * \brief diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 9c6053bed07..77c06f11cf2 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -96,9 +96,9 @@ computePhi (amrex::Vector const & rho, amrex::Real absolute_tolerance, int const max_iters, int const verbosity, - amrex::Vector const geom, - amrex::Vector const dmap, - amrex::Vector const grids, + amrex::Vector const& geom, + amrex::Vector const& dmap, + amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index df2287df7f4..d49335723d8 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -89,9 +89,9 @@ computeVectorPotential ( amrex::Vector > co amrex::Real absolute_tolerance, int const max_iters, int const verbosity, - amrex::Vector const geom, - amrex::Vector const dmap, - amrex::Vector const grids, + amrex::Vector const& geom, + amrex::Vector const& dmap, + amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.H b/Source/ablastr/utils/msg_logger/MsgLogger.H index 01a61be5c80..401432f5dda 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.H +++ b/Source/ablastr/utils/msg_logger/MsgLogger.H @@ -196,7 +196,7 @@ namespace ablastr::utils::msg_logger * * @param[in] msg a Msg struct */ - void record_msg(Msg msg); + void record_msg(const Msg& msg); /** * \brief This function returns a vector containing the recorded messages diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.cpp b/Source/ablastr/utils/msg_logger/MsgLogger.cpp index dde041652da..6537a8f61e5 100644 --- a/Source/ablastr/utils/msg_logger/MsgLogger.cpp +++ b/Source/ablastr/utils/msg_logger/MsgLogger.cpp @@ -216,7 +216,7 @@ Logger::Logger() : m_io_rank{amrex::ParallelDescriptor::IOProcessorNumber()} {} -void Logger::record_msg(Msg msg) +void Logger::record_msg(const Msg& msg) { m_messages[msg]++; } diff --git a/Source/ablastr/warn_manager/WarnManager.H b/Source/ablastr/warn_manager/WarnManager.H index 737fb8fba73..56ff1bf1a1a 100644 --- a/Source/ablastr/warn_manager/WarnManager.H +++ b/Source/ablastr/warn_manager/WarnManager.H @@ -84,9 +84,9 @@ namespace ablastr::warn_manager * @param[in] priority priority of the warning message ("medium" by default) */ void RecordWarning( - std::string topic, - std::string text, - WarnPriority priority = WarnPriority::medium); + const std::string& topic, + const std::string& text, + const WarnPriority& priority = WarnPriority::medium); /** * \brief This function prints all the warning messages collected on the present MPI rank @@ -225,9 +225,9 @@ namespace ablastr::warn_manager * @param[in] priority priority of the warning message ("medium" by default) */ void WMRecordWarning( - std::string topic, - std::string text, - WarnPriority priority = WarnPriority::medium); + const std::string& topic, + const std::string& text, + const WarnPriority& priority = WarnPriority::medium); } #endif //ABLASTR_WARN_MANAGER_H_ diff --git a/Source/ablastr/warn_manager/WarnManager.cpp b/Source/ablastr/warn_manager/WarnManager.cpp index ee052ded299..b1a194b2633 100644 --- a/Source/ablastr/warn_manager/WarnManager.cpp +++ b/Source/ablastr/warn_manager/WarnManager.cpp @@ -55,9 +55,9 @@ WarnManager::WarnManager(): {} void WarnManager::RecordWarning( - std::string topic, - std::string text, - WarnPriority priority) + const std::string& topic, + const std::string& text, + const WarnPriority& priority) { auto msg_priority = abl_msg_logger::Priority::high; if(priority == WarnPriority::low) { @@ -316,9 +316,9 @@ WarnManager& ablastr::warn_manager::GetWMInstance() } void ablastr::warn_manager::WMRecordWarning( - std::string topic, - std::string text, - WarnPriority priority) + const std::string& topic, + const std::string& text, + const WarnPriority& priority) { WarnManager::GetInstance().RecordWarning( topic, text, priority); From e30472d531188c16b99987bce24f9e14dc6f92bd Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 1 Apr 2024 13:26:30 -0700 Subject: [PATCH 020/190] AMReX/pyAMReX/PICSAR: Weekly Update (#4814) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 739684f6c74..259c0d8d351 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 9e8f8d60eca337e3c14396f06f0a77500289d45c && cd - + cd ../amrex && git checkout --detach 47347f785f5c26f1f9e65bef8e10f2d383406ef7 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 8f7da493f24..f0ce221df0c 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 9e8f8d60eca337e3c14396f06f0a77500289d45c +branch = 47347f785f5c26f1f9e65bef8e10f2d383406ef7 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 42274fa30ae..897fcde3cf9 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 9e8f8d60eca337e3c14396f06f0a77500289d45c +branch = 47347f785f5c26f1f9e65bef8e10f2d383406ef7 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index fdc628f3f06..db230573853 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "9e8f8d60eca337e3c14396f06f0a77500289d45c" +set(WarpX_amrex_branch "47347f785f5c26f1f9e65bef8e10f2d383406ef7" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index e2b83469010..8ea26792217 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "bde190184effaf095f14e173df8f61f31ed67308" +set(WarpX_pyamrex_branch "b41b83d1c0f37c66ad0543afa7edaca9e0c82565" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 593d7bf07ca..f74eb50af8c 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 9e8f8d60eca337e3c14396f06f0a77500289d45c && cd - +cd amrex && git checkout --detach 47347f785f5c26f1f9e65bef8e10f2d383406ef7 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 6a4ebd9d2c58d11386cdf544ca9ea16c77d05a10 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 1 Apr 2024 22:40:46 +0200 Subject: [PATCH 021/190] Add clang-tidy workflow (#4806) * added instructions and script to run clang-tidy locally * make script executable * remove installation of blaspp and lapackpp from runClangTidy.sh script * improved documentation --- .../developers/run_clang_tidy_locally.rst | 55 ++++++++ Docs/source/developers/workflows.rst | 1 + Tools/Linter/runClangTidy.sh | 117 ++++++++++++++++++ 3 files changed, 173 insertions(+) create mode 100644 Docs/source/developers/run_clang_tidy_locally.rst create mode 100755 Tools/Linter/runClangTidy.sh diff --git a/Docs/source/developers/run_clang_tidy_locally.rst b/Docs/source/developers/run_clang_tidy_locally.rst new file mode 100644 index 00000000000..3f600019fe7 --- /dev/null +++ b/Docs/source/developers/run_clang_tidy_locally.rst @@ -0,0 +1,55 @@ +.. _developers-run_clang_tidy_locally: + +The clang-tidy linter +===================== + +Clang-tidy CI test +------------------ + +WarpX's CI tests include several checks performed with the +`clang-tidy `__ linter +(currently the version 15 of this tool). The complete list of checks +enforced in CI tests can be found in the ``.clang-tidy`` configuration file. + +.. dropdown:: clang-tidy configuration file + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../.clang-tidy + :language: yaml + +Run clang-tidy linter locally +----------------------------- + +We provide a script to run clang-tidy locally. The script can be run as follows, +provided that all the requirements to compile WarpX are met (see `building from source `). +The script generates a simple wrapper to ensure that `clang-tidy` is only applied to WarpX source files +and compiles WarpX in 1D,2D,3D, and RZ using such wrapper. By default WarpX is compiled in single precision +with PSATD solver, QED module, QED table generator and Embedded boundary in order to find more +potential issues with the `clang-tidy` tool. + +Few optional environment variables can be set to tune the behavior of the script: + +* ``WARPX_TOOLS_LINTER_PARALLEL``: sets the number of cores to be used for the compilation +* ``CLANG``, ``CLANGXX``, and ``CLANGTIDY`` : set the version of the compiler and of the linter + +Note: clang v15 is currently used in CI tests. It is therefore recommended to use this version. +Otherwise, a newer version may find issues not currently covered by CI tests (checks are opt-in) +while older versions may not find all the issues. + +.. code-block:: bash + + export WARPX_TOOLS_LINTER_PARALLEL=12 + export CLANG=clang-15 + export CLANGXX=clang++-15 + export CLANGTIDY=clang-tidy-15 + ./Tools/Linter/runClangTidy.sh + +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../Tools/Linter/runClangTidy.sh + :language: bash diff --git a/Docs/source/developers/workflows.rst b/Docs/source/developers/workflows.rst index 8b357f5d4da..00279018e9d 100644 --- a/Docs/source/developers/workflows.rst +++ b/Docs/source/developers/workflows.rst @@ -11,3 +11,4 @@ Workflows documentation checksum local_compile + run_clang_tidy_locally diff --git a/Tools/Linter/runClangTidy.sh b/Tools/Linter/runClangTidy.sh new file mode 100755 index 00000000000..39a96cc97d4 --- /dev/null +++ b/Tools/Linter/runClangTidy.sh @@ -0,0 +1,117 @@ +#!/usr/bin/env bash +# +# Copyright 2024 Luca Fedeli +# +# This file is part of WarpX. +# + +# This script is a developer's tool to perform the +# checks done by the clang-tidy CI test locally. +# +# Note: this script is only tested on Linux + +echo "=============================================" +echo +echo "This script is a developer's tool to perform the" +echo "checks done by the clang-tidy CI test locally" +echo "_____________________________________________" + +# Check source dir +REPO_DIR=$(cd $(dirname ${BASH_SOURCE})/../../ && pwd) +echo +echo "Your current source directory is: ${REPO_DIR}" +echo "_____________________________________________" + +# Set number of jobs to use for compilation +PARALLEL="${WARPX_TOOLS_LINTER_PARALLEL:-4}" +echo +echo "${PARALLEL} jobs will be used for compilation." +echo "This can be overridden by setting the environment" +echo "variable WARPX_TOOLS_LINTER_PARALLEL, e.g.: " +echo +echo "$ export WARPX_TOOLS_LINTER_PARALLEL=8" +echo "$ ./Tools/Linter/runClangTidy.sh" +echo "_____________________________________________" + +# Check clang version +export CC="${CLANG:-"clang"}" +export CXX="${CLANGXX:-"clang++"}" +export CTIDY="${CLANGTIDY:-"clang-tidy"}" +echo +echo "The following versions of the clang compiler and" +echo "of the clang-tidy linter will be used:" +echo +echo "clang version:" +which ${CC} +${CC} --version +echo +echo "clang++ version:" +which ${CXX} +${CXX} --version +echo +echo "clang-tidy version:" +which ${CTIDY} +${CTIDY} --version +echo +echo "This can be overridden by setting the environment" +echo "variables CLANG, CLANGXX, and CLANGTIDY e.g.: " +echo "$ export CLANG=clang-15" +echo "$ export CLANGXX=clang++-15" +echo "$ export CTIDCLANGTIDYY=clang-tidy-15" +echo "$ ./Tools/Linter/runClangTidy.sh" +echo +echo "******************************************************" +echo "* Warning: clang v15 is currently used in CI tests. *" +echo "* It is therefore recommended to use this version. *" +echo "* Otherwise, a newer version may find issues not *" +echo "* currently covered by CI tests while older versions *" +echo "* may not find all the issues. *" +echo "******************************************************" +echo "_____________________________________________" + +# Prepare clang-tidy wrapper +echo +echo "Prepare clang-tidy wrapper" +echo "The following wrapper ensures that only source files" +echo "in WarpX/Source/* are actually processed by clang-tidy" +echo +cat > ${REPO_DIR}/clang_tidy_wrapper << EOF +#!/bin/bash +REGEX="[a-z_A-Z0-9\/]*WarpX\/Source[a-z_A-Z0-9\/]+.cpp" +if [[ \$4 =~ \$REGEX ]];then + ${CTIDY} \$@ +fi +EOF +chmod +x ${REPO_DIR}/clang_tidy_wrapper +echo "clang_tidy_wrapper: " +cat ${REPO_DIR}/clang_tidy_wrapper +echo "_____________________________________________" + +# Compile Warpx using clang-tidy +echo +echo "*******************************************" +echo "* Compile Warpx using clang-tidy *" +echo "* Please ensure that all the dependencies *" +echo "* required to compile WarpX are met *" +echo "*******************************************" +echo + +rm -rf ${REPO_DIR}/build_clang_tidy + +cmake -S ${REPO_DIR} -B ${REPO_DIR}/build_clang_tidy \ + -DCMAKE_CXX_CLANG_TIDY="${REPO_DIR}/clang_tidy_wrapper;--system-headers=0;--config-file=${REPO_DIR}/.clang-tidy" \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DWarpX_DIMS="1;2;3;RZ" \ + -DWarpX_MPI=ON \ + -DWarpX_COMPUTE=OMP \ + -DWarpX_PSATD=ON \ + -DWarpX_QED=ON \ + -DWarpX_QED_TABLE_GEN=ON \ + -DWarpX_OPENPMD=ON \ + -DWarpX_PRECISION=SINGLE + +cmake --build ${REPO_DIR}/build_clang_tidy -j ${PARALLEL} 2> ${REPO_DIR}/build_clang_tidy/clang-tidy.log + +cat ${REPO_DIR}/build_clang_tidy/clang-tidy.log +echo +echo "=============================================" From 0466145fc0edf14d23ec5acd3845e52f8c30127e Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 1 Apr 2024 23:07:31 +0200 Subject: [PATCH 022/190] Clang tidy CI test: add several clang-analyzer-* checks to clang tidy CI test (#4684) * Clang tidy CI test: add several clang-analyzer-* checks to clang tidy CI test * address issues found with clang-tidy * fix bug * fix spaces * simplify * avoid using potentially confusing v[counter++] syntax * disable clang-analyzer-optin.mpi.MPI-Checker * make thread_required constexpr in mpi_init * make thread_required constexpr in mpi_init * disable clang-analyzer-optin.performance.Padding * start using get_field_pointer * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * introduce AMREX_ASSUME * update amrex * using get_field_pointer more * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * use new function get_field_pointer * use new functions to get fields and field pointers everywhere * fix bug * bugfixing * add missing [[nodiscard]] and remove redundant [[nodiscard]] * address issue related to auto* const found with clang-tidy * fix bug * fix issue found with clang tidy * fixed bug * fix bug --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .clang-tidy | 5 + Source/BoundaryConditions/PML.cpp | 37 +++-- Source/BoundaryConditions/PML_RZ.cpp | 8 +- .../WarpXFieldBoundaries.cpp | 48 +++--- Source/Diagnostics/BTDiagnostics.cpp | 40 ++--- .../BackTransformFunctor.cpp | 22 ++- .../ComputeDiagFunctors/CellCenterFunctor.cpp | 10 +- .../ComputeDiagFunctors/JFunctor.cpp | 17 +- .../ComputeDiagFunctors/JdispFunctor.cpp | 23 +-- .../FlushFormats/FlushFormatCheckpoint.cpp | 60 +++---- .../FlushFormats/FlushFormatPlotfile.cpp | 76 ++++----- Source/Diagnostics/FullDiagnostics.cpp | 127 +++++++-------- .../Diagnostics/ReducedDiags/ChargeOnEB.cpp | 6 +- .../ReducedDiags/ColliderRelevant.cpp | 12 +- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 12 +- .../Diagnostics/ReducedDiags/FieldMaximum.cpp | 12 +- .../ReducedDiags/FieldMomentum.cpp | 12 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 12 +- .../Diagnostics/ReducedDiags/FieldReduction.H | 18 +- .../ReducedDiags/LoadBalanceCosts.cpp | 2 +- .../ReducedDiags/ParticleExtrema.cpp | 12 +- Source/Diagnostics/SliceDiagnostic.cpp | 12 +- Source/FieldSolver/ElectrostaticSolver.cpp | 12 +- .../HybridPICModel/HybridPICModel.cpp | 26 +-- .../MacroscopicProperties.cpp | 6 +- Source/Parallelization/GuardCellManager.cpp | 2 + Source/Parallelization/WarpXComm_K.H | 34 ++-- Source/Particles/MultiParticleContainer.cpp | 12 +- .../Particles/PhysicalParticleContainer.cpp | 10 +- Source/WarpX.H | 154 ++++++++++-------- Source/WarpX.cpp | 124 ++++++++++++++ .../ablastr/parallelization/MPIInitHelpers.H | 2 +- .../parallelization/MPIInitHelpers.cpp | 20 +-- 33 files changed, 574 insertions(+), 411 deletions(-) diff --git a/.clang-tidy b/.clang-tidy index 5d2422af214..04d1419c5c7 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -7,6 +7,11 @@ Checks: ' -bugprone-unchecked-optional-access, cert-*, -cert-err58-cpp, + clang-analyzer-*, + -clang-analyzer-optin.performance.Padding, + -clang-analyzer-optin.mpi.MPI-Checker, + -clang-analyzer-osx.*, + -clang-analyzer-optin.osx.*, clang-diagnostic-*, cppcoreguidelines-*, -cppcoreguidelines-avoid-c-arrays, diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 805b4fec181..0b2806b3151 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -678,23 +678,23 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri const int ncompe = (m_dive_cleaning) ? 3 : 2; const int ncompb = (m_divb_cleaning) ? 3 : 2; - const amrex::BoxArray ba_Ex = amrex::convert(ba, WarpX::GetInstance().getEfield_fp(0,0).ixType().toIntVect()); - const amrex::BoxArray ba_Ey = amrex::convert(ba, WarpX::GetInstance().getEfield_fp(0,1).ixType().toIntVect()); - const amrex::BoxArray ba_Ez = amrex::convert(ba, WarpX::GetInstance().getEfield_fp(0,2).ixType().toIntVect()); + const amrex::BoxArray ba_Ex = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,0).ixType().toIntVect()); + const amrex::BoxArray ba_Ey = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,1).ixType().toIntVect()); + const amrex::BoxArray ba_Ez = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Efield_fp, 0,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Ex, dm, ncompe, nge, lev, "pml_E_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Ey, dm, ncompe, nge, lev, "pml_E_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_fp[2], ba_Ez, dm, ncompe, nge, lev, "pml_E_fp[z]", 0.0_rt); - const amrex::BoxArray ba_Bx = amrex::convert(ba, WarpX::GetInstance().getBfield_fp(0,0).ixType().toIntVect()); - const amrex::BoxArray ba_By = amrex::convert(ba, WarpX::GetInstance().getBfield_fp(0,1).ixType().toIntVect()); - const amrex::BoxArray ba_Bz = amrex::convert(ba, WarpX::GetInstance().getBfield_fp(0,2).ixType().toIntVect()); + const amrex::BoxArray ba_Bx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect()); + const amrex::BoxArray ba_By = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect()); + const amrex::BoxArray ba_Bz = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Bx, dm, ncompb, ngb, lev, "pml_B_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[1], ba_By, dm, ncompb, ngb, lev, "pml_B_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_fp[2], ba_Bz, dm, ncompb, ngb, lev, "pml_B_fp[z]", 0.0_rt); - const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().getcurrent_fp(0,0).ixType().toIntVect()); - const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().getcurrent_fp(0,1).ixType().toIntVect()); - const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().getcurrent_fp(0,2).ixType().toIntVect()); + const amrex::BoxArray ba_jx = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,0).ixType().toIntVect()); + const amrex::BoxArray ba_jy = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,1).ixType().toIntVect()); + const amrex::BoxArray ba_jz = amrex::convert(ba, WarpX::GetInstance().getField(FieldType::current_fp, 0,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_j_fp[0], ba_jx, dm, 1, ngb, lev, "pml_j_fp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_fp[1], ba_jy, dm, 1, ngb, lev, "pml_j_fp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_fp[2], ba_jz, dm, 1, ngb, lev, "pml_j_fp[z]", 0.0_rt); @@ -812,16 +812,17 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri } else { cdm.define(cba); } - const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,0).ixType().toIntVect()); - const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,1).ixType().toIntVect()); - const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().getEfield_cp(1,2).ixType().toIntVect()); + + const amrex::BoxArray cba_Ex = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,0).ixType().toIntVect()); + const amrex::BoxArray cba_Ey = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,1).ixType().toIntVect()); + const amrex::BoxArray cba_Ez = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Efield_cp, 1,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_E_cp[0], cba_Ex, cdm, ncompe, nge, lev, "pml_E_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_cp[1], cba_Ey, cdm, ncompe, nge, lev, "pml_E_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_cp[2], cba_Ez, cdm, ncompe, nge, lev, "pml_E_cp[z]", 0.0_rt); - const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().getBfield_cp(1,0).ixType().toIntVect()); - const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().getBfield_cp(1,1).ixType().toIntVect()); - const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().getBfield_cp(1,2).ixType().toIntVect()); + const amrex::BoxArray cba_Bx = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,0).ixType().toIntVect()); + const amrex::BoxArray cba_By = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,1).ixType().toIntVect()); + const amrex::BoxArray cba_Bz = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::Bfield_cp, 1,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_cp[0], cba_Bx, cdm, ncompb, ngb, lev, "pml_B_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_cp[1], cba_By, cdm, ncompb, ngb, lev, "pml_B_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_B_cp[2], cba_Bz, cdm, ncompb, ngb, lev, "pml_B_cp[z]", 0.0_rt); @@ -842,9 +843,9 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri WarpX::AllocInitMultiFab( pml_G_cp, cba_G_nodal, cdm, 3, ngf, lev, "pml_G_cp", 0.0_rt); } - const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().getcurrent_cp(1,0).ixType().toIntVect()); - const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().getcurrent_cp(1,1).ixType().toIntVect()); - const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().getcurrent_cp(1,2).ixType().toIntVect()); + const amrex::BoxArray cba_jx = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,0).ixType().toIntVect()); + const amrex::BoxArray cba_jy = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,1).ixType().toIntVect()); + const amrex::BoxArray cba_jz = amrex::convert(cba, WarpX::GetInstance().getField(FieldType::current_cp, 1,2).ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_j_cp[0], cba_jx, cdm, 1, ngb, lev, "pml_j_cp[x]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_cp[1], cba_jy, cdm, 1, ngb, lev, "pml_j_cp[y]", 0.0_rt); WarpX::AllocInitMultiFab(pml_j_cp[2], cba_jz, cdm, 1, ngb, lev, "pml_j_cp[z]", 0.0_rt); diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 7eb011a6bb2..5eb30f43cca 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -41,15 +41,15 @@ PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::Dist m_geom(geom) { - const amrex::MultiFab & Er_fp = WarpX::GetInstance().getEfield_fp(lev,0); - const amrex::MultiFab & Et_fp = WarpX::GetInstance().getEfield_fp(lev,1); + const amrex::MultiFab & Er_fp = WarpX::GetInstance().getField(FieldType::Efield_fp, lev,0); + const amrex::MultiFab & Et_fp = WarpX::GetInstance().getField(FieldType::Efield_fp, lev,1); const amrex::BoxArray ba_Er = amrex::convert(grid_ba, Er_fp.ixType().toIntVect()); const amrex::BoxArray ba_Et = amrex::convert(grid_ba, Et_fp.ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_E_fp[0], ba_Er, grid_dm, Er_fp.nComp(), Er_fp.nGrowVect(), lev, "pml_E_fp[0]", 0.0_rt); WarpX::AllocInitMultiFab(pml_E_fp[1], ba_Et, grid_dm, Et_fp.nComp(), Et_fp.nGrowVect(), lev, "pml_E_fp[1]", 0.0_rt); - const amrex::MultiFab & Br_fp = WarpX::GetInstance().getBfield_fp(lev,0); - const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getBfield_fp(lev,1); + const amrex::MultiFab & Br_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,0); + const amrex::MultiFab & Bt_fp = WarpX::GetInstance().getField(FieldType::Bfield_fp, lev,1); const amrex::BoxArray ba_Br = amrex::convert(grid_ba, Br_fp.ixType().toIntVect()); const amrex::BoxArray ba_Bt = amrex::convert(grid_ba, Bt_fp.ixType().toIntVect()); WarpX::AllocInitMultiFab(pml_B_fp[0], ba_Br, grid_dm, Br_fp.nComp(), Br_fp.nGrowVect(), lev, "pml_B_fp[0]", 0.0_rt); diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 4844585cc90..ebf3d1e39eb 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -22,18 +22,18 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) { if (PEC::isAnyBoundaryPEC()) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoEfield( { get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 1), - get_pointer_Efield_fp(lev, 2) }, lev, patch_type); + PEC::ApplyPECtoEfield( { getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2) }, lev, patch_type); if (WarpX::isAnyBoundaryPML()) { // apply pec on split E-fields in PML region const bool split_pml_field = true; PEC::ApplyPECtoEfield( pml[lev]->GetE_fp(), lev, patch_type, split_pml_field); } } else { - PEC::ApplyPECtoEfield( { get_pointer_Efield_cp(lev, 0), - get_pointer_Efield_cp(lev, 1), - get_pointer_Efield_cp(lev, 2) }, lev, patch_type); + PEC::ApplyPECtoEfield( { getFieldPointer(FieldType::Efield_cp, lev, 0), + getFieldPointer(FieldType::Efield_cp, lev, 1), + getFieldPointer(FieldType::Efield_cp, lev, 2) }, lev, patch_type); if (WarpX::isAnyBoundaryPML()) { // apply pec on split E-fields in PML region const bool split_pml_field = true; @@ -44,13 +44,13 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 1), - get_pointer_Efield_fp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2), lev); } else { - ApplyFieldBoundaryOnAxis(get_pointer_Efield_cp(lev, 0), - get_pointer_Efield_cp(lev, 1), - get_pointer_Efield_cp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Efield_cp, lev, 0), + getFieldPointer(FieldType::Efield_cp, lev, 1), + getFieldPointer(FieldType::Efield_cp, lev, 2), lev); } #endif } @@ -59,13 +59,13 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d { if (PEC::isAnyBoundaryPEC()) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoBfield( { get_pointer_Bfield_fp(lev, 0), - get_pointer_Bfield_fp(lev, 1), - get_pointer_Bfield_fp(lev, 2) }, lev, patch_type); + PEC::ApplyPECtoBfield( { getFieldPointer(FieldType::Bfield_fp, lev, 0), + getFieldPointer(FieldType::Bfield_fp, lev, 1), + getFieldPointer(FieldType::Bfield_fp, lev, 2) }, lev, patch_type); } else { - PEC::ApplyPECtoBfield( { get_pointer_Bfield_cp(lev, 0), - get_pointer_Bfield_cp(lev, 1), - get_pointer_Bfield_cp(lev, 2) }, lev, patch_type); + PEC::ApplyPECtoBfield( { getFieldPointer(FieldType::Bfield_cp, lev, 0), + getFieldPointer(FieldType::Bfield_cp, lev, 1), + getFieldPointer(FieldType::Bfield_cp, lev, 2) }, lev, patch_type); } } @@ -92,13 +92,13 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d #ifdef WARPX_DIM_RZ if (patch_type == PatchType::fine) { - ApplyFieldBoundaryOnAxis(get_pointer_Bfield_fp(lev, 0), - get_pointer_Bfield_fp(lev, 1), - get_pointer_Bfield_fp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_fp, lev, 0), + getFieldPointer(FieldType::Bfield_fp, lev, 1), + getFieldPointer(FieldType::Bfield_fp, lev, 2), lev); } else { - ApplyFieldBoundaryOnAxis(get_pointer_Bfield_cp(lev, 0), - get_pointer_Bfield_cp(lev, 1), - get_pointer_Bfield_cp(lev, 2), lev); + ApplyFieldBoundaryOnAxis(getFieldPointer(FieldType::Bfield_cp, lev, 0), + getFieldPointer(FieldType::Bfield_cp, lev, 1), + getFieldPointer(FieldType::Bfield_cp, lev, 2), lev); } #endif } diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 2422d1324af..40a76eac946 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -565,23 +565,23 @@ BTDiagnostics::InitializeFieldFunctors (int lev) m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ey" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "By" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jx" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 0), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jy" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 1), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 2), lev, m_crse_ratio); } else if ( m_cellcenter_varnames[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio); } @@ -596,7 +596,7 @@ BTDiagnostics::UpdateVarnamesForRZopenPMD () { #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0,0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); const int ncomp = ncomp_multimodefab; @@ -655,7 +655,7 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) { #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0,0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0,0)->nComp(); const int ncomp = ncomp_multimodefab; // Clear any pre-existing vector to release stored data // This ensures that when domain is load-balanced, the functors point @@ -681,23 +681,23 @@ BTDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_cell_center_functors_at_lev_size = static_cast(m_cell_center_functors.at(lev).size()); for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Et" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Ez" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Br" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "Bz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jr" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 0), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 0), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jt" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 1), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 1), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "jz" ){ - m_cell_center_functors[lev][comp] = std::make_unique(warpx.get_pointer_current_fp(lev, 2), lev, m_crse_ratio, false, ncomp); + m_cell_center_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::current_fp, lev, 2), lev, m_crse_ratio, false, ncomp); } else if ( m_cellcenter_varnames_fields[comp] == "rho" ){ m_cell_center_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, false, -1, false, ncomp); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp index c4809df11a3..199b785096d 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformFunctor.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -58,15 +59,22 @@ BackTransformFunctor::operator ()(amrex::MultiFab& mf_dst, int /*dcomp*/, const const bool interpolate = true; std::unique_ptr< amrex::MultiFab > slice = nullptr; const int scomp = 0; + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); + AMREX_ASSUME(m_mf_src != nullptr); + // Generate slice of the cell-centered multifab containing boosted-frame field-data // at current z-boost location for the ith buffer - slice = amrex::get_slice_data(moving_window_dir, - m_current_z_boost[i_buffer], - *m_mf_src, - geom, - scomp, - m_mf_src->nComp(), - interpolate); + slice = amrex::get_slice_data( + moving_window_dir, + m_current_z_boost[i_buffer], + *m_mf_src, + geom, + scomp, + m_mf_src->nComp(), + interpolate); + + // Perform in-place Lorentz-transform of all the fields stored in the slice. LorentzTransformZ( *slice, gamma_boost, beta_boost); diff --git a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp index fe503713141..8c2af223304 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/CellCenterFunctor.cpp @@ -3,6 +3,7 @@ #include "WarpX.H" #include +#include #include #include @@ -17,6 +18,11 @@ void CellCenterFunctor::operator()(amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/) const { auto& warpx = WarpX::GetInstance(); - InterpolateMFForDiag(mf_dst, *m_mf_src, dcomp, warpx.DistributionMap(m_lev), - m_convertRZmodes2cartesian); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); + AMREX_ASSUME(m_mf_src != nullptr); + + InterpolateMFForDiag( + mf_dst, *m_mf_src, dcomp, + warpx.DistributionMap(m_lev),m_convertRZmodes2cartesian); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index 9938cf095bd..a8fd142815a 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -10,6 +10,7 @@ #include "WarpX.H" #include +#include #include #include @@ -27,7 +28,7 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ { auto& warpx = WarpX::GetInstance(); /** pointer to source multifab (can be multi-component) */ - amrex::MultiFab* m_mf_src = warpx.get_pointer_current_fp(m_lev, m_dir); + amrex::MultiFab* m_mf_src = warpx.getFieldPointer(FieldType::current_fp, m_lev, m_dir); // Deposit current if no solver or the electrostatic solver is being used if (m_deposit_current) @@ -36,16 +37,16 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ amrex::Vector, 3 > > current_fp_temp; current_fp_temp.resize(1); - const auto& current_fp_x = warpx.getcurrent_fp(m_lev,0); + const auto& current_fp_x = warpx.getField(FieldType::current_fp, m_lev,0); current_fp_temp[0][0] = std::make_unique( current_fp_x, amrex::make_alias, 0, current_fp_x.nComp() ); - const auto& current_fp_y = warpx.getcurrent_fp(m_lev,1); + const auto& current_fp_y = warpx.getField(FieldType::current_fp, m_lev,1); current_fp_temp[0][1] = std::make_unique( current_fp_y, amrex::make_alias, 0, current_fp_y.nComp() ); - const auto& current_fp_z = warpx.getcurrent_fp(m_lev,2); + const auto& current_fp_z = warpx.getField(FieldType::current_fp, m_lev,2); current_fp_temp[0][2] = std::make_unique( current_fp_z, amrex::make_alias, 0, current_fp_z.nComp() ); @@ -60,6 +61,10 @@ JFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buffer*/ } } - InterpolateMFForDiag(mf_dst, *m_mf_src, dcomp, warpx.DistributionMap(m_lev), - m_convertRZmodes2cartesian); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_mf_src != nullptr, "m_mf_src can't be a nullptr."); + AMREX_ASSUME(m_mf_src != nullptr); + + InterpolateMFForDiag( + mf_dst, *m_mf_src, dcomp, + warpx.DistributionMap(m_lev), m_convertRZmodes2cartesian); } diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index 373004d4633..b73931814b6 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -10,6 +10,7 @@ #include "Particles/MultiParticleContainer.H" #include +#include #include #include @@ -28,24 +29,26 @@ JdispFunctor::operator() (amrex::MultiFab& mf_dst, int dcomp, const int /*i_buff auto* hybrid_pic_model = warpx.get_pointer_HybridPICModel(); /** pointer to total simulation current (J) multifab */ - amrex::MultiFab* mf_j = warpx.get_pointer_current_fp(m_lev, m_dir); - amrex::MultiFab* mf_curlB; - if (hybrid_pic_model) { - /** pointer to current calculated from Ampere's Law (Jamp) multifab */ - mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir); - } else { + amrex::MultiFab* mf_j = warpx.getFieldPointer(FieldType::current_fp, m_lev, m_dir); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(hybrid_pic_model, + "Displacement current diagnostic is only implemented for the HybridPICModel."); + AMREX_ASSUME(hybrid_pic_model != nullptr); + + /** pointer to current calculated from Ampere's Law (Jamp) multifab */ + amrex::MultiFab* mf_curlB = hybrid_pic_model->get_pointer_current_fp_ampere(m_lev, m_dir);; + + //if (!hybrid_pic_model) { // To finish this implementation, we need to implement a method to // calculate (∇ x B). - mf_curlB = nullptr; // Remove when curlB implemented - WARPX_ABORT_WITH_MESSAGE( - "Displacement current diagnostic is only implemented for the HybridPICModel."); // Skeleton for future implementation for solvers other than HybridPIC. // Get curlB multifab // Divide curlB multifab by mu0 to get units of current // mf_curlB->mult(1.0/PhysConsts::mu0) - } + //} + // A Jdisp multifab is generated to hold displacement current. amrex::MultiFab Jdisp( mf_j->boxArray(), mf_j->DistributionMap(), 1, mf_j->nGrowVect() ); Jdisp.setVal(0); diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index b083e60529f..653ac0c26ba 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -62,85 +62,85 @@ FlushFormatCheckpoint::WriteToFile ( for (int lev = 0; lev < nlev; ++lev) { - VisMF::Write(warpx.getEfield_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_fp")); - VisMF::Write(warpx.getEfield_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_fp")); - VisMF::Write(warpx.getEfield_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_fp")); - VisMF::Write(warpx.getBfield_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_fp")); - VisMF::Write(warpx.getBfield_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_fp")); - VisMF::Write(warpx.getBfield_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_fp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(warpx.getEfield_avg_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_fp")); - VisMF::Write(warpx.getEfield_avg_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_fp")); - VisMF::Write(warpx.getEfield_avg_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_avg_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_fp")); - VisMF::Write(warpx.getBfield_avg_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_fp")); - VisMF::Write(warpx.getBfield_avg_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_fp")); - VisMF::Write(warpx.getBfield_avg_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_fp")); } if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(warpx.getcurrent_fp(lev, 0), + VisMF::Write(warpx.getField(FieldType::current_fp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_fp")); - VisMF::Write(warpx.getcurrent_fp(lev, 1), + VisMF::Write(warpx.getField(FieldType::current_fp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_fp")); - VisMF::Write(warpx.getcurrent_fp(lev, 2), + VisMF::Write(warpx.getField(FieldType::current_fp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_fp")); } if (lev > 0) { - VisMF::Write(warpx.getEfield_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_cp")); - VisMF::Write(warpx.getEfield_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_cp")); - VisMF::Write(warpx.getEfield_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_cp")); - VisMF::Write(warpx.getBfield_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_cp")); - VisMF::Write(warpx.getBfield_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_cp")); - VisMF::Write(warpx.getBfield_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_cp")); if (WarpX::fft_do_time_averaging) { - VisMF::Write(warpx.getEfield_avg_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ex_avg_cp")); - VisMF::Write(warpx.getEfield_avg_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ey_avg_cp")); - VisMF::Write(warpx.getEfield_avg_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Efield_avg_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Ez_avg_cp")); - VisMF::Write(warpx.getBfield_avg_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bx_avg_cp")); - VisMF::Write(warpx.getBfield_avg_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "By_avg_cp")); - VisMF::Write(warpx.getBfield_avg_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::Bfield_avg_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "Bz_avg_cp")); } if (warpx.getis_synchronized()) { // Need to save j if synchronized because after restart we need j to evolve E by dt/2. - VisMF::Write(warpx.getcurrent_cp(lev, 0), + VisMF::Write(warpx.getField(FieldType::current_cp, lev, 0), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jx_cp")); - VisMF::Write(warpx.getcurrent_cp(lev, 1), + VisMF::Write(warpx.getField(FieldType::current_cp, lev, 1), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jy_cp")); - VisMF::Write(warpx.getcurrent_cp(lev, 2), + VisMF::Write(warpx.getField(FieldType::current_cp, lev, 2), amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "jz_cp")); } } diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 555ffb123e7..7b10edf0d2b 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -562,84 +562,84 @@ FlushFormatPlotfile::WriteAllRawFields( // Auxiliary patch - WriteRawMF( warpx.getEfield(lev, 0), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield(lev, 1), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield(lev, 2), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield(lev, 0), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield(lev, 1), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield(lev, 2), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 1), dm, raw_pltname, default_level_prefix, "By_aux", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_aux, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_aux", lev, plot_raw_fields_guards); // fine patch - WriteRawMF( warpx.getEfield_fp(lev, 0), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield_fp(lev, 1), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getEfield_fp(lev, 2), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getcurrent_fp(lev, 0), dm, raw_pltname, default_level_prefix, "jx_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getcurrent_fp(lev, 1), dm, raw_pltname, default_level_prefix, "jy_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getcurrent_fp(lev, 2), dm, raw_pltname, default_level_prefix, "jz_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield_fp(lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield_fp(lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); - WriteRawMF( warpx.getBfield_fp(lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); - if (warpx.get_pointer_F_fp(lev)) + WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Ex_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "Ey_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Efield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Ez_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::current_fp, lev, 0), dm, raw_pltname, default_level_prefix, "jx_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::current_fp, lev, 1), dm, raw_pltname, default_level_prefix, "jy_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::current_fp, lev, 2), dm, raw_pltname, default_level_prefix, "jz_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 0), dm, raw_pltname, default_level_prefix, "Bx_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 1), dm, raw_pltname, default_level_prefix, "By_fp", lev, plot_raw_fields_guards); + WriteRawMF( warpx.getField(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, "Bz_fp", lev, plot_raw_fields_guards); + if (warpx.isFieldInitialized(FieldType::F_fp, lev)) { - WriteRawMF(warpx.getF_fp(lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); + WriteRawMF(warpx.getField(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, "F_fp", lev, plot_raw_fields_guards); } - if (warpx.get_pointer_rho_fp(lev)) + if (warpx.isFieldInitialized(FieldType::rho_fp, lev)) { // rho_fp will have either ncomps or 2*ncomps (2 being the old and new). When 2, return the new so // there is time synchronization. - const int nstart = warpx.getrho_fp(lev).nComp() - WarpX::ncomps; - const MultiFab rho_new(warpx.getrho_fp(lev), amrex::make_alias, nstart, WarpX::ncomps); + const int nstart = warpx.getField(FieldType::rho_fp, lev).nComp() - WarpX::ncomps; + const MultiFab rho_new(warpx.getField(FieldType::rho_fp, lev), amrex::make_alias, nstart, WarpX::ncomps); WriteRawMF(rho_new, dm, raw_pltname, default_level_prefix, "rho_fp", lev, plot_raw_fields_guards); } - if (warpx.get_pointer_phi_fp(lev) != nullptr) { - WriteRawMF(warpx.getphi_fp(lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); + if (warpx.isFieldInitialized(FieldType::phi_fp, lev)) { + WriteRawMF(warpx.getField(FieldType::phi_fp, lev), dm, raw_pltname, default_level_prefix, "phi_fp", lev, plot_raw_fields_guards); } // Averaged fields on fine patch if (WarpX::fft_do_time_averaging) { - WriteRawMF(warpx.getEfield_avg_fp(lev, 0) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 0) , dm, raw_pltname, default_level_prefix, "Ex_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getEfield_avg_fp(lev, 1) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 1) , dm, raw_pltname, default_level_prefix, "Ey_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getEfield_avg_fp(lev, 2) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Efield_avg_fp, lev, 2) , dm, raw_pltname, default_level_prefix, "Ez_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getBfield_avg_fp(lev, 0) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 0) , dm, raw_pltname, default_level_prefix, "Bx_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getBfield_avg_fp(lev, 1) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 1) , dm, raw_pltname, default_level_prefix, "By_avg_fp", lev, plot_raw_fields_guards); - WriteRawMF(warpx.getBfield_avg_fp(lev, 2) , dm, raw_pltname, default_level_prefix, + WriteRawMF(warpx.getField(FieldType::Bfield_avg_fp, lev, 2) , dm, raw_pltname, default_level_prefix, "Bz_avg_fp", lev, plot_raw_fields_guards); } // Coarse path if (lev > 0) { WriteCoarseVector( "E", - warpx.get_pointer_Efield_cp(lev, 0), warpx.get_pointer_Efield_cp(lev, 1), warpx.get_pointer_Efield_cp(lev, 2), - warpx.get_pointer_Efield_fp(lev, 0), warpx.get_pointer_Efield_fp(lev, 1), warpx.get_pointer_Efield_fp(lev, 2), + warpx.getFieldPointer(FieldType::Efield_cp, lev, 0), warpx.getFieldPointer(FieldType::Efield_cp, lev, 1), warpx.getFieldPointer(FieldType::Efield_cp, lev, 2), + warpx.getFieldPointer(FieldType::Efield_fp, lev, 0), warpx.getFieldPointer(FieldType::Efield_fp, lev, 1), warpx.getFieldPointer(FieldType::Efield_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "B", - warpx.get_pointer_Bfield_cp(lev, 0), warpx.get_pointer_Bfield_cp(lev, 1), warpx.get_pointer_Bfield_cp(lev, 2), - warpx.get_pointer_Bfield_fp(lev, 0), warpx.get_pointer_Bfield_fp(lev, 1), warpx.get_pointer_Bfield_fp(lev, 2), + warpx.getFieldPointer(FieldType::Bfield_cp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_cp, lev, 2), + warpx.getFieldPointer(FieldType::Bfield_fp, lev, 0), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 1), warpx.getFieldPointer(FieldType::Bfield_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); WriteCoarseVector( "j", - warpx.get_pointer_current_cp(lev, 0), warpx.get_pointer_current_cp(lev, 1), warpx.get_pointer_current_cp(lev, 2), - warpx.get_pointer_current_fp(lev, 0), warpx.get_pointer_current_fp(lev, 1), warpx.get_pointer_current_fp(lev, 2), + warpx.getFieldPointer(FieldType::current_cp, lev, 0), warpx.getFieldPointer(FieldType::current_cp, lev, 1), warpx.getFieldPointer(FieldType::current_cp, lev, 2), + warpx.getFieldPointer(FieldType::current_fp, lev, 0), warpx.getFieldPointer(FieldType::current_fp, lev, 1), warpx.getFieldPointer(FieldType::current_fp, lev, 2), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards); - if (warpx.get_pointer_F_fp(lev) && warpx.get_pointer_F_cp(lev)) + if (warpx.isFieldInitialized(FieldType::F_fp, lev) && warpx.isFieldInitialized(FieldType::F_cp, lev)) { - WriteCoarseScalar("F", warpx.get_pointer_F_cp(lev), warpx.get_pointer_F_fp(lev), + WriteCoarseScalar("F", warpx.getFieldPointer(FieldType::F_cp, lev), warpx.getFieldPointer(FieldType::F_fp, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 0); } - if (warpx.get_pointer_rho_fp(lev) && warpx.get_pointer_rho_cp(lev)) + if (warpx.isFieldInitialized(FieldType::rho_fp, lev) && warpx.isFieldInitialized(FieldType::rho_cp, lev)) { // Use the component 1 of `rho_cp`, i.e. rho_new for time synchronization - WriteCoarseScalar("rho", warpx.get_pointer_rho_cp(lev), warpx.get_pointer_rho_fp(lev), + WriteCoarseScalar("rho", warpx.getFieldPointer(FieldType::rho_cp, lev), warpx.getFieldPointer(FieldType::rho_fp, lev), dm, raw_pltname, default_level_prefix, lev, plot_raw_fields_guards, 1); } } diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index bf4cd3f95fc..63683690483 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -170,15 +170,15 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) #ifdef WARPX_DIM_RZ auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Efield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Efield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Bfield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_current_fp(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::current_fp, lev, dim)->nComp() == ncomp_multimodefab ); } // Species index to loop over species that dump rho per species @@ -211,37 +211,37 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) const auto m_varname_fields_size = static_cast(m_varnames_fields.size()); for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Er"), ncomp); } } else if ( m_varnames_fields[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Et"), ncomp); } } else if ( m_varnames_fields[comp] == "Ez" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Ez"), ncomp); } } else if ( m_varnames_fields[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Br"), ncomp); } } else if ( m_varnames_fields[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Bt"), ncomp); } } else if ( m_varnames_fields[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("Bz"), ncomp); @@ -301,19 +301,19 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) } i++; } else if ( m_varnames_fields[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_F_fp(lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("F"), ncomp); } } else if ( m_varnames_fields[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_G_fp(lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::G_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("G"), ncomp); } } else if ( m_varnames_fields[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_phi_fp(lev), lev, m_crse_ratio, + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("phi"), ncomp); @@ -329,14 +329,16 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) m_varnames.push_back(std::string("part_per_grid")); } } else if ( m_varnames_fields[comp] == "divB" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Bfield_aux(lev), lev, m_crse_ratio, - false, ncomp); + m_all_field_functors[lev][comp] = std::make_unique( + warpx.getFieldPointerArray(FieldType::Bfield_aux, lev), + lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divB"), ncomp); } } else if ( m_varnames_fields[comp] == "divE" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, m_crse_ratio, - false, ncomp); + m_all_field_functors[lev][comp] = std::make_unique( + warpx.getFieldPointerArray(FieldType::Efield_aux, lev), + lev, m_crse_ratio, false, ncomp); if (update_varnames) { AddRZModesToOutputNames(std::string("divE"), ncomp); } @@ -382,15 +384,15 @@ FullDiagnostics::AddRZModesToDiags (int lev) if (!m_dump_rz_modes) { return; } auto & warpx = WarpX::GetInstance(); - const int ncomp_multimodefab = warpx.get_pointer_Efield_aux(0, 0)->nComp(); + const int ncomp_multimodefab = warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp(); // Make sure all multifabs have the same number of components for (int dim=0; dim<3; dim++){ AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Efield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Efield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_Bfield_aux(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim)->nComp() == ncomp_multimodefab ); AMREX_ALWAYS_ASSERT( - warpx.get_pointer_current_fp(lev, dim)->nComp() == ncomp_multimodefab ); + warpx.getFieldPointer(FieldType::current_fp, lev, dim)->nComp() == ncomp_multimodefab ); } // Check if divE is requested @@ -409,8 +411,6 @@ FullDiagnostics::AddRZModesToDiags (int lev) // diagnostic output bool deposit_current = !m_solver_deposits_current; - // First index of m_all_field_functors[lev] where RZ modes are stored - auto icomp =static_cast(m_all_field_functors[0].size()); const std::array coord {"r", "theta", "z"}; // Er, Etheta, Ez, Br, Btheta, Bz, jr, jtheta, jz @@ -422,48 +422,45 @@ FullDiagnostics::AddRZModesToDiags (int lev) if (rho_requested) { n_new_fields += 1; } - m_all_field_functors[lev].resize( m_all_field_functors[0].size() + n_new_fields ); + m_all_field_functors[lev].reserve( m_all_field_functors[0].size() + n_new_fields ); // E for (int dim=0; dim<3; dim++){ // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(warpx.get_pointer_Efield_aux(lev, dim), lev, - m_crse_ratio, false, ncomp_multimodefab); + m_all_field_functors[lev].push_back(std::make_unique( + warpx.getFieldPointer(FieldType::Efield_aux, lev, dim), lev, + m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("E") + coord[dim], - warpx.get_pointer_Efield_aux(0, 0)->nComp()); - icomp += 1; + warpx.getFieldPointer(FieldType::Efield_aux, 0, 0)->nComp()); } // B for (int dim=0; dim<3; dim++){ // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(warpx.get_pointer_Bfield_aux(lev, dim), lev, - m_crse_ratio, false, ncomp_multimodefab); + m_all_field_functors[lev].push_back(std::make_unique( + warpx.getFieldPointer(FieldType::Bfield_aux, lev, dim), lev, + m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("B") + coord[dim], - warpx.get_pointer_Bfield_aux(0, 0)->nComp()); - icomp += 1; + warpx.getFieldPointer(FieldType::Bfield_aux, 0, 0)->nComp()); } // j for (int dim=0; dim<3; dim++){ // 3 components, r theta z - m_all_field_functors[lev][icomp] = - std::make_unique(dim, lev, m_crse_ratio, false, deposit_current, ncomp_multimodefab); + m_all_field_functors[lev].push_back(std::make_unique( + dim, lev, m_crse_ratio, false, deposit_current, ncomp_multimodefab)); deposit_current = false; - icomp += 1; AddRZModesToOutputNames(std::string("J") + coord[dim], - warpx.get_pointer_current_fp(0, 0)->nComp()); + warpx.getFieldPointer(FieldType::current_fp, 0, 0)->nComp()); } // divE if (divE_requested) { - m_all_field_functors[lev][icomp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, - m_crse_ratio, false, ncomp_multimodefab); - icomp += 1; + m_all_field_functors[lev].push_back(std::make_unique( + warpx.getFieldPointerArray(FieldType::Efield_aux, lev), + lev, m_crse_ratio, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("divE"), ncomp_multimodefab); } // rho if (rho_requested) { - m_all_field_functors[lev][icomp] = std::make_unique(lev, m_crse_ratio, true, -1, false, ncomp_multimodefab); - icomp += 1; + m_all_field_functors[lev].push_back(std::make_unique( + lev, m_crse_ratio, true, -1, false, ncomp_multimodefab)); AddRZModesToOutputNames(std::string("rho"), ncomp_multimodefab); } // Sum the number of components in input vector m_all_field_functors @@ -649,16 +646,16 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // Fill vector of functors for all components except individual cylindrical modes. for (int comp=0; comp(warpx.get_pointer_Efield_aux(lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bz" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jz" ){ m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true, deposit_current); deposit_current = false; } else if ( m_varnames[comp] == "jz_displacement" ) { m_all_field_functors[lev][comp] = std::make_unique(2, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Az" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 2), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 2), lev, m_crse_ratio); } else if ( m_varnames[comp] == "rho" ){ // Initialize rho functor to dump total rho m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true); @@ -667,31 +664,31 @@ FullDiagnostics::InitializeFieldFunctors (int lev) m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, m_rho_per_species_index[i]); i++; } else if ( m_varnames[comp] == "F" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_F_fp(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_G_fp(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::G_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "phi" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_phi_fp(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::phi_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_cell" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "part_per_grid" ){ m_all_field_functors[lev][comp] = std::make_unique(nullptr, lev, m_crse_ratio); } else if ( m_varnames[comp] == "divB" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Bfield_aux(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointerArray(FieldType::Bfield_aux, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "divE" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_array_Efield_aux(lev), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointerArray(FieldType::Efield_aux, lev), lev, m_crse_ratio); } else { #ifdef WARPX_DIM_RZ if ( m_varnames[comp] == "Er" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Et" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Br" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bt" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jr" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; @@ -703,24 +700,22 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if (m_varnames[comp] == "jt_displacement" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio, true); } else if ( m_varnames[comp] == "Ar" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "At" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 1), lev, m_crse_ratio); - } - else { - std::cout << "Error on component " << m_varnames[comp] << std::endl; + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 1), lev, m_crse_ratio); + } else { WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for RZ geometry"); } #else // Valid transverse fields in Cartesian coordinates if ( m_varnames[comp] == "Ex" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ey" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Efield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Efield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Bx" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "By" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_Bfield_aux(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::Bfield_aux, lev, 1), lev, m_crse_ratio); } else if ( m_varnames[comp] == "jx" ){ m_all_field_functors[lev][comp] = std::make_unique(0, lev, m_crse_ratio, true, deposit_current); deposit_current = false; @@ -732,9 +727,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) } else if ( m_varnames[comp] == "jy_displacement" ){ m_all_field_functors[lev][comp] = std::make_unique(1, lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ax" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 0), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 0), lev, m_crse_ratio); } else if ( m_varnames[comp] == "Ay" ){ - m_all_field_functors[lev][comp] = std::make_unique(warpx.get_pointer_vector_potential_fp(lev, 1), lev, m_crse_ratio); + m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::vector_potential_fp, lev, 1), lev, m_crse_ratio); } else { std::cout << "Error on component " << m_varnames[comp] << std::endl; WARPX_ABORT_WITH_MESSAGE(m_varnames[comp] + " is not a known field output type for this geometry"); diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index 1398072a84d..7e66061fa48 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -93,9 +93,9 @@ void ChargeOnEB::ComputeDiags (const int step) int const lev = 0; // get MultiFab data at lev - const amrex::MultiFab & Ex = warpx.getEfield_fp(lev,0); - const amrex::MultiFab & Ey = warpx.getEfield_fp(lev,1); - const amrex::MultiFab & Ez = warpx.getEfield_fp(lev,2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_fp, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_fp, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_fp, lev,2); // get EB structures amrex::EBFArrayBoxFactory const& eb_box_factory = warpx.fieldEBFactory(lev); diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 3229f10c89e..88dd0c0922b 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -441,12 +441,12 @@ void ColliderRelevant::ComputeDiags (int step) // define variables in preparation for field gathering const std::array& dx = WarpX::CellSize(std::max(lev, 0)); const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::MultiFab & Ex = warpx.getEfield(lev,0); - const amrex::MultiFab & Ey = warpx.getEfield(lev,1); - const amrex::MultiFab & Ez = warpx.getEfield(lev,2); - const amrex::MultiFab & Bx = warpx.getBfield(lev,0); - const amrex::MultiFab & By = warpx.getBfield(lev,1); - const amrex::MultiFab & Bz = warpx.getBfield(lev,2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // declare reduce_op ReduceOps reduce_op; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index 1271e285c6c..65b33341d1a 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -89,12 +89,12 @@ void FieldEnergy::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = warpx.getEfield(lev,0); - const MultiFab & Ey = warpx.getEfield(lev,1); - const MultiFab & Ez = warpx.getEfield(lev,2); - const MultiFab & Bx = warpx.getBfield(lev,0); - const MultiFab & By = warpx.getBfield(lev,1); - const MultiFab & Bz = warpx.getBfield(lev,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // get cell size Geometry const & geom = warpx.Geom(lev); diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index 8c66404056b..80979cd845e 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -114,12 +114,12 @@ void FieldMaximum::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // get MultiFab data at lev - const MultiFab & Ex = warpx.getEfield(lev,0); - const MultiFab & Ey = warpx.getEfield(lev,1); - const MultiFab & Ez = warpx.getEfield(lev,2); - const MultiFab & Bx = warpx.getBfield(lev,0); - const MultiFab & By = warpx.getBfield(lev,1); - const MultiFab & Bz = warpx.getBfield(lev,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); constexpr int noutputs = 8; // max of Ex,Ey,Ez,|E|,Bx,By,Bz and |B| constexpr int index_Ex = 0; diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index 3b3b7463593..901748bd6a9 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -106,12 +106,12 @@ void FieldMomentum::ComputeDiags (int step) for (int lev = 0; lev < nLevel; ++lev) { // Get MultiFab data at given refinement level - const amrex::MultiFab & Ex = warpx.getEfield(lev, 0); - const amrex::MultiFab & Ey = warpx.getEfield(lev, 1); - const amrex::MultiFab & Ez = warpx.getEfield(lev, 2); - const amrex::MultiFab & Bx = warpx.getBfield(lev, 0); - const amrex::MultiFab & By = warpx.getBfield(lev, 1); - const amrex::MultiFab & Bz = warpx.getBfield(lev, 2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev, 0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev, 1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev, 2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev, 0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev, 1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev, 2); // Cell-centered index type const amrex::GpuArray cc{0,0,0}; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index f0780622d54..4be5a928d96 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -391,12 +391,12 @@ void FieldProbe::ComputeDiags (int step) } // get MultiFab data at lev - const amrex::MultiFab &Ex = warpx.getEfield(lev, 0); - const amrex::MultiFab &Ey = warpx.getEfield(lev, 1); - const amrex::MultiFab &Ez = warpx.getEfield(lev, 2); - const amrex::MultiFab &Bx = warpx.getBfield(lev, 0); - const amrex::MultiFab &By = warpx.getBfield(lev, 1); - const amrex::MultiFab &Bz = warpx.getBfield(lev, 2); + const amrex::MultiFab &Ex = warpx.getField(FieldType::Efield_aux, lev, 0); + const amrex::MultiFab &Ey = warpx.getField(FieldType::Efield_aux, lev, 1); + const amrex::MultiFab &Ez = warpx.getField(FieldType::Efield_aux, lev, 2); + const amrex::MultiFab &Bx = warpx.getField(FieldType::Bfield_aux, lev, 0); + const amrex::MultiFab &By = warpx.getField(FieldType::Bfield_aux, lev, 1); + const amrex::MultiFab &Bz = warpx.getField(FieldType::Bfield_aux, lev, 2); /* * Prepare interpolation of field components to probe_position diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index 940222867b1..a2813bf51e8 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -99,15 +99,15 @@ public: const auto dx = geom.CellSizeArray(); // get MultiFab data - const amrex::MultiFab & Ex = warpx.getEfield(lev,0); - const amrex::MultiFab & Ey = warpx.getEfield(lev,1); - const amrex::MultiFab & Ez = warpx.getEfield(lev,2); - const amrex::MultiFab & Bx = warpx.getBfield(lev,0); - const amrex::MultiFab & By = warpx.getBfield(lev,1); - const amrex::MultiFab & Bz = warpx.getBfield(lev,2); - const amrex::MultiFab & jx = warpx.getcurrent_fp(lev,0); - const amrex::MultiFab & jy = warpx.getcurrent_fp(lev,1); - const amrex::MultiFab & jz = warpx.getcurrent_fp(lev,2); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const amrex::MultiFab & jx = warpx.getField(FieldType::current_fp, lev,0); + const amrex::MultiFab & jy = warpx.getField(FieldType::current_fp, lev,1); + const amrex::MultiFab & jz = warpx.getField(FieldType::current_fp, lev,2); // General preparation of interpolation and reduction operations diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index 8af2c66711c..e5e8a7adc07 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -125,7 +125,7 @@ void LoadBalanceCosts::ComputeDiags (int step) for (int lev = 0; lev < nLevels; ++lev) { const amrex::DistributionMapping& dm = warpx.DistributionMap(lev); - const MultiFab & Ex = warpx.getEfield(lev,0); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); for (MFIter mfi(Ex, false); mfi.isValid(); ++mfi) { const Box& tbx = mfi.tilebox(); diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 0414d9ce869..79b66a92e66 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -373,12 +373,12 @@ void ParticleExtrema::ComputeDiags (int step) // define variables in preparation for field gathering const std::array& dx = WarpX::CellSize(std::max(lev, 0)); const GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const MultiFab & Ex = warpx.getEfield(lev,0); - const MultiFab & Ey = warpx.getEfield(lev,1); - const MultiFab & Ez = warpx.getEfield(lev,2); - const MultiFab & Bx = warpx.getBfield(lev,0); - const MultiFab & By = warpx.getBfield(lev,1); - const MultiFab & Bz = warpx.getBfield(lev,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // declare reduce_op ReduceOps reduce_op; diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index f373e78c9b2..aaadac52610 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -195,27 +195,27 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, amrex::amrex_avgdown_nodes(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio); } - if( SliceType == WarpX::GetInstance().getEfield(0,0).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,0).ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } - if( SliceType == WarpX::GetInstance().getEfield(0,1).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,1).ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } - if( SliceType == WarpX::GetInstance().getEfield(0,2).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Efield_aux, 0,2).ixType().toIntVect() ) { amrex::amrex_avgdown_edges(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } - if( SliceType == WarpX::GetInstance().getBfield(0,0).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,0).ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 0); } - if( SliceType == WarpX::GetInstance().getBfield(0,1).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,1).ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 1); } - if( SliceType == WarpX::GetInstance().getBfield(0,2).ixType().toIntVect() ) { + if( SliceType == WarpX::GetInstance().getField(FieldType::Bfield_aux, 0,2).ixType().toIntVect() ) { amrex::amrex_avgdown_faces(Dst_bx, Dst_fabox, Src_fabox, dcomp, scomp, ncomp, slice_cr_ratio, 2); } diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index f1bda102c8b..755c9d6e2a0 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -320,18 +320,18 @@ WarpX::computePhi (const amrex::Vector >& rho, e_field.push_back( # if defined(WARPX_DIM_1D_Z) amrex::Array{ - get_pointer_Efield_fp(lev, 2) + getFieldPointer(FieldType::Efield_fp, lev, 2) } # elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) amrex::Array{ - get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 2) + getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 2) } # elif defined(WARPX_DIM_3D) amrex::Array{ - get_pointer_Efield_fp(lev, 0), - get_pointer_Efield_fp(lev, 1), - get_pointer_Efield_fp(lev, 2) + getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2) } # endif ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index e8330242fe8..228201886eb 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -149,15 +149,15 @@ void HybridPICModel::InitData () auto & warpx = WarpX::GetInstance(); // Get the grid staggering of the fields involved in calculating E - amrex::IntVect Jx_stag = warpx.getcurrent_fp(0,0).ixType().toIntVect(); - amrex::IntVect Jy_stag = warpx.getcurrent_fp(0,1).ixType().toIntVect(); - amrex::IntVect Jz_stag = warpx.getcurrent_fp(0,2).ixType().toIntVect(); - amrex::IntVect Bx_stag = warpx.getBfield_fp(0,0).ixType().toIntVect(); - amrex::IntVect By_stag = warpx.getBfield_fp(0,1).ixType().toIntVect(); - amrex::IntVect Bz_stag = warpx.getBfield_fp(0,2).ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.getEfield_fp(0,0).ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.getEfield_fp(0,1).ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.getEfield_fp(0,2).ixType().toIntVect(); + amrex::IntVect Jx_stag = warpx.getField(FieldType::current_fp, 0,0).ixType().toIntVect(); + amrex::IntVect Jy_stag = warpx.getField(FieldType::current_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Jz_stag = warpx.getField(FieldType::current_fp, 0,2).ixType().toIntVect(); + amrex::IntVect Bx_stag = warpx.getField(FieldType::Bfield_fp, 0,0).ixType().toIntVect(); + amrex::IntVect By_stag = warpx.getField(FieldType::Bfield_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Bz_stag = warpx.getField(FieldType::Bfield_fp, 0,2).ixType().toIntVect(); + amrex::IntVect Ex_stag = warpx.getField(FieldType::Efield_fp, 0,0).ixType().toIntVect(); + amrex::IntVect Ey_stag = warpx.getField(FieldType::Efield_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Ez_stag = warpx.getField(FieldType::Efield_fp, 0,2).ixType().toIntVect(); // Check that the grid types are appropriate const bool appropriate_grids = ( @@ -224,9 +224,9 @@ void HybridPICModel::InitData () for (int lev = 0; lev <= warpx.finestLevel(); ++lev) { #ifdef AMREX_USE_EB - auto& edge_lengths_x = warpx.getedgelengths(lev, 0); - auto& edge_lengths_y = warpx.getedgelengths(lev, 1); - auto& edge_lengths_z = warpx.getedgelengths(lev, 2); + auto& edge_lengths_x = warpx.getField(FieldType::edge_lengths, lev, 0); + auto& edge_lengths_y = warpx.getField(FieldType::edge_lengths, lev, 1); + auto& edge_lengths_z = warpx.getField(FieldType::edge_lengths, lev, 2); const auto edge_lengths = std::array< std::unique_ptr, 3 >{ std::make_unique( @@ -499,7 +499,7 @@ void HybridPICModel::CalculateElectronPressure(const int lev, DtType a_dt_type) // charge density. if (a_dt_type == DtType::Full) { FillElectronPressureMF( - electron_pressure_fp[lev], warpx.get_pointer_rho_fp(lev) + electron_pressure_fp[lev], warpx.getFieldPointer(FieldType::rho_fp, lev) ); } else { FillElectronPressureMF( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index 965db68a558..c60a0b4c4d5 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -180,9 +180,9 @@ MacroscopicProperties::InitData () amrex::IntVect sigma_stag = m_sigma_mf->ixType().toIntVect(); amrex::IntVect epsilon_stag = m_eps_mf->ixType().toIntVect(); amrex::IntVect mu_stag = m_mu_mf->ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.getEfield_fp(0,0).ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.getEfield_fp(0,1).ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.getEfield_fp(0,2).ixType().toIntVect(); + amrex::IntVect Ex_stag = warpx.getField(FieldType::Efield_fp, 0,0).ixType().toIntVect(); + amrex::IntVect Ey_stag = warpx.getField(FieldType::Efield_fp, 0,1).ixType().toIntVect(); + amrex::IntVect Ez_stag = warpx.getField(FieldType::Efield_fp, 0,2).ixType().toIntVect(); for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { sigma_IndexType[idim] = sigma_stag[idim]; diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index dc7800f490a..28157e09d8c 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -120,9 +120,11 @@ guardCellManager::Init ( #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) ng_alloc_EB = IntVect(ngx,ngz); ng_alloc_J = IntVect(ngJx,ngJz); + amrex::ignore_unused(ngy, ngJy); #elif defined(WARPX_DIM_1D_Z) ng_alloc_EB = IntVect(ngz); ng_alloc_J = IntVect(ngJz); + amrex::ignore_unused(ngx, ngJx, ngy, ngJy); #endif // TODO Adding one cell for rho should not be necessary, given that the number of guard cells diff --git a/Source/Parallelization/WarpXComm_K.H b/Source/Parallelization/WarpXComm_K.H index 4cec33bc84f..a2b8fe38ed4 100644 --- a/Source/Parallelization/WarpXComm_K.H +++ b/Source/Parallelization/WarpXComm_K.H @@ -179,10 +179,6 @@ void warpx_interp (int j, int k, int l, amrex::Real fine = 0.0_rt; amrex::Real coarse = 0.0_rt; - amrex::Real wj; - amrex::Real wk; - amrex::Real wl; - // 1) Interpolation from coarse nodal to fine nodal nj = 2; @@ -197,20 +193,18 @@ void warpx_interp (int j, int k, int l, nl = 2; #endif - wj = 1.0_rt; - wk = 1.0_rt; - wl = 1.0_rt; for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { for (int ll = 0; ll < nl; ll++) { - wj = (rj - amrex::Math::abs(j - (jc + jj) * rj)) / static_cast(rj); + auto c = arr_tmp_zeropad(jc+jj,kc+kk,lc+ll); + c *= (rj - amrex::Math::abs(j - (jc + jj) * rj)) / static_cast(rj); #if (AMREX_SPACEDIM >= 2) - wk = (rk - amrex::Math::abs(k - (kc + kk) * rk)) / static_cast(rk); + c *= (rk - amrex::Math::abs(k - (kc + kk) * rk)) / static_cast(rk); #endif #if (AMREX_SPACEDIM == 3) - wl = (rl - amrex::Math::abs(l - (lc + ll) * rl)) / static_cast(rl); + c *= (rl - amrex::Math::abs(l - (lc + ll) * rl)) / static_cast(rl); #endif - tmp += wj * wk * wl * arr_tmp_zeropad(jc+jj,kc+kk,lc+ll); + tmp += c; } } } @@ -237,20 +231,18 @@ void warpx_interp (int j, int k, int l, kc = amrex::coarsen(kn, rk); lc = amrex::coarsen(ln, rl); - wj = 1.0_rt; - wk = 1.0_rt; - wl = 1.0_rt; for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { for (int ll = 0; ll < nl; ll++) { - wj = (rj - amrex::Math::abs(jn - (jc + jj) * rj)) / static_cast(rj); + auto c = arr_coarse_zeropad(jc+jj,kc+kk,lc+ll); + c *= (rj - amrex::Math::abs(jn - (jc + jj) * rj)) / static_cast(rj); #if (AMREX_SPACEDIM >= 2) - wk = (rk - amrex::Math::abs(kn - (kc + kk) * rk)) / static_cast(rk); + c *= (rk - amrex::Math::abs(kn - (kc + kk) * rk)) / static_cast(rk); #endif #if (AMREX_SPACEDIM == 3) - wl = (rl - amrex::Math::abs(ln - (lc + ll) * rl)) / static_cast(rl); + c *= (rl - amrex::Math::abs(ln - (lc + ll) * rl)) / static_cast(rl); #endif - coarse += wj * wk * wl * arr_coarse_zeropad(jc+jj,kc+kk,lc+ll); + coarse += c; } } } @@ -284,13 +276,11 @@ void warpx_interp (int j, int k, int l, for (int jj = 0; jj < nj; jj++) { for (int kk = 0; kk < nk; kk++) { for (int ll = 0; ll < nl; ll++) { - wj = 1.0_rt / static_cast(nj); - wk = 1.0_rt / static_cast(nk); - wl = 1.0_rt / static_cast(nl); - fine += wj * wk * wl * arr_fine_zeropad(jm+jj,km+kk,lm+ll); + fine += arr_fine_zeropad(jm+jj,km+kk,lm+ll); } } } + fine = fine/static_cast(nj*nk*nl); // Final result arr_aux(j,k,l) = tmp + (fine - coarse); diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index b70844a8d15..97c51c6350b 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -1344,12 +1344,12 @@ MultiParticleContainer::doQEDSchwinger () pc_product_ele->defineAllParticleTiles(); pc_product_pos->defineAllParticleTiles(); - const MultiFab & Ex = warpx.getEfield(level_0,0); - const MultiFab & Ey = warpx.getEfield(level_0,1); - const MultiFab & Ez = warpx.getEfield(level_0,2); - const MultiFab & Bx = warpx.getBfield(level_0,0); - const MultiFab & By = warpx.getBfield(level_0,1); - const MultiFab & Bz = warpx.getBfield(level_0,2); + const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, level_0,0); + const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, level_0,1); + const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, level_0,2); + const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, level_0,0); + const MultiFab & By = warpx.getField(FieldType::Bfield_aux, level_0,1); + const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, level_0,2); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 9923d93a907..4b2d73c7d6a 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2867,11 +2867,11 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, // Using this version of ParallelFor with compile time options // improves performance when qed or external EB are not used by reducing // register pressure. - amrex::ParallelFor(TypeList, - CompileTimeOptions>{}, - {exteb_runtime_flag, qed_runtime_flag}, - np_to_push, [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, - auto qed_control) + amrex::ParallelFor( + TypeList, CompileTimeOptions>{}, + {exteb_runtime_flag, qed_runtime_flag}, + np_to_push, + [=] AMREX_GPU_DEVICE (long ip, auto exteb_control, auto qed_control) { amrex::ParticleReal xp, yp, zp; getPosition(ip, xp, yp, zp); diff --git a/Source/WarpX.H b/Source/WarpX.H index 0c86a1c9afe..3be95f47865 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -69,12 +69,11 @@ #include #include #include +#include #include #include #include #include -#include - enum struct PatchType : int { @@ -82,6 +81,33 @@ enum struct PatchType : int coarse }; +enum struct FieldType : int +{ + Efield_aux, + Bfield_aux, + Efield_fp, + Bfield_fp, + current_fp, + current_fp_nodal, + rho_fp, + F_fp, + G_fp, + phi_fp, + vector_potential_fp, + Efield_cp, + Bfield_cp, + current_cp, + rho_cp, + F_cp, + G_cp, + edge_lengths, + face_areas, + Efield_avg_fp, + Bfield_avg_fp, + Efield_avg_cp, + Bfield_avg_cp +}; + class WARPX_EXPORT WarpX : public amrex::AmrCore { @@ -463,72 +489,56 @@ public: static std::map multifab_map; static std::map imultifab_map; - [[nodiscard]] std::array - get_array_Bfield_aux (const int lev) const { - return { - Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get() - }; - } + /** + * \brief + * Check if a field is initialized. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return true if the field is initialized, false otherwise + */ + [[nodiscard]] bool + isFieldInitialized (FieldType field_type, int lev, int direction = 0) const; + + /** + * \brief + * Get a pointer to the field data. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return the pointer to an amrex::MultiFab containing the field data + */ + [[nodiscard]] amrex::MultiFab* + getFieldPointer (FieldType field_type, int lev, int direction = 0) const; + /** + * \brief + * For vector fields, get an array of three pointers to the field data. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * + * \return an array of three pointers amrex::MultiFab* containing the field data + */ [[nodiscard]] std::array - get_array_Efield_aux (const int lev) const { - return { - Efield_aux[lev][0].get(), - Efield_aux[lev][1].get(), - Efield_aux[lev][2].get() - }; - } + getFieldPointerArray (FieldType field_type, int lev) const; - [[nodiscard]] amrex::MultiFab * get_pointer_Efield_aux (int lev, int direction) const { return Efield_aux[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_aux (int lev, int direction) const { return Bfield_aux[lev][direction].get(); } - - [[nodiscard]] amrex::MultiFab * get_pointer_Efield_fp (int lev, int direction) const { return Efield_fp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_fp (int lev, int direction) const { return Bfield_fp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_current_fp (int lev, int direction) const { return current_fp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_current_fp_nodal (int lev, int direction) const { return current_fp_nodal[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_rho_fp (int lev) const { return rho_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_F_fp (int lev) const { return F_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_G_fp (int lev) const { return G_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_phi_fp (int lev) const { return phi_fp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_vector_potential_fp (int lev, int direction) const { return vector_potential_fp_nodal[lev][direction].get(); } - - [[nodiscard]] amrex::MultiFab * get_pointer_Efield_cp (int lev, int direction) const { return Efield_cp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_Bfield_cp (int lev, int direction) const { return Bfield_cp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_current_cp (int lev, int direction) const { return current_cp[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_rho_cp (int lev) const { return rho_cp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_F_cp (int lev) const { return F_cp[lev].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_G_cp (int lev) const { return G_cp[lev].get(); } - - [[nodiscard]] amrex::MultiFab * get_pointer_edge_lengths (int lev, int direction) const { return m_edge_lengths[lev][direction].get(); } - [[nodiscard]] amrex::MultiFab * get_pointer_face_areas (int lev, int direction) const { return m_face_areas[lev][direction].get(); } - - const amrex::MultiFab& getEfield (int lev, int direction) {return *Efield_aux[lev][direction];} - const amrex::MultiFab& getBfield (int lev, int direction) {return *Bfield_aux[lev][direction];} - - const amrex::MultiFab& getcurrent_cp (int lev, int direction) {return *current_cp[lev][direction];} - const amrex::MultiFab& getEfield_cp (int lev, int direction) {return *Efield_cp[lev][direction];} - const amrex::MultiFab& getBfield_cp (int lev, int direction) {return *Bfield_cp[lev][direction];} - const amrex::MultiFab& getrho_cp (int lev) {return *rho_cp[lev];} - const amrex::MultiFab& getF_cp (int lev) {return *F_cp[lev];} - const amrex::MultiFab& getG_cp (int lev) {return *G_cp[lev];} - - const amrex::MultiFab& getcurrent_fp (int lev, int direction) {return *current_fp[lev][direction];} - const amrex::MultiFab& getEfield_fp (int lev, int direction) {return *Efield_fp[lev][direction];} - const amrex::MultiFab& getBfield_fp (int lev, int direction) {return *Bfield_fp[lev][direction];} - const amrex::MultiFab& getrho_fp (int lev) {return *rho_fp[lev];} - const amrex::MultiFab& getphi_fp (int lev) {return *phi_fp[lev];} - const amrex::MultiFab& getF_fp (int lev) {return *F_fp[lev];} - const amrex::MultiFab& getG_fp (int lev) {return *G_fp[lev];} - - const amrex::MultiFab& getEfield_avg_fp (int lev, int direction) {return *Efield_avg_fp[lev][direction];} - const amrex::MultiFab& getBfield_avg_fp (int lev, int direction) {return *Bfield_avg_fp[lev][direction];} - const amrex::MultiFab& getEfield_avg_cp (int lev, int direction) {return *Efield_avg_cp[lev][direction];} - const amrex::MultiFab& getBfield_avg_cp (int lev, int direction) {return *Bfield_avg_cp[lev][direction];} - - const amrex::MultiFab& getedgelengths (int lev, int direction) {return *m_edge_lengths[lev][direction];} - const amrex::MultiFab& getfaceareas (int lev, int direction) {return *m_face_areas[lev][direction];} + /** + * \brief + * Get a constant reference to the field data. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return a constant refernce to an amrex::MultiFab containing the field data + */ + [[nodiscard]] const amrex::MultiFab& + getField(FieldType field_type, int lev, int direction = 0) const; [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} @@ -1586,6 +1596,20 @@ private: amrex::Vector, 3 > > current_buf; amrex::Vector > charge_buf; + /** + * \brief + * Get a pointer to the field data. Does not check if the pointer + * is not nullptr. + * + * \param field_type[in] the field type + * \param lev[in] the mesh refinement level + * \param direction[in] the field component (0 by default) + * + * \return the pointer to an amrex::MultiFab containing the field data + */ + [[nodiscard]] amrex::MultiFab* + getFieldPointerUnchecked (FieldType field_type, int lev, int direction = 0) const; + // PML int do_pml = 0; int do_silver_mueller = 0; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 83f48394400..11646927863 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -3370,3 +3370,127 @@ WarpX::AllocInitMultiFabFromModel ( } multifab_map[name_with_suffix] = mf.get(); } + +amrex::MultiFab* +WarpX::getFieldPointerUnchecked (const FieldType field_type, const int lev, const int direction) const +{ + // This function does *not* check if the returned field pointer is != nullptr + + amrex::MultiFab* field_pointer = nullptr; + + switch(field_type) + { + case FieldType::Efield_aux : + field_pointer = Efield_aux[lev][direction].get(); + break; + case FieldType::Bfield_aux : + field_pointer = Bfield_aux[lev][direction].get(); + break; + case FieldType::Efield_fp : + field_pointer = Efield_fp[lev][direction].get(); + break; + case FieldType::Bfield_fp : + field_pointer = Bfield_fp[lev][direction].get(); + break; + case FieldType::current_fp : + field_pointer = current_fp[lev][direction].get(); + break; + case FieldType::current_fp_nodal : + field_pointer = current_fp_nodal[lev][direction].get(); + break; + case FieldType::rho_fp : + field_pointer = rho_fp[lev].get(); + break; + case FieldType::F_fp : + field_pointer = F_fp[lev].get(); + break; + case FieldType::G_fp : + field_pointer = G_fp[lev].get(); + break; + case FieldType::phi_fp : + field_pointer = phi_fp[lev].get(); + break; + case FieldType::vector_potential_fp : + field_pointer = vector_potential_fp_nodal[lev][direction].get(); + break; + case FieldType::Efield_cp : + field_pointer = Efield_cp[lev][direction].get(); + break; + case FieldType::Bfield_cp : + field_pointer = Bfield_cp[lev][direction].get(); + break; + case FieldType::current_cp : + field_pointer = current_cp[lev][direction].get(); + break; + case FieldType::rho_cp : + field_pointer = rho_cp[lev].get(); + break; + case FieldType::F_cp : + field_pointer = F_cp[lev].get(); + break; + case FieldType::G_cp : + field_pointer = G_cp[lev].get(); + break; + case FieldType::edge_lengths : + field_pointer = m_edge_lengths[lev][direction].get(); + break; + case FieldType::face_areas : + field_pointer = m_face_areas[lev][direction].get(); + break; + case FieldType::Efield_avg_fp : + field_pointer = Efield_avg_fp[lev][direction].get(); + break; + case FieldType::Bfield_avg_fp : + field_pointer = Bfield_avg_fp[lev][direction].get(); + break; + case FieldType::Efield_avg_cp : + field_pointer = Efield_avg_cp[lev][direction].get(); + break; + case FieldType::Bfield_avg_cp : + field_pointer = Bfield_avg_cp[lev][direction].get(); + break; + default: + WARPX_ABORT_WITH_MESSAGE("Invalid field type"); + break; + } + + return field_pointer; +} + +bool +WarpX::isFieldInitialized (const FieldType field_type, const int lev, const int direction) const +{ + const bool is_field_init = (getFieldPointerUnchecked(field_type, lev, direction) != nullptr); + return is_field_init; +} + +amrex::MultiFab* +WarpX::getFieldPointer (const FieldType field_type, const int lev, const int direction) const +{ + auto* const field_pointer = getFieldPointerUnchecked(field_type, lev, direction); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + field_pointer != nullptr, "Requested field is not initialized!"); + return field_pointer; +} + +std::array +WarpX::getFieldPointerArray (const FieldType field_type, const int lev) const +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (field_type == FieldType::Efield_aux) || (field_type == FieldType::Bfield_aux) || + (field_type == FieldType::Efield_fp) || (field_type == FieldType::Bfield_fp) || + (field_type == FieldType::current_fp) || (field_type == FieldType::current_fp_nodal) || + (field_type == FieldType::Efield_cp) || (field_type == FieldType::Bfield_cp) || + (field_type == FieldType::current_cp), "Requested field type is not a vector."); + + return std::array{ + getFieldPointer(field_type, lev, 0), + getFieldPointer(field_type, lev, 1), + getFieldPointer(field_type, lev, 2)}; +} + +const amrex::MultiFab& +WarpX::getField(FieldType field_type, const int lev, const int direction) const +{ + return *getFieldPointer(field_type, lev, direction); +} diff --git a/Source/ablastr/parallelization/MPIInitHelpers.H b/Source/ablastr/parallelization/MPIInitHelpers.H index 6b9bdfa4737..660e5e56d06 100644 --- a/Source/ablastr/parallelization/MPIInitHelpers.H +++ b/Source/ablastr/parallelization/MPIInitHelpers.H @@ -15,7 +15,7 @@ namespace ablastr::parallelization * * @return the MPI_THREAD_* level required for MPI_Init_thread */ - int + constexpr int mpi_thread_required (); /** Initialize MPI diff --git a/Source/ablastr/parallelization/MPIInitHelpers.cpp b/Source/ablastr/parallelization/MPIInitHelpers.cpp index 633c004c93a..513035c483c 100644 --- a/Source/ablastr/parallelization/MPIInitHelpers.cpp +++ b/Source/ablastr/parallelization/MPIInitHelpers.cpp @@ -29,20 +29,20 @@ namespace ablastr::parallelization { - int + constexpr int mpi_thread_required () { - int thread_required = -1; #ifdef AMREX_USE_MPI - thread_required = MPI_THREAD_SINGLE; // equiv. to MPI_Init -# ifdef AMREX_USE_OMP - thread_required = MPI_THREAD_FUNNELED; -# endif # ifdef AMREX_MPI_THREAD_MULTIPLE // i.e. for async_io - thread_required = MPI_THREAD_MULTIPLE; + return MPI_THREAD_MULTIPLE; +# elif AMREX_USE_OMP + return MPI_THREAD_FUNNELED; +# else + return MPI_THREAD_SINGLE; // equiv. to MPI_Init # endif +#else + return -1; #endif - return thread_required; } std::pair< int, int > @@ -58,7 +58,7 @@ namespace ablastr::parallelization } #endif - const int thread_required = mpi_thread_required(); + constexpr int thread_required = mpi_thread_required(); #ifdef AMREX_USE_MPI int thread_provided = -1; MPI_Init_thread(&argc, &argv, thread_required, &thread_provided); @@ -82,7 +82,7 @@ namespace ablastr::parallelization check_mpi_thread_level () { #ifdef AMREX_USE_MPI - const int thread_required = mpi_thread_required(); + constexpr int thread_required = mpi_thread_required(); int thread_provided = -1; MPI_Query_thread(&thread_provided); auto mtn = amrex::ParallelDescriptor::mpi_level_to_string; From 63807df9f1c63de6c71ee2ac0944a6cb17ce2262 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 1 Apr 2024 16:03:28 -0700 Subject: [PATCH 023/190] Doc: Perlmutter Update (#4809) - use a TMPFS build directory instead of HOME: less quota strain, faster - update optimas install: from PyPI, install torch on CPU, too - rename venvs: ensure -cpu or -gpu is seen on the prompt when activated --- .../install_cpu_dependencies.sh | 44 ++++++++++------- .../install_gpu_dependencies.sh | 47 ++++++++++--------- .../perlmutter_cpu_warpx.profile.example | 4 +- .../perlmutter_gpu_warpx.profile.example | 4 +- 4 files changed, 56 insertions(+), 43 deletions(-) diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index dbe59438a16..f65e43891d0 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -44,6 +44,9 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + # c-blosc (I/O compression) if [ -d $HOME/src/c-blosc ] then @@ -55,9 +58,9 @@ else git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc fi rm -rf $HOME/src/c-blosc-pm-cpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-pm-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-pm-cpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-pm-cpu-build +cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-pm-cpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pm-cpu-build # ADIOS2 if [ -d $HOME/src/adios2 ] @@ -70,9 +73,9 @@ else git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-pm-cpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_CUDA=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-pm-cpu-build --target install -j 16 -rm -rf $HOME/src/adios2-pm-cpu-build +cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-pm-cpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_CUDA=OFF -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build ${build_dir}/adios2-pm-cpu-build --target install -j 16 +rm -rf ${build_dir}/adios2-pm-cpu-build # BLAS++ (for PSATD+RZ) if [ -d $HOME/src/blaspp ] @@ -86,9 +89,9 @@ else git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-cpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-pm-cpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-pm-cpu-build +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pm-cpu-build # LAPACK++ (for PSATD+RZ) if [ -d $HOME/src/lapackpp ] @@ -102,9 +105,9 @@ else git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-cpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-pm-cpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-pm-cpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pm-cpu-build # Python ###################################################################### @@ -112,9 +115,9 @@ rm -rf $HOME/src/lapackpp-pm-cpu-build python3 -m pip install --upgrade pip python3 -m pip install --upgrade virtualenv python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx -python3 -m venv ${SW_DIR}/venvs/warpx -source ${SW_DIR}/venvs/warpx/bin/activate +rm -rf ${SW_DIR}/venvs/warpx-cpu +python3 -m venv ${SW_DIR}/venvs/warpx-cpu +source ${SW_DIR}/venvs/warpx-cpu/bin/activate python3 -m pip install --upgrade pip python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging @@ -128,7 +131,12 @@ MPICC="cc -shared" python3 -m pip install --upgrade mpi4py --no-cache-dir --no-b python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard +# install or update WarpX dependencies python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +# optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/cpu +python3 -m pip install --upgrade optimas[all] + + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index 3fe76359953..9ac5800c6ce 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -44,6 +44,9 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + # c-blosc (I/O compression) if [ -d $HOME/src/c-blosc ] then @@ -55,9 +58,9 @@ else git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc fi rm -rf $HOME/src/c-blosc-pm-gpu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-pm-gpu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-pm-gpu-build +cmake -S $HOME/src/c-blosc -B ${build_dir}/c-blosc-pm-gpu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pm-gpu-build # ADIOS2 if [ -d $HOME/src/adios2 ] @@ -70,9 +73,9 @@ else git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-pm-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build $HOME/src/adios2-pm-gpu-build --target install -j 16 -rm -rf $HOME/src/adios2-pm-gpu-build +cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake --build ${build_dir}/adios2-pm-gpu-build --target install -j 16 +rm -rf ${build_dir}/adios2-pm-gpu-build # BLAS++ (for PSATD+RZ) if [ -d $HOME/src/blaspp ] @@ -86,9 +89,9 @@ else git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-pm-gpu-build +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pm-gpu-build # LAPACK++ (for PSATD+RZ) if [ -d $HOME/src/lapackpp ] @@ -102,9 +105,9 @@ else git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-pm-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pm-gpu-build # Python ###################################################################### @@ -112,9 +115,9 @@ rm -rf $HOME/src/lapackpp-pm-gpu-build python3 -m pip install --upgrade pip python3 -m pip install --upgrade virtualenv python3 -m pip cache purge -rm -rf ${SW_DIR}/venvs/warpx -python3 -m venv ${SW_DIR}/venvs/warpx -source ${SW_DIR}/venvs/warpx/bin/activate +rm -rf ${SW_DIR}/venvs/warpx-gpu +python3 -m venv ${SW_DIR}/venvs/warpx-gpu +source ${SW_DIR}/venvs/warpx-gpu/bin/activate python3 -m pip install --upgrade pip python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging @@ -128,11 +131,13 @@ MPICC="cc -target-accel=nvidia80 -shared" python3 -m pip install --upgrade mpi4p python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt -# install or update WarpX dependencies such as picmistandard +# install or update WarpX dependencies python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt -python3 -m pip install cupy-cuda12x # CUDA 12 compatible wheel -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt -# optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade cupy-cuda12x # CUDA 12 compatible wheel +# optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) python3 -m pip install --upgrade torch # CUDA 12 compatible wheel -python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt +python3 -m pip install --upgrade optimas[all] + + +# remove build temporary directory +rm -rf ${build_dir} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 67ae36c9f1a..1b0ac3182d5 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -33,9 +33,9 @@ export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/ # optional: for Python bindings or libEnsemble module load cray-python/3.11.5 -if [ -d "${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx" ] +if [ -d "${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu" ] then - source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx/bin/activate + source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu/bin/activate fi # an alias to request an interactive batch node for one hour diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 7bf5dd13116..759df0b923a 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -37,9 +37,9 @@ export PATH=/global/common/software/spackecp/perlmutter/e4s-23.08/default/spack/ # optional: for Python bindings or libEnsemble module load cray-python/3.11.5 -if [ -d "${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx" ] +if [ -d "${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu" ] then - source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx/bin/activate + source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu/bin/activate fi # an alias to request an interactive batch node for one hour From 5a8fdd8d3ebfbaf56960716ee4773571c5ffe059 Mon Sep 17 00:00:00 2001 From: Olga Shapoval <30510597+oshapoval@users.noreply.github.com> Date: Tue, 2 Apr 2024 06:22:08 -0700 Subject: [PATCH 024/190] Printing out the time dependencies of the charge and current densities with multi-J PSATD. (#4815) --- Source/Initialization/WarpXInitData.cpp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 5309b878f08..59459f7ad63 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -300,6 +300,18 @@ WarpX::PrintMainPICparameters () amrex::Print() << " | - multi-J deposition is ON \n"; amrex::Print() << " | - do_multi_J_n_depositions = " << WarpX::do_multi_J_n_depositions << "\n"; + if (J_in_time == JInTime::Linear){ + amrex::Print() << " | - J_in_time = linear \n"; + } + if (J_in_time == JInTime::Constant){ + amrex::Print() << " | - J_in_time = constant \n"; + } + if (rho_in_time == RhoInTime::Linear){ + amrex::Print() << " | - rho_in_time = linear \n"; + } + if (rho_in_time == RhoInTime::Constant){ + amrex::Print() << " | - rho_in_time = constant \n"; + } } if (fft_do_time_averaging == 1){ amrex::Print()<<" | - time-averaged is ON \n"; From 5e07c8047fcafd7fd945bfaa1fa78223d2738c10 Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Tue, 2 Apr 2024 16:18:07 -0700 Subject: [PATCH 025/190] updated docs about focal distance (#4762) * updated docs about focal distance --- Docs/source/refs.bib | 9 +++++++++ Docs/source/usage/parameters.rst | 15 ++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 6c7665dac84..1746e2285cd 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -393,3 +393,12 @@ @article{GrismayerNJP2021 volume = {23}, year = {2021} } + +@book{Wiedemann2015, +author = {Wiedemann, H.}, +isbn = {978-3-319-18317-6}, +publisher = {Springer Cham}, +title = {{Particle Accelerator Physics}}, +doi = {https://doi.org/10.1007/978-3-319-18317-6}, +year = {2015} +} diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index fa1d5e99670..1eaf22f0183 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -810,7 +810,20 @@ Particle initialization * ``.focal_distance`` (optional, distance between the beam centroid and the position of the focal plane of the beam, along the direction of the beam mean velocity; space charge is ignored in the initialization of the particles) - If ``.focal_distance`` is specified, ``x_rms``, ``y_rms`` and ``z_rms`` are the size of the beam in the focal plane. Since the beam is not necessarily initialized close to its focal plane, the initial size of the beam will differ from ``x_rms``, ``y_rms``, ``z_rms``. + If ``.focal_distance`` is specified, ``x_rms``, ``y_rms`` and ``z_rms`` are the sizes of the beam in the focal plane. Since the beam is not necessarily initialized close to its focal plane, the initial size of the beam will differ from ``x_rms``, ``y_rms``, ``z_rms``. + + Usually, in accelerator physics the operative quantities are the normalized emittances :math:`\epsilon_{x,y}` and beta functions :math:`\beta_{x,y}`. + We assume that the beam travels along :math:`z` and we mark the quantities evaluated at the focal plane with a :math:`*`. + Therefore, the normalized transverse emittances and beta functions are related to the focal distance :math:`f = z - z^*`, the beam sizes :math:`\sigma_{x,y}` (which in the code are ``x_rms``, ``y_rms``), the beam relativistic Lorentz factor :math:`\gamma`, and the normalized momentum spread :math:`\Delta u_{x,y}` according to the equations below (:cite:t:`param-Wiedemann2015`). + + .. math:: + + \Delta u_{x,y} &= \frac{\epsilon^*_{x,y}}{\sigma^*_{x,y}}, + + \sigma*_{x, y} &= \sqrt{ \frac{ \epsilon^*_{x,y} \beta^*_{x,y} }{\gamma}}, + + \sigma_{x,y}(z) &= \sigma^*_{x,y} \sqrt{1 + \left( \frac{z - z^*}{\beta^*_{x,y}} \right)^2} + * ``external_file``: Inject macroparticles with properties (mass, charge, position, and momentum - :math:`\gamma \beta m c`) read from an external openPMD file. With it users can specify the additional arguments: From 30ed0fbac4c0c4cf47f10e2a255c52c50166c79f Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Wed, 3 Apr 2024 16:27:47 -0700 Subject: [PATCH 026/190] Implement Poisson solver based on integrated Green functions (#4648) * Allocate 2x rho * Copy rho * Allocate G * Implemented Green function * Filled values of the integrated Green function * Call computePhiIGF * Added test script and correct periodicity * Do not call the Poisson solver if the charge is 0 * Correct compilation errors * Perform copy back to phi * Correct a few bugs * Improve comparison with theory * Simplify implementation * Add relativistic factor * Use new distribution mapping * Use ParallelCopy for rho and phi * Use correct DistributionMapping * Cleaner implementation with nx,ny,nz * More compact syntax for BoxArrays * Avoid floating-point error * Fixes to avoid issues with photon containers and empty containers * Add more timers * Fix ParallelCopy usage * Use Taylor expansion for pathological terms * Use more accurate expression * Update Source/ablastr/fields/PoissonSolver.H * Correctly fill ghost cells of phi * Update Green function formula * start automated test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add warpx.poisson_solver to test * moved computePhiIGF in separate file * updated test analysis * add user interface and checks * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed BC assert * add ifdefs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * removed unused import * fix bracket * fix checks * check 3D and PSATD * fix and add asserts * fix rho mult by eps0 * fix typo * update test * reset benchmark from azure * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix clang-tidy tantrums * add literals * other checks * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * forgot WarpX:: * forgot semicolons :| * docs and comments * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * forgot semicolon again * Update Source/ablastr/fields/IntegratedGreenFunctionSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/fields/IntegratedGreenFunctionSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/fields/IntegratedGreenFunctionSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/math/fft/WrapCuFFT.cpp Co-authored-by: Axel Huebl * Update Source/ablastr/fields/IntegratedGreenFunctionSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/fields/PoissonSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/fields/PoissonSolver.H Co-authored-by: Axel Huebl * false for device syncs * remove WarpX:: * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add maybe unused * fixed solver check * update test inputs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Source/FieldSolver/ElectrostaticSolver.cpp Co-authored-by: Axel Huebl * Update Source/WarpX.cpp Co-authored-by: Axel Huebl * Update Source/FieldSolver/ElectrostaticSolver.cpp Co-authored-by: Axel Huebl * Update Source/FieldSolver/ElectrostaticSolver.cpp Co-authored-by: Axel Huebl * Update Source/FieldSolver/ElectrostaticSolver.cpp Co-authored-by: Axel Huebl * Update Source/Utils/WarpXUtil.cpp Co-authored-by: Axel Huebl * Update Source/Initialization/WarpXInitData.cpp Co-authored-by: Axel Huebl * renamed solvers * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * avoid rescaling rho twice * fix doxy string * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Simplify is_solver_multigrid * Update Docs/source/usage/parameters.rst Co-authored-by: Axel Huebl * Update Docs/source/usage/parameters.rst Co-authored-by: Axel Huebl * Update Examples/Tests/openbc_poisson_solver/inputs_3d Co-authored-by: Axel Huebl * Update Docs/source/refs.bib Co-authored-by: Axel Huebl * Update Source/FieldSolver/ElectrostaticSolver.cpp Co-authored-by: Axel Huebl * adds cpp in ablastr/fields * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add Missing Files * `BaseFab`: Fix Complex Type All complex types are equal, but some complex types are more equal than others. -- George Orwell * Cleaning: Includes * Cleaning: Const Refs and Doxygen * Update Source/Utils/WarpXAlgorithmSelection.cpp * Update Source/FieldSolver/ElectrostaticSolver.cpp * Fix .bib File Syntax * Update Regression/Checksum/benchmarks_json/openbc_poisson_solver.json --------- Co-authored-by: Remi Lehe Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Tools Co-authored-by: Axel Huebl --- Docs/source/refs.bib | 40 +++- Docs/source/usage/parameters.rst | 36 +++- .../Tests/openbc_poisson_solver/analysis.py | 65 ++++++ .../Tests/openbc_poisson_solver/inputs_3d | 46 ++++ .../openbc_poisson_solver.json | 20 ++ Regression/WarpX-tests.ini | 18 ++ Source/FieldSolver/ElectrostaticSolver.cpp | 43 +++- Source/Initialization/WarpXInitData.cpp | 13 +- Source/Utils/WarpXAlgorithmSelection.H | 12 +- Source/Utils/WarpXAlgorithmSelection.cpp | 9 + Source/Utils/WarpXUtil.cpp | 9 + Source/WarpX.H | 1 + Source/WarpX.cpp | 22 ++ Source/ablastr/CMakeLists.txt | 2 +- Source/ablastr/Make.package | 1 + Source/ablastr/fields/CMakeLists.txt | 9 + .../fields/IntegratedGreenFunctionSolver.H | 66 ++++++ .../fields/IntegratedGreenFunctionSolver.cpp | 196 ++++++++++++++++++ Source/ablastr/fields/Make.package | 7 + Source/ablastr/fields/PoissonSolver.H | 33 ++- Source/ablastr/math/fft/WrapCuFFT.cpp | 4 + 21 files changed, 625 insertions(+), 27 deletions(-) create mode 100755 Examples/Tests/openbc_poisson_solver/analysis.py create mode 100644 Examples/Tests/openbc_poisson_solver/inputs_3d create mode 100644 Regression/Checksum/benchmarks_json/openbc_poisson_solver.json create mode 100644 Source/ablastr/fields/CMakeLists.txt create mode 100644 Source/ablastr/fields/IntegratedGreenFunctionSolver.H create mode 100644 Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp create mode 100644 Source/ablastr/fields/Make.package diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 1746e2285cd..2de27908c5d 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -394,11 +394,39 @@ @article{GrismayerNJP2021 year = {2021} } +@article{QiangPhysRevSTAB2006, + title = {Three-dimensional quasistatic model for high brightness beam dynamics simulation}, + author = {Qiang, Ji and Lidia, Steve and Ryne, Robert D. and Limborg-Deprey, Cecile}, + journal = {Phys. Rev. ST Accel. Beams}, + volume = {9}, + issue = {4}, + pages = {044204}, + numpages = {10}, + year = {2006}, + month = {Apr}, + publisher = {American Physical Society}, + doi = {10.1103/PhysRevSTAB.9.044204} +} + +@article{QiangPhysRevSTAB2006err, + title = {Erratum: Three-dimensional quasistatic model for high brightness beam dynamics simulation [Phys. Rev. ST Accel. Beams 9, 044204 (2006)]}, + author = {Qiang, Ji and Lidia, Steve and Ryne, Robert D. and Limborg-Deprey, Cecile}, + journal = {Phys. Rev. ST Accel. Beams}, + volume = {10}, + issue = {12}, + pages = {129901}, + numpages = {2}, + year = {2007}, + month = {Dec}, + publisher = {American Physical Society}, + doi = {10.1103/PhysRevSTAB.10.129901} +} + @book{Wiedemann2015, -author = {Wiedemann, H.}, -isbn = {978-3-319-18317-6}, -publisher = {Springer Cham}, -title = {{Particle Accelerator Physics}}, -doi = {https://doi.org/10.1007/978-3-319-18317-6}, -year = {2015} + author = {Wiedemann, H.}, + isbn = {978-3-319-18317-6}, + publisher = {Springer Cham}, + title = {{Particle Accelerator Physics}}, + doi = {10.1007/978-3-319-18317-6}, + year = {2015} } diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 1eaf22f0183..4ea9a3dc65d 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -157,16 +157,28 @@ Overall simulation parameters where :math:`\boldsymbol{\beta}` is the average (normalized) velocity of the considered species (which can be relativistic). See, e.g., :cite:t:`param-Vaypop2008` for more information. - See the `AMReX documentation `_ - for details of the MLMG solver (the default solver used with electrostatic - simulations). The default behavior of the code is to check whether there is - non-zero charge density in the system and if so force the MLMG solver to - use the solution max norm when checking convergence. If there is no charge - density, the MLMG solver will switch to using the initial guess max norm - error when evaluating convergence and an absolute error tolerance of - :math:`10^{-6}` :math:`\mathrm{V/m}^2` will be used (unless a different - non-zero value is specified by the user via - ``warpx.self_fields_absolute_tolerance``). +* ``warpx.poisson_solver`` (`string`) optional (default `multigrid`) + + * ``multigrid``: Poisson's equation is solved using an iterative multigrid (MLMG) solver. + See the `AMReX documentation `__ + for details of the MLMG solver (the default solver used with electrostatic + simulations). The default behavior of the code is to check whether there is + non-zero charge density in the system and if so force the MLMG solver to + use the solution max norm when checking convergence. If there is no charge + density, the MLMG solver will switch to using the initial guess max norm + error when evaluating convergence and an absolute error tolerance of + :math:`10^{-6}` :math:`\mathrm{V/m}^2` will be used (unless a different + non-zero value is specified by the user via + ``warpx.self_fields_absolute_tolerance``). + + * ``fft``: Poisson's equation is solved using an Integrated Green Function method (which requires FFT calculations). + See these references for more details :cite:t:`QiangPhysRevSTAB2006`, :cite:t:`QiangPhysRevSTAB2006err`. + It only works in 3D and it requires the compilation flag ``-DWarpX_PSATD=ON``. + If mesh refinement is enabled, this solver only works on the coarsest level. + On the refined patches, the Poisson equation is solved with the multigrid solver. + In electrostatic mode, this solver requires open field boundary conditions (``boundary.field_lo,hi = open``). + In electromagnetic mode, this solver can be used to initialize the species' self fields + (``.initialize_self_fields=1``) provided that the field BCs are PML (``boundary.field_lo,hi = PML``). * ``warpx.self_fields_required_precision`` (`float`, default: 1.e-11) The relative precision with which the electrostatic space-charge fields should @@ -398,7 +410,9 @@ Domain Boundary Conditions * ``none``: No boundary condition is applied to the fields with the electromagnetic solver. This option must be used for the RZ-solver at `r=0`. - * ``neumann``: For the electrostatic solver, a Neumann boundary condition (with gradient of the potential equal to 0) will be applied on the specified boundary. + * ``neumann``: For the electrostatic multigrid solver, a Neumann boundary condition (with gradient of the potential equal to 0) will be applied on the specified boundary. + + * ``open``: For the electrostatic Poisson solver based on a Integrated Green Function method. * ``boundary.potential_lo_x/y/z`` and ``boundary.potential_hi_x/y/z`` (default `0`) Gives the value of the electric potential at the boundaries, for ``pec`` boundaries. With electrostatic solvers diff --git a/Examples/Tests/openbc_poisson_solver/analysis.py b/Examples/Tests/openbc_poisson_solver/analysis.py new file mode 100755 index 00000000000..dc7b5dca8bd --- /dev/null +++ b/Examples/Tests/openbc_poisson_solver/analysis.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 + +import os +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import epsilon_0, pi +from scipy.special import erf + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +sigmaz = 300e-6 +sigmax = 516e-9 +sigmay = 7.7e-9 +Q = -3.2e-9 + +def w(z): + return np.exp(-z**2) * ( 1 + erf(1.j*z) ) + +def evaluate_E(x, y, z): + ''' Basseti-Erskine formula https://cds.cern.ch/record/122227/files/198005132.pdf ''' + den = np.sqrt(2*(sigmax**2-sigmay**2)) + arg1 = (x+1j*y)/den + term1 = w(arg1) + arg2 = (x*sigmay/sigmax + 1j*y*sigmax/sigmay)/den + term2 = -np.exp(-x**2/(2*sigmax**2)-y**2/(2*sigmay**2))*w(arg2) + factor = Q/(2.*np.sqrt(2.)*pi*epsilon_0*sigmaz*den)*np.exp(-z**2/(2*sigmaz**2)) + E_complex = factor*(term1 + term2) + return E_complex.imag, E_complex.real + + +fn = sys.argv[1] + +path=os.path.join('diags', 'diag2') +ts = OpenPMDTimeSeries(path) + +Ex, info = ts.get_field(field='E', coord='x', iteration=0, plot=False) +Ey, info = ts.get_field(field='E', coord='y', iteration=0, plot=False) + +grid_x = info.x[1:-1] +grid_y = info.y[1:-1] +grid_z = info.z[1:-1] + +hnx = int(0.5*len(grid_x)) +hny = int(0.5*len(grid_y)) + +# Compare theoretical and WarpX Ex, Ey fields for every z +for k, z in enumerate(grid_z,start=1): + Ex_warpx = Ex[k,hny,1:-1] + Ey_warpx = Ey[k,1:-1,hnx] + + Ex_theory = evaluate_E(grid_x, 0., z)[0] + Ey_theory = evaluate_E(0., grid_y, z)[1] + + assert(np.allclose(Ex_warpx, Ex_theory, rtol=0.032, atol=0)) + assert(np.allclose(Ey_warpx, Ey_theory, rtol=0.029, atol=0)) + + +# Get name of the test +test_name = os.path.split(os.getcwd())[1] + +# Run checksum regression test +checksumAPI.evaluate_checksum(test_name, fn, rtol=1e-2) diff --git a/Examples/Tests/openbc_poisson_solver/inputs_3d b/Examples/Tests/openbc_poisson_solver/inputs_3d new file mode 100644 index 00000000000..fd4299f360b --- /dev/null +++ b/Examples/Tests/openbc_poisson_solver/inputs_3d @@ -0,0 +1,46 @@ +my_constants.sigmaz = 300e-6 +my_constants.sigmax = 516e-9 +my_constants.sigmay = 7.7e-9 +my_constants.Q = 3.2e-9 + +max_step = 1 +amr.n_cell = 128 128 128 +amr.max_level = 0 +amr.max_grid_size = 128 +geometry.dims = 3 +geometry.prob_lo = -5*sigmax -5*sigmay -5*sigmaz +geometry.prob_hi = +5*sigmax +5*sigmay +5*sigmaz +boundary.field_lo = open open open +boundary.field_hi = open open open +warpx.const_dt = 1e-14 +warpx.do_electrostatic = relativistic +warpx.poisson_solver = fft +algo.field_gathering = momentum-conserving +algo.particle_shape = 1 + + +particles.species_names = electron + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 2 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "Q/(sqrt(2*pi)**3 * sigmax*sigmay*sigmaz * q_e) * exp( -x*x/(2*sigmax*sigmax) -y*y/(2*sigmay*sigmay) - z*z/(2*sigmaz*sigmaz) )" +electron.momentum_distribution_type = "constant" +electron.ux = 0.0 +electron.uy = 0.0 +electron.uz = 25000 +electron.initialize_self_fields = 1 + +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 1 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho +diag1.format = plotfile + +diag2.intervals = 1 +diag2.diag_type = Full +diag2.fields_to_plot = Ex Ey +diag2.format = openpmd diff --git a/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json new file mode 100644 index 00000000000..50b402b4bab --- /dev/null +++ b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 67876594.52390265, + "By": 112770930.05280717, + "Bz": 1.1060167389533003e-13, + "Ex": 3.380787637240387e+16, + "Ey": 2.034889235345363e+16, + "Ez": 2601470.9380206033, + "rho": 5630115646680.221 + }, + "electron": { + "particle_momentum_x": 6.378073505535043e-19, + "particle_momentum_y": 3.838951101276526e-19, + "particle_momentum_z": 1.1454327682972433e-10, + "particle_position_x": 21.64260864000591, + "particle_position_y": 0.3229614080035443, + "particle_position_z": 12582.912000000004, + "particle_weight": 19972794743.418434 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 897fcde3cf9..e9c53bb37fd 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4737,3 +4737,21 @@ compileTest = 0 doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py + +[openbc_poisson_solver] +buildDir = . +inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d +runtime_params = warpx.abort_on_warning_threshold = high +dim = 3 +addToCompileString = USE_OPENPMD=TRUE USE_PSATD=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +doComparison = 0 +analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 755c9d6e2a0..8f022c308af 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -146,6 +146,10 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) { WARPX_PROFILE("WarpX::AddSpaceChargeField"); + if (pc.getCharge() == 0) { + return; + } + // Store the boundary conditions for the field solver if they haven't been // stored yet if (!m_poisson_boundary_handler.bcs_set) { @@ -249,7 +253,7 @@ WarpX::AddSpaceChargeFieldLabFrame () // Use the tridiag solver with 1D computePhiTriDiagonal(rho_fp, phi_fp); #else - // Use the AMREX MLMG solver otherwise + // Use the AMREX MLMG or the FFT (IGF) solver otherwise computePhi(rho_fp, phi_fp, beta, self_fields_required_precision, self_fields_absolute_tolerance, self_fields_max_iters, self_fields_verbosity); @@ -352,6 +356,9 @@ WarpX::computePhi (const amrex::Vector >& rho, const std::optional > eb_farray_box_factory; #endif + bool const is_solver_multigrid = + WarpX::poisson_solver_id != PoissonSolverAlgo::IntegratedGreenFunction; + ablastr::fields::computePhi( sorted_rho, sorted_phi, @@ -364,6 +371,7 @@ WarpX::computePhi (const amrex::Vector >& rho, this->dmap, this->grids, this->m_poisson_boundary_handler, + is_solver_multigrid, WarpX::do_single_precision_comms, this->ref_ratio, post_phi_calculation, @@ -1001,6 +1009,7 @@ void ElectrostaticSolver::PoissonBoundaryHandler::definePhiBCs (const amrex::Geo amrex::ignore_unused(geom); #endif for (int idim=dim_start; idim electrostatic_solver_algo_to_int = { {"default", ElectrostaticSolverAlgo::None } }; +const std::map poisson_solver_algo_to_int = { + {"multigrid", PoissonSolverAlgo::Multigrid}, + {"fft", PoissonSolverAlgo::IntegratedGreenFunction}, + {"default", PoissonSolverAlgo::Multigrid } +}; + const std::map particle_pusher_algo_to_int = { {"boris", ParticlePusherAlgo::Boris }, {"vay", ParticlePusherAlgo::Vay }, @@ -125,6 +131,7 @@ const std::map FieldBCType_algo_to_int = { {"damped", FieldBoundaryType::Damped}, {"absorbing_silver_mueller", FieldBoundaryType::Absorbing_SilverMueller}, {"neumann", FieldBoundaryType::Neumann}, + {"open", FieldBoundaryType::Open}, {"none", FieldBoundaryType::None}, {"default", FieldBoundaryType::PML} }; @@ -163,6 +170,8 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ algo_to_int = grid_to_int; } else if (0 == std::strcmp(pp_search_key, "do_electrostatic")) { algo_to_int = electrostatic_solver_algo_to_int; + } else if (0 == std::strcmp(pp_search_key, "poisson_solver")) { + algo_to_int = poisson_solver_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "particle_pusher")) { algo_to_int = particle_pusher_algo_to_int; } else if (0 == std::strcmp(pp_search_key, "current_deposition")) { diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index e08a6304f09..d2691685c53 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -404,6 +404,7 @@ void ReadBCParams () const ParmParse pp_warpx("warpx"); const ParmParse pp_algo("algo"); const int electromagnetic_solver_id = GetAlgorithmInteger(pp_algo, "maxwell_solver"); + const int poisson_solver_id = GetAlgorithmInteger(pp_warpx, "poisson_solver"); if (pp_geometry.queryarr("is_periodic", geom_periodicity)) { @@ -469,6 +470,14 @@ void ReadBCParams () ), "PEC boundary not implemented for PSATD, yet!" ); + + if(WarpX::field_boundary_lo[idim] == FieldBoundaryType::Open && + WarpX::field_boundary_hi[idim] == FieldBoundaryType::Open){ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + poisson_solver_id == PoissonSolverAlgo::IntegratedGreenFunction, + "Field open boundary conditions are only implemented for the FFT-based Poisson solver" + ); + } } // Appending periodicity information to input so that it can be used by amrex diff --git a/Source/WarpX.H b/Source/WarpX.H index 3be95f47865..cdd2447ed5e 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -929,6 +929,7 @@ public: static const amrex::iMultiFab* GatherBufferMasks (int lev); static int electrostatic_solver_id; + static int poisson_solver_id; // Parameters for lab frame electrostatic static amrex::Real self_fields_required_precision; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 11646927863..f89a61e524b 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -190,6 +190,7 @@ amrex::IntVect WarpX::sort_idx_type(AMREX_D_DECL(0,0,0)); bool WarpX::do_dynamic_scheduling = true; int WarpX::electrostatic_solver_id; +int WarpX::poisson_solver_id; Real WarpX::self_fields_required_precision = 1.e-11_rt; Real WarpX::self_fields_absolute_tolerance = 0.0_rt; int WarpX::self_fields_max_iters = 200; @@ -763,6 +764,27 @@ WarpX::ReadParameters () pp_warpx, "self_fields_max_iters", self_fields_max_iters); pp_warpx.query("self_fields_verbosity", self_fields_verbosity); } + + poisson_solver_id = GetAlgorithmInteger(pp_warpx, "poisson_solver"); +#ifndef WARPX_DIM_3D + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, + "The FFT Poisson solver only works in 3D."); +#endif +#ifndef WARPX_USE_PSATD + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, + "To use the FFT Poisson solver, compile with WARPX_USE_PSATD=ON."); +#endif + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + ( + electrostatic_solver_id!=ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic || + poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction + ), + "The FFT Poisson solver is not implemented in labframe-electromagnetostatic mode yet." + ); + // Parse the input file for domain boundary potentials const ParmParse pp_boundary("boundary"); bool potential_specified = false; diff --git a/Source/ablastr/CMakeLists.txt b/Source/ablastr/CMakeLists.txt index a41fa4ae3a0..03b876fa1ab 100644 --- a/Source/ablastr/CMakeLists.txt +++ b/Source/ablastr/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(coarsen) add_subdirectory(math) -#add_subdirectory(fields) +add_subdirectory(fields) add_subdirectory(parallelization) #add_subdirectory(particles) #add_subdirectory(profiler) diff --git a/Source/ablastr/Make.package b/Source/ablastr/Make.package index 3426020e45d..b9ff3c72560 100644 --- a/Source/ablastr/Make.package +++ b/Source/ablastr/Make.package @@ -2,6 +2,7 @@ include $(WARPX_HOME)/Source/ablastr/coarsen/Make.package include $(WARPX_HOME)/Source/ablastr/math/Make.package +include $(WARPX_HOME)/Source/ablastr/fields/Make.package include $(WARPX_HOME)/Source/ablastr/parallelization/Make.package include $(WARPX_HOME)/Source/ablastr/particles/Make.package include $(WARPX_HOME)/Source/ablastr/utils/Make.package diff --git a/Source/ablastr/fields/CMakeLists.txt b/Source/ablastr/fields/CMakeLists.txt new file mode 100644 index 00000000000..f7dfb6763ec --- /dev/null +++ b/Source/ablastr/fields/CMakeLists.txt @@ -0,0 +1,9 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + if(WarpX_PSATD AND D EQUAL 3) + target_sources(ablastr_${SD} + PRIVATE + IntegratedGreenFunctionSolver.cpp + ) + endif() +endforeach() diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.H b/Source/ablastr/fields/IntegratedGreenFunctionSolver.H new file mode 100644 index 00000000000..97ffdb5ac36 --- /dev/null +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.H @@ -0,0 +1,66 @@ +/* Copyright 2019-2024 Arianna Formenti, Remi Lehe + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef ABLASTR_IGF_SOLVER_H +#define ABLASTR_IGF_SOLVER_H + +#include +#include +#include +#include +#include + + +#include +#include + + +namespace ablastr::fields +{ + + /** @brief Implements equation 2 in https://doi.org/10.1103/PhysRevSTAB.10.129901 + * with some modification to symmetrize the function. + * + * @param[in] x x-coordinate of given location + * @param[in] y y-coordinate of given location + * @param[in] z z-coordinate of given location + * + * @return the integrated Green function G + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + amrex::Real + IntegratedPotential (amrex::Real x, amrex::Real y, amrex::Real z) + { + using namespace amrex::literals; + + amrex::Real const r = std::sqrt( x*x + y*y + z*z ); + amrex::Real const G = + - 0.5_rt * z*z * std::atan( x*y/(z*r) ) + - 0.5_rt * y*y * std::atan( x*z/(y*r) ) + - 0.5_rt * x*x * std::atan( y*z/(x*r) ) + + y*z*std::asinh( x/std::sqrt(y*y + z*z) ) + + x*z*std::asinh( y/std::sqrt(x*x + z*z) ) + + x*y*std::asinh( z/std::sqrt(x*x + y*y) ); + return G; + } + + /** @brief Compute the electrostatic potential using the Integrated Green Function method + * as in http://dx.doi.org/10.1103/PhysRevSTAB.9.044204 + * + * @param[in] rho the charge density amrex::MultiFab + * @param[out] phi the electrostatic potential amrex::MultiFab + * @param[in] cell_size an arreay of 3 reals dx dy dz + * @param[in] ba amrex::BoxArray with the grid of a given level + */ + void + computePhiIGF (amrex::MultiFab const & rho, + amrex::MultiFab & phi, + std::array const & cell_size, + amrex::BoxArray const & ba); + +} // namespace ablastr::fields + +#endif // ABLASTR_IGF_SOLVER_H diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp new file mode 100644 index 00000000000..0767ecfb2f3 --- /dev/null +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp @@ -0,0 +1,196 @@ +/* Copyright 2019-2024 Arianna Formenti, Remi Lehe + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ +#include "IntegratedGreenFunctionSolver.H" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace ablastr::fields { + +void +computePhiIGF ( amrex::MultiFab const & rho, + amrex::MultiFab & phi, + std::array const & cell_size, + amrex::BoxArray const & ba ) +{ + using namespace amrex::literals; + + // Define box that encompasses the full domain + amrex::Box domain = ba.minimalBox(); + domain.surroundingNodes(); // get nodal points, since `phi` and `rho` are nodal + domain.grow( phi.nGrowVect() ); // include guard cells + + int const nx = domain.length(0); + int const ny = domain.length(1); + int const nz = domain.length(2); + + // Allocate 2x wider arrays for the convolution of rho with the Green function + // This also defines the box arrays for the global FFT: contains only one box; + amrex::Box const realspace_box = amrex::Box( + {domain.smallEnd(0), domain.smallEnd(1), domain.smallEnd(2)}, + {2*nx-1+domain.smallEnd(0), 2*ny-1+domain.smallEnd(1), 2*nz-1+domain.smallEnd(2)}, + amrex::IntVect::TheNodeVector() ); + amrex::BoxArray const realspace_ba = amrex::BoxArray( realspace_box ); + amrex::Box const spectralspace_box = amrex::Box( + {0,0,0}, + {nx, 2*ny-1, 2*nz-1}, + amrex::IntVect::TheNodeVector() ); + amrex::BoxArray const spectralspace_ba = amrex::BoxArray( spectralspace_box ); + // Define a distribution mapping for the global FFT, with only one box + amrex::DistributionMapping dm_global_fft; + dm_global_fft.define( realspace_ba ); + // Allocate required arrays + amrex::MultiFab tmp_rho = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); + tmp_rho.setVal(0); + amrex::MultiFab tmp_G = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); + tmp_G.setVal(0); + // Allocate corresponding arrays in Fourier space + using SpectralField = amrex::FabArray< amrex::BaseFab< amrex::GpuComplex< amrex::Real > > >; + SpectralField tmp_rho_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); + SpectralField tmp_G_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); + + // Copy from rho to tmp_rho + tmp_rho.ParallelCopy( rho, 0, 0, 1, amrex::IntVect::TheZeroVector(), amrex::IntVect::TheZeroVector() ); + + // Compute the integrated Green function + { + BL_PROFILE("Initialize Green function"); + amrex::BoxArray const domain_ba = amrex::BoxArray( domain ); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (amrex::MFIter mfi(domain_ba, dm_global_fft,amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + + amrex::Box const bx = mfi.tilebox(); + + amrex::IntVect const lo = realspace_box.smallEnd(); + amrex::IntVect const hi = realspace_box.bigEnd(); + + // Fill values of the Green function + amrex::Real const dx = cell_size[0]; + amrex::Real const dy = cell_size[1]; + amrex::Real const dz = cell_size[2]; + amrex::Array4 const tmp_G_arr = tmp_G.array(mfi); + amrex::ParallelFor( bx, + [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + int const i0 = i - lo[0]; + int const j0 = j - lo[1]; + int const k0 = k - lo[2]; + amrex::Real const x = i0*dx; + amrex::Real const y = j0*dy; + amrex::Real const z = k0*dz; + + amrex::Real const G_value = 1._rt/(4._rt*ablastr::constant::math::pi*ablastr::constant::SI::ep0) * ( + IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) + ); + + tmp_G_arr(i,j,k) = G_value; + // Fill the rest of the array by periodicity + if (i0>0) {tmp_G_arr(hi[0]+1-i0, j , k ) = G_value;} + if (j0>0) {tmp_G_arr(i , hi[1]+1-j0, k ) = G_value;} + if (k0>0) {tmp_G_arr(i , j , hi[2]+1-k0) = G_value;} + if ((i0>0)&&(j0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, k ) = G_value;} + if ((j0>0)&&(k0>0)) {tmp_G_arr(i , hi[1]+1-j0, hi[2]+1-k0) = G_value;} + if ((i0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, j , hi[2]+1-k0) = G_value;} + if ((i0>0)&&(j0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, hi[2]+1-k0) = G_value;} + } + ); + } + } + + // Perform forward FFTs + auto forward_plan_rho = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); + auto forward_plan_G = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); + // Loop over boxes perform FFTs + for ( amrex::MFIter mfi(realspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ + + // Note: the size of the real-space box and spectral-space box + // differ when using real-to-complex FFT. When initializing + // the FFT plan, the valid dimensions are those of the real-space box. + const amrex::IntVect fft_size = realspace_ba[mfi].length(); + + // FFT of rho + forward_plan_rho[mfi] = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_rho[mfi].dataPtr(), + reinterpret_cast(tmp_rho_fft[mfi].dataPtr()), + ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); + ablastr::math::anyfft::Execute(forward_plan_rho[mfi]); + + // FFT of G + forward_plan_G[mfi] = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_G[mfi].dataPtr(), + reinterpret_cast(tmp_G_fft[mfi].dataPtr()), + ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); + ablastr::math::anyfft::Execute(forward_plan_G[mfi]); + + } + + // Multiply tmp_G_fft and tmp_rho_fft in spectral space + // Store the result in-place in Gtmp_G_fft, to save memory + amrex::Multiply( tmp_G_fft, tmp_rho_fft, 0, 0, 1, 0); + + // Perform inverse FFT + auto backward_plan = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); + // Loop over boxes perform FFTs + for ( amrex::MFIter mfi(spectralspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ + + // Note: the size of the real-space box and spectral-space box + // differ when using real-to-complex FFT. When initializing + // the FFT plan, the valid dimensions are those of the real-space box. + const amrex::IntVect fft_size = realspace_ba[mfi].length(); + + // Inverse FFT: is done in-place, in the array of G + backward_plan[mfi] = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_G[mfi].dataPtr(), + reinterpret_cast( tmp_G_fft[mfi].dataPtr()), + ablastr::math::anyfft::direction::C2R, AMREX_SPACEDIM); + ablastr::math::anyfft::Execute(backward_plan[mfi]); + } + // Normalize, since (FFT + inverse FFT) results in a factor N + const amrex::Real normalization = 1._rt / realspace_box.numPts(); + tmp_G.mult( normalization ); + + // Copy from tmp_G to phi + phi.ParallelCopy( tmp_G, 0, 0, 1, amrex::IntVect::TheZeroVector(), phi.nGrowVect() ); + + // Loop to destroy FFT plans + for ( amrex::MFIter mfi(spectralspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ + ablastr::math::anyfft::DestroyPlan(forward_plan_G[mfi]); + ablastr::math::anyfft::DestroyPlan(forward_plan_rho[mfi]); + ablastr::math::anyfft::DestroyPlan(backward_plan[mfi]); + } +} +} // namespace ablastr::fields diff --git a/Source/ablastr/fields/Make.package b/Source/ablastr/fields/Make.package new file mode 100644 index 00000000000..4fb7bd88e5f --- /dev/null +++ b/Source/ablastr/fields/Make.package @@ -0,0 +1,7 @@ +ifeq ($(USE_PSATD),TRUE) + ifeq ($(DIM),3) + CEXE_sources += IntegratedGreenFunctionSolver.cpp + endif +endif + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/ablastr/fields diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 77c06f11cf2..3489991671c 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -11,8 +11,12 @@ #include #include #include +#include #include +#if defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D) +#include +#endif #include #include @@ -77,6 +81,7 @@ namespace ablastr::fields { * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler + * \param[in] is_solver_multigrid boolean to select the Poisson solver: 1 for Multigrid, 0 for FFT * \param[in] do_single_precision_comms perform communications in single precision * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) * \param[in] post_phi_calculation perform a calculation per level directly after phi was calculated; required for embedded boundaries (default: none) @@ -100,6 +105,7 @@ computePhi (amrex::Vector const & rho, amrex::Vector const& dmap, amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, + [[maybe_unused]] bool is_solver_multigrid, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, @@ -109,6 +115,8 @@ computePhi (amrex::Vector const & rho, { using namespace amrex::literals; + ABLASTR_PROFILE("computePhi", false); + if (!rel_ref_ratio.has_value()) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(rho.size() == 1u, "rel_ref_ratio must be set if mesh-refinement is used"); @@ -117,11 +125,9 @@ computePhi (amrex::Vector const & rho, auto const finest_level = static_cast(rho.size() - 1); - // scale rho appropriately; also determine if rho is zero everywhere + // determine if rho is zero everywhere amrex::Real max_norm_b = 0.0; for (int lev=0; lev<=finest_level; lev++) { - using namespace ablastr::constant::SI; - rho[lev]->mult(-1._rt/ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! max_norm_b = amrex::max(max_norm_b, rho[lev]->norm0()); } amrex::ParallelDescriptor::ReduceRealMax(max_norm_b); @@ -153,6 +159,27 @@ computePhi (amrex::Vector const & rho, {{ beta[0], beta[1], beta[2] }}; #endif +#if (defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D)) + // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected + if(!is_solver_multigrid && lev==0){ + amrex::Array const dx_igf + {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), + geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), + geom[lev].CellSize(2)/std::sqrt(1._rt-beta_solver[2]*beta_solver[2]))}; + if ( max_norm_b == 0 ) { + phi[lev]->setVal(0); + } else { + computePhiIGF( *rho[lev], *phi[lev], dx_igf, grids[lev] ); + } + continue; + } +#endif + + // Use the Multigrid (MLMG) solver if selected or on refined patches + // but first scale rho appropriately + using namespace ablastr::constant::SI; + rho[lev]->mult(-1._rt/PhysConst::ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! + #if !(defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ)) // Determine whether to use semi-coarsening amrex::Array dx_scaled diff --git a/Source/ablastr/math/fft/WrapCuFFT.cpp b/Source/ablastr/math/fft/WrapCuFFT.cpp index 8ecea2a2486..b6b1706f807 100644 --- a/Source/ablastr/math/fft/WrapCuFFT.cpp +++ b/Source/ablastr/math/fft/WrapCuFFT.cpp @@ -8,6 +8,7 @@ #include "AnyFFT.H" #include "ablastr/utils/TextMsg.H" +#include "ablastr/profiler/ProfilerWrapper.H" namespace ablastr::math::anyfft { @@ -30,6 +31,7 @@ namespace ablastr::math::anyfft Complex * const complex_array, const direction dir, const int dim) { FFTplan fft_plan; + ABLASTR_PROFILE("ablastr::math::anyfft::CreatePlan", false); // Initialize fft_plan.m_plan with the vendor fft plan. cufftResult result; @@ -69,10 +71,12 @@ namespace ablastr::math::anyfft void DestroyPlan(FFTplan& fft_plan) { + ABLASTR_PROFILE("ablastr::math::anyfft::DestroyPlan", false); cufftDestroy( fft_plan.m_plan ); } void Execute(FFTplan& fft_plan){ + ABLASTR_PROFILE("ablastr::math::anyfft::Execute", false); // make sure that this is done on the same GPU stream as the above copy cudaStream_t stream = amrex::Gpu::Device::cudaStream(); cufftSetStream ( fft_plan.m_plan, stream); From f013b1f8601bfd883ae142f5fdf8054eecbaefb1 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 3 Apr 2024 19:34:52 -0700 Subject: [PATCH 027/190] Fix: ABLASTR FFT Poisson Solver (#4821) Small compile issues seen in ImpactX. --- Source/ablastr/fields/PoissonSolver.H | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 3489991671c..da0078f8b5a 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -13,6 +13,7 @@ #include #include #include +#include #if defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D) #include @@ -178,7 +179,7 @@ computePhi (amrex::Vector const & rho, // Use the Multigrid (MLMG) solver if selected or on refined patches // but first scale rho appropriately using namespace ablastr::constant::SI; - rho[lev]->mult(-1._rt/PhysConst::ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! + rho[lev]->mult(-1._rt/ep0); // TODO: when do we "un-multiply" this? We need to document this side-effect! #if !(defined(AMREX_USE_EB) || defined(WARPX_DIM_RZ)) // Determine whether to use semi-coarsening From febd6da36c6900aff8cd3308199dde60f3f86b0d Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Fri, 5 Apr 2024 09:29:26 -0700 Subject: [PATCH 028/190] repetitive member var in derived class (#4827) --- Source/Diagnostics/BTDiagnostics.H | 3 --- 1 file changed, 3 deletions(-) diff --git a/Source/Diagnostics/BTDiagnostics.H b/Source/Diagnostics/BTDiagnostics.H index 679dd6b8fa4..d5dd67226b7 100644 --- a/Source/Diagnostics/BTDiagnostics.H +++ b/Source/Diagnostics/BTDiagnostics.H @@ -176,9 +176,6 @@ private: /** Vector of lab-frame time corresponding to each snapshot */ amrex::Vector m_t_lab; - /** Vector of user-defined physical region for diagnostics in lab-frame - * for each back-transformed snapshot */ - amrex::Vector m_snapshot_domain_lab; /** Vector of physical region corresponding to the buffer that spans a part * of the full back-transformed snapshot */ amrex::Vector m_buffer_domain_lab; From 87e07677d927a9182940add1d8450c9958c9417e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 5 Apr 2024 09:39:52 -0700 Subject: [PATCH 029/190] Release 24.04 (#4824) * AMReX: 24.04 * pyAMReX: 24.04 * WarpX: 24.04 --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 259c0d8d351..bfcd0d3e44b 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 47347f785f5c26f1f9e65bef8e10f2d383406ef7 && cd - + cd ../amrex && git checkout --detach 24.04 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 55fae3db78d..92f235d91ec 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.03) +project(WarpX VERSION 24.04) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 10752c21bda..7299b865746 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.03' +version = u'24.04' # The full version, including alpha/beta/rc tags. -release = u'24.03' +release = u'24.04' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index 08e616cc279..d1b8ddd27e5 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.03', + version = '24.04', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index f0ce221df0c..65f3de6c43a 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 47347f785f5c26f1f9e65bef8e10f2d383406ef7 +branch = 24.04 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index e9c53bb37fd..b4db494516d 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 47347f785f5c26f1f9e65bef8e10f2d383406ef7 +branch = 24.04 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index db230573853..aa7c4ead995 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.03 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.04 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "47347f785f5c26f1f9e65bef8e10f2d383406ef7" +set(WarpX_amrex_branch "24.04" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 8ea26792217..cc43b9e3a29 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.03 CONFIG REQUIRED) + find_package(pyAMReX 24.04 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "b41b83d1c0f37c66ad0543afa7edaca9e0c82565" +set(WarpX_pyamrex_branch "24.04" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index f74eb50af8c..400ccf1737c 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 47347f785f5c26f1f9e65bef8e10f2d383406ef7 && cd - +cd amrex && git checkout --detach 24.04 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index b0f919d707a..70b333fc6b9 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.03', + version = '24.04', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From e7af8bfd463237119d5ec1fd38d359374f423516 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Fri, 5 Apr 2024 16:41:09 -0700 Subject: [PATCH 030/190] Fix outdated element-wise comparison based assertions (#4829) * Fix outdated element-wise comparison based assertions * Fix `<` to `<=` Co-authored-by: Weiqun Zhang --------- Co-authored-by: Weiqun Zhang --- Source/Parallelization/WarpXComm.cpp | 12 ++++++------ Source/ablastr/coarsen/sample.cpp | 4 ++-- Source/ablastr/particles/DepositCharge.H | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 7d5b25b1560..9daf37af3f8 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -596,7 +596,7 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In for (int i = 0; i < 3; ++i) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= mf[i]->nGrowVect(), + ng.allLE(mf[i]->nGrowVect()), "Error: in FillBoundaryE, requested more guard cells than allocated"); const amrex::IntVect nghost = (safe_guard_cells) ? mf[i]->nGrowVect() : ng; @@ -653,7 +653,7 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In for (int i = 0; i < 3; ++i) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= mf[i]->nGrowVect(), + ng.allLE(mf[i]->nGrowVect()), "Error: in FillBoundaryB, requested more guard cells than allocated"); const amrex::IntVect nghost = (safe_guard_cells) ? mf[i]->nGrowVect() : ng; @@ -684,7 +684,7 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Efield_avg_fp[lev][0]->nGrowVect(), + ng.allLE(Efield_avg_fp[lev][0]->nGrowVect()), "Error: in FillBoundaryE_avg, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Efield_avg_fp[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Efield_avg_fp[lev][1], ng, WarpX::do_single_precision_comms, period); @@ -705,7 +705,7 @@ WarpX::FillBoundaryE_avg (int lev, PatchType patch_type, IntVect ng) } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Efield_avg_cp[lev][0]->nGrowVect(), + ng.allLE(Efield_avg_cp[lev][0]->nGrowVect()), "Error: in FillBoundaryE, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Efield_avg_cp[lev][0], ng, WarpX::do_single_precision_comms, cperiod); ablastr::utils::communication::FillBoundary(*Efield_avg_cp[lev][1], ng, WarpX::do_single_precision_comms, cperiod); @@ -737,7 +737,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, period); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Bfield_fp[lev][0]->nGrowVect(), + ng.allLE(Bfield_fp[lev][0]->nGrowVect()), "Error: in FillBoundaryB, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][0], ng, WarpX::do_single_precision_comms, period); ablastr::utils::communication::FillBoundary(*Bfield_avg_fp[lev][1], ng, WarpX::do_single_precision_comms, period); @@ -757,7 +757,7 @@ WarpX::FillBoundaryB_avg (int lev, PatchType patch_type, IntVect ng) ablastr::utils::communication::FillBoundary(mf, WarpX::do_single_precision_comms, cperiod); } else { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - ng <= Bfield_avg_cp[lev][0]->nGrowVect(), + ng.allLE(Bfield_avg_cp[lev][0]->nGrowVect()), "Error: in FillBoundaryB_avg, requested more guard cells than allocated"); ablastr::utils::communication::FillBoundary(*Bfield_avg_cp[lev][0], ng, WarpX::do_single_precision_comms, cperiod); ablastr::utils::communication::FillBoundary(*Bfield_avg_cp[lev][1], ng, WarpX::do_single_precision_comms, cperiod); diff --git a/Source/ablastr/coarsen/sample.cpp b/Source/ablastr/coarsen/sample.cpp index b5661037c7e..51dcfc7dd08 100644 --- a/Source/ablastr/coarsen/sample.cpp +++ b/Source/ablastr/coarsen/sample.cpp @@ -41,12 +41,12 @@ namespace ablastr::coarsen::sample const amrex::IntVect stag_src = mf_src.boxArray().ixType().toIntVect(); const amrex::IntVect stag_dst = mf_dst.boxArray().ixType().toIntVect(); - if ( crse_ratio > amrex::IntVect(1) ) { + if ( crse_ratio.allGT(amrex::IntVect(1)) ) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( ngrowvect == amrex::IntVect(0), "option of filling guard cells of destination MultiFab with coarsening not supported for this interpolation" ); } - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( mf_src.nGrowVect() >= stag_dst-stag_src+ngrowvect, + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( mf_src.nGrowVect().allGE(stag_dst-stag_src+ngrowvect), "source fine MultiFab does not have enough guard cells for this interpolation" ); // Auxiliary integer arrays (always 3D) diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index 4d78e309f85..b2dedccd03e 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -71,7 +71,7 @@ deposit_charge (typename T_PC::ParIterType& pti, if (num_rho_deposition_guards.has_value()) { ng_rho = num_rho_deposition_guards.value(); } - ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(ng_rho <= rho->nGrowVect(), + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(ng_rho.allLE(rho->nGrowVect()), "num_rho_deposition_guards are larger than allocated!"); // particle shape auto const[nox, noy, noz] = std::array{particle_shape, particle_shape, particle_shape}; From 4a37ffbe1a6e2737b2794477946255730bc48566 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 8 Apr 2024 12:11:16 -0700 Subject: [PATCH 031/190] Fix out of bounds memory access in `HybridPICModel::GetCurrentExternal` (#4830) --- .../HybridPICModel/HybridPICModel.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 228201886eb..6a688ce38b3 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -97,12 +97,13 @@ void HybridPICModel::AllocateLevelMFs (int lev, const BoxArray& ba, const Distri // the external current density multifab is made nodal to avoid needing to interpolate // to a nodal grid as has to be done for the ion and total current density multifabs + // this also allows the external current multifab to not have any ghost cells WarpX::AllocInitMultiFab(current_fp_external[lev][0], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, ngJ, lev, "current_fp_external[x]", 0.0_rt); + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[x]", 0.0_rt); WarpX::AllocInitMultiFab(current_fp_external[lev][1], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, ngJ, lev, "current_fp_external[y]", 0.0_rt); + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[y]", 0.0_rt); WarpX::AllocInitMultiFab(current_fp_external[lev][2], amrex::convert(ba, IntVect(AMREX_D_DECL(1,1,1))), - dm, ncomps, ngJ, lev, "current_fp_external[z]", 0.0_rt); + dm, ncomps, IntVect(AMREX_D_DECL(0,0,0)), lev, "current_fp_external[z]", 0.0_rt); #ifdef WARPX_DIM_RZ WARPX_ALWAYS_ASSERT_WITH_MESSAGE( From ef6d0853b0115dbbaf95b74d87c3367a4104b5cd Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 8 Apr 2024 15:03:00 -0700 Subject: [PATCH 032/190] AMReX/pyAMReX/PICSAR: Weekly Update (#4834) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * AMReX: Weekly Update * pyAMReX: Weekly Update * Silence Parser Warning ``` Source/Utils/Parser/ParserUtils.cpp:177:31: error: ‘v’ may be used uninitialized in this function [-Werror=maybe-uninitialized] 177 | parser.setConstant(*it, v); | ~~~~~~~~~~~~~~~~~~^~~~~~~~ ``` --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- Source/Utils/Parser/ParserUtils.cpp | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index bfcd0d3e44b..f47dd01e057 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.04 && cd - + cd ../amrex && git checkout --detach 2a3955a5f5aac1aef6e6e72687f182331e049c39 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 65f3de6c43a..d1c98b315a0 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.04 +branch = 2a3955a5f5aac1aef6e6e72687f182331e049c39 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index b4db494516d..2f7fff84be7 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.04 +branch = 2a3955a5f5aac1aef6e6e72687f182331e049c39 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/Source/Utils/Parser/ParserUtils.cpp b/Source/Utils/Parser/ParserUtils.cpp index c2da9577947..0339b766e38 100644 --- a/Source/Utils/Parser/ParserUtils.cpp +++ b/Source/Utils/Parser/ParserUtils.cpp @@ -164,7 +164,7 @@ amrex::Parser utils::parser::makeParser ( for (auto it = symbols.begin(); it != symbols.end(); ) { // Always parsing in double precision avoids potential overflows that may occur when parsing // user's expressions because of the limited range of exponentials in single precision - double v; + double v = 0.0; WARPX_ALWAYS_ASSERT_WITH_MESSAGE( recursive_symbols.count(*it)==0, diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index aa7c4ead995..3f2987ec9cd 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.04" +set(WarpX_amrex_branch "2a3955a5f5aac1aef6e6e72687f182331e049c39" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index cc43b9e3a29..d4c488f9113 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.04" +set(WarpX_pyamrex_branch "9b0a4d82bc14756530efe4e0c9ce3463ca5df742" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 400ccf1737c..1d95dd5f2d2 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.04 && cd - +cd amrex && git checkout --detach 2a3955a5f5aac1aef6e6e72687f182331e049c39 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 76d65006798d7a70539d1f3b959d2a5038dd63a2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 8 Apr 2024 15:19:43 -0700 Subject: [PATCH 033/190] [pre-commit.ci] pre-commit autoupdate (#4837) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/pre-commit/pre-commit-hooks: v4.5.0 → v4.6.0](https://github.com/pre-commit/pre-commit-hooks/compare/v4.5.0...v4.6.0) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 812a046b602..cef24aed2cc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ exclude: '^share/openPMD/thirdParty' # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] From 9f5be94cd0b99063efcea7e568d438a53dc2df5b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 8 Apr 2024 15:34:35 -0700 Subject: [PATCH 034/190] HPC3 (UCI): Fix ADIOS2 HDF5 Build (#4836) Disable building examples and tests for ADIOS2 for speed. Do not build HDF5 bindings of ADIOS2 due an incompatibility in this version. --- Tools/machines/hpc3-uci/install_gpu_dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh index a2b9c7b3855..51d4bcfe2c5 100755 --- a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh +++ b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh @@ -70,7 +70,7 @@ else git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-pm-gpu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 +cmake -S $HOME/src/adios2 -B $HOME/src/adios2-pm-gpu-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_HDF5=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 cmake --build $HOME/src/adios2-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/adios2-pm-gpu-build From b4ed7064f84a87e7ae9e133f21b3bd29fb7158b1 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 8 Apr 2024 15:35:29 -0700 Subject: [PATCH 035/190] Output the potential phi on the macroparticles (#4599) * Output the potential phi on the macroparticles * Fix compilation * Cleaner code by using `getParticlePosition` * Add include statement * Update const-ness * Update code so that `phi` can be calculated * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Better boundary error message * Define function to define tiles, for a NamedComponentParticleContainer * Call parent class in WarpXParticleContainer * Call `defineAllParticleTiles` in diagnostics * Fix compilation issue * Fix compilation for fixed precision * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add automated test * Fix analysis script * Update test so that they work correctly in RZ * Update automated test * Update implementation of `phi` on the particles * Update test so as to trigger output of `phi` * Update documentation * Update Source/Diagnostics/ParticleDiag/ParticleDiag.H * Apply suggestions from code review Co-authored-by: Axel Huebl --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- Docs/source/usage/parameters.rst | 3 +- .../analysis_electrostatic_sphere.py | 18 ++- Examples/Tests/electrostatic_sphere/inputs_3d | 8 +- Examples/Tests/electrostatic_sphere/inputs_rz | 9 +- Regression/WarpX-tests.ini | 2 +- .../Diagnostics/ParticleDiag/ParticleDiag.H | 1 + .../Diagnostics/ParticleDiag/ParticleDiag.cpp | 19 ++- Source/Diagnostics/WarpXOpenPMD.cpp | 117 +++++++++++------- Source/Particles/ParticleIO.H | 46 +++++++ 9 files changed, 167 insertions(+), 56 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 4ea9a3dc65d..0a089358d21 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2676,7 +2676,8 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``..variables`` (list of `strings` separated by spaces, optional) List of particle quantities to write to output. Choices are ``w`` for the particle weight and ``ux`` ``uy`` ``uz`` for the particle momenta. - By default, all particle quantities are written. + When using the lab-frame electrostatic solver, ``phi`` (electrostatic potential, on the macroparticles) is also available. + By default, all particle quantities (except ``phi``) are written. If ``..variables = none``, no particle data are written, except for particle positions, which are always included. * ``..random_fraction`` (`float`) optional diff --git a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py index 35281084c01..8e05bfa0c68 100755 --- a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py +++ b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py @@ -22,6 +22,8 @@ import numpy as np import yt +from openpmd_viewer import OpenPMDTimeSeries +from scipy.constants import c from scipy.optimize import fsolve sys.path.insert(1, '../../../../warpx/Regression/Checksum/') @@ -125,7 +127,6 @@ def calculate_error(E_axis, xmin, dx, nx): return L2_error - L2_error_x = calculate_error(Ex_axis, xmin, dx, nx) L2_error_y = calculate_error(Ey_axis, ymin, dy, ny) L2_error_z = calculate_error(Ez_axis, zmin, dz, nz) @@ -137,6 +138,21 @@ def calculate_error(E_axis, xmin, dx, nx): assert L2_error_y < 0.05 assert L2_error_z < 0.05 +# Check conservation of energy +def return_energies(iteration): + ux, uy, uz, phi, m, q, w = ts.get_particle(['ux', 'uy', 'uz', 'phi', 'mass', 'charge', 'w'], iteration=iteration) + E_kinetic = (w*m*c**2 * (np.sqrt(1 + ux**2 + uy**2 + uz**2) - 1)).sum() + E_potential = 0.5*(w*q*phi).sum() # potential energy of particles in their own space-charge field: includes factor 1/2 + return E_kinetic, E_potential +ts = OpenPMDTimeSeries('./diags/diag2') +if 'phi' in ts.avail_record_components['electron']: + # phi is only available when this script is run with the labframe poisson solver + print('Checking conservation of energy') + Ek_i, Ep_i = return_energies(0) + Ek_f, Ep_f = return_energies(30) + assert Ep_f < 0.7*Ep_i # Check that potential energy changes significantly + assert abs( (Ek_i + Ep_i) - (Ek_f + Ep_f) ) < 0.003 * (Ek_i + Ep_i) # Check conservation of energy + # Checksum regression analysis test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/electrostatic_sphere/inputs_3d b/Examples/Tests/electrostatic_sphere/inputs_3d index afc6649f012..1f15b2da0de 100644 --- a/Examples/Tests/electrostatic_sphere/inputs_3d +++ b/Examples/Tests/electrostatic_sphere/inputs_3d @@ -29,7 +29,13 @@ electron.profile = parse_density_function electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0" electron.momentum_distribution_type = at_rest -diagnostics.diags_names = diag1 +diagnostics.diags_names = diag1 diag2 + diag1.intervals = 30 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez rho + +diag2.intervals = 30 +diag2.diag_type = Full +diag2.fields_to_plot = none +diag2.format = openpmd diff --git a/Examples/Tests/electrostatic_sphere/inputs_rz b/Examples/Tests/electrostatic_sphere/inputs_rz index 58f31046d89..26d8b0c7afc 100644 --- a/Examples/Tests/electrostatic_sphere/inputs_rz +++ b/Examples/Tests/electrostatic_sphere/inputs_rz @@ -30,7 +30,14 @@ electron.profile = parse_density_function electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0" electron.momentum_distribution_type = at_rest -diagnostics.diags_names = diag1 +diagnostics.diags_names = diag1 diag2 + diag1.intervals = 30 diag1.diag_type = Full diag1.fields_to_plot = Er Et Ez rho + +diag2.intervals = 30 +diag2.diag_type = Full +diag2.fields_to_plot = none +diag2.electron.variables = ux uy uz w phi +diag2.format = openpmd diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 2f7fff84be7..12fed1e8bb9 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -554,7 +554,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereLabFrame] buildDir = . inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.do_electrostatic=labframe +runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=ux uy uz w phi dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.H b/Source/Diagnostics/ParticleDiag/ParticleDiag.H index e9f54d2e5e1..45bc596a78e 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.H +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.H @@ -23,6 +23,7 @@ public: [[nodiscard]] PinnedMemoryParticleContainer* getPinnedParticleContainer() const { return m_pinned_pc; } [[nodiscard]] std::string getSpeciesName() const { return m_name; } amrex::Vector m_plot_flags; + bool m_plot_phi = false; // Whether to output the potential phi on the particles bool m_do_random_filter = false; bool m_do_uniform_filter = false; diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp index 9954fa24285..7ed64ccf8ec 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp @@ -35,12 +35,19 @@ ParticleDiag::ParticleDiag( if (variables[0] != "none"){ const std::map existing_variable_names = pc->getParticleComps(); for (const auto& var : variables){ - const auto search = existing_variable_names.find(var); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - search != existing_variable_names.end(), - "variables argument '" + var - +"' is not an existing attribute for this species"); - m_plot_flags[existing_variable_names.at(var)] = 1; + if (var == "phi") { + // User requests phi on particle. This is *not* part of the variables that + // the particle container carries, and is only added to particles during output. + // Therefore, this case needs to be treated specifically. + m_plot_phi = true; + } else { + const auto search = existing_variable_names.find(var); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + search != existing_variable_names.end(), + "variables argument '" + var + +"' is not an existing attribute for this species"); + m_plot_flags[existing_variable_names.at(var)] = 1; + } } } } diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index d90f3b45c96..6882b962445 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -548,48 +548,6 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { pinned_pc->make_alike() : pc->make_alike(); - // names of amrex::Real and int particle attributes in SoA data - amrex::Vector real_names; - amrex::Vector int_names; - amrex::Vector int_flags; - amrex::Vector real_flags; - - // see openPMD ED-PIC extension for namings - // note: an underscore separates the record name from its component - // for non-scalar records - // note: in RZ, we reconstruct x,y,z positions from r,z,theta in WarpX -#if !defined (WARPX_DIM_1D_Z) - real_names.push_back("position_x"); -#endif -#if defined (WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - real_names.push_back("position_y"); -#endif - real_names.push_back("position_z"); - real_names.push_back("weighting"); - real_names.push_back("momentum_x"); - real_names.push_back("momentum_y"); - real_names.push_back("momentum_z"); - - // get the names of the real comps - real_names.resize(tmp.NumRealComps()); - auto runtime_rnames = tmp.getParticleRuntimeComps(); - for (auto const& x : runtime_rnames) - { - real_names[x.second+PIdx::nattribs] = detail::snakeToCamel(x.first); - } - // plot any "extra" fields by default - real_flags = particle_diags[i].m_plot_flags; - real_flags.resize(tmp.NumRealComps(), 1); - // and the names - int_names.resize(tmp.NumIntComps()); - auto runtime_inames = tmp.getParticleRuntimeiComps(); - for (auto const& x : runtime_inames) - { - int_names[x.second+0] = detail::snakeToCamel(x.first); - } - // plot by default - int_flags.resize(tmp.NumIntComps(), 1); - const auto mass = pc->AmIA() ? PhysConst::m_e : pc->getMass(); RandomFilter const random_filter(particle_diags[i].m_do_random_filter, particle_diags[i].m_random_fraction); @@ -631,6 +589,76 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { particlesConvertUnits(ConvertDirection::SI_to_WarpX, pc, mass); } + // Gather the electrostatic potential (phi) on the macroparticles + if ( particle_diags[i].m_plot_phi ) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame) || + (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic), + "Output of the electrostatic potential (phi) on the particles was requested, " + "but this is only available for `warpx.do_electrostatic=labframe` or `labframe-electromagnetostatic`."); + // Using pinned PC indicates that the particles are not written at the same physical time (i.e. PIC iteration) + // that they were collected. This happens for diagnostics that use buffering (e.g. BackTransformed, BoundaryScraping). + // Here `phi` is gathered at the iteration when particles are written (not collected) and is thus mismatched. + // To avoid confusion, we raise an error in this case. + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + use_pinned_pc == false, + "Output of the electrostatic potential (phi) on the particles was requested, " + "but this is only available with `diag_type = Full`."); + tmp.AddRealComp("phi"); + int const phi_index = tmp.getParticleComps().at("phi"); + auto& warpx = WarpX::GetInstance(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (int lev=0; lev<=warpx.finestLevel(); lev++) { + const amrex::Geometry& geom = warpx.Geom(lev); + auto plo = geom.ProbLoArray(); + auto dxi = geom.InvCellSizeArray(); + amrex::MultiFab const& phi = warpx.getField( FieldType::phi_fp, lev, 0 ); + storeFieldOnParticles( tmp, plo, dxi, phi, phi_index, lev ); + } + } + + // names of amrex::Real and int particle attributes in SoA data + amrex::Vector real_names; + amrex::Vector int_names; + amrex::Vector int_flags; + amrex::Vector real_flags; + // see openPMD ED-PIC extension for namings + // note: an underscore separates the record name from its component + // for non-scalar records + // note: in RZ, we reconstruct x,y,z positions from r,z,theta in WarpX +#if !defined (WARPX_DIM_1D_Z) + real_names.push_back("position_x"); +#endif +#if defined (WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + real_names.push_back("position_y"); +#endif + real_names.push_back("position_z"); + real_names.push_back("weighting"); + real_names.push_back("momentum_x"); + real_names.push_back("momentum_y"); + real_names.push_back("momentum_z"); + // get the names of the real comps + real_names.resize(tmp.NumRealComps()); + auto runtime_rnames = tmp.getParticleRuntimeComps(); + for (auto const& x : runtime_rnames) + { + real_names[x.second+PIdx::nattribs] = detail::snakeToCamel(x.first); + } + // plot any "extra" fields by default + real_flags = particle_diags[i].m_plot_flags; + real_flags.resize(tmp.NumRealComps(), 1); + // and the names + int_names.resize(tmp.NumIntComps()); + auto runtime_inames = tmp.getParticleRuntimeiComps(); + for (auto const& x : runtime_inames) + { + int_names[x.second+0] = detail::snakeToCamel(x.first); + } + // plot by default + int_flags.resize(tmp.NumIntComps(), 1); + // real_names contains a list of all real particle attributes. // real_flags is 1 or 0, whether quantity is dumped or not. DumpToFile(&tmp, @@ -640,9 +668,8 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { int_flags, real_names, int_names, pc->getCharge(), pc->getMass(), - isBTD, isLastBTDFlush - ); -} + isBTD, isLastBTDFlush); + } } void diff --git a/Source/Particles/ParticleIO.H b/Source/Particles/ParticleIO.H index c417f7467b3..1abda6ea9af 100644 --- a/Source/Particles/ParticleIO.H +++ b/Source/Particles/ParticleIO.H @@ -8,6 +8,9 @@ #define PARTICLEIO_H_ #include "Particles/WarpXParticleContainer.H" +#include "Particles/Pusher/GetAndSetPosition.H" + +#include #include #include @@ -77,4 +80,47 @@ particlesConvertUnits (ConvertDirection convert_direction, T_ParticleContainer* } } +/** Gathers the field from a MultiFab to the macroparticles + * and stores it into a runtime component of the particle container + * + * @tparam T_ParticleContainer a WarpX particle container or AmrParticleContainer + * @param tmp the particle container on which to store the gathered field + * @param plo the coordinates of the low bound of the box, along each dimension + * @param dxi the inverse of the cell size, along each dimension + * @param phi a multifab that contains the field to be gathered + * @param phi_index the index of the runtime component where to store the gathered field + * @param lev the MR level + */ +template< typename T_ParticleContainer > +void +storeFieldOnParticles ( T_ParticleContainer& tmp, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + amrex::MultiFab const& phi, + int const phi_index, int const lev ) { + + using PinnedParIter = typename T_ParticleContainer::ParIterType; + + for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { + + auto phi_grid = phi[pti].array(); + const auto getPosition = GetParticlePosition(pti); + amrex::ParticleReal* phi_particle_arr = pti.GetStructOfArrays().GetRealData(phi_index).dataPtr(); + + // Loop over the particles and update their position + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long ip) { + + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); + amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi_grid); + phi_particle_arr[ip] = phi_value; + } + ); + } +} + #endif /* PARTICLEIO_H_ */ From 56a00e947722cc392ea9e4f4bf81746789562ac2 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 9 Apr 2024 03:39:45 +0200 Subject: [PATCH 036/190] Remove calls to WarpX::GetInstance() in PEC class and cleaning (#4771) * remove calls to WarpX instance in PEC class * continue work * fixed bugs * complete removal of warpx instance in WarpX_PEC * update doc * default for field boundary is PML * add missing :: * fixed bug * fix bugs * make notation more compact * use isAnyBoundary function also for SilverMuller --- Source/BoundaryConditions/PML.H | 4 +- Source/BoundaryConditions/PML_RZ.H | 4 +- Source/BoundaryConditions/PML_fwd.H | 2 - .../WarpXFieldBoundaries.cpp | 132 +++- Source/BoundaryConditions/WarpX_PEC.H | 582 +++------------- Source/BoundaryConditions/WarpX_PEC.cpp | 631 +++++++++++++++--- .../ApplySilverMuellerBoundary.cpp | 4 +- .../FiniteDifferenceSolver.H | 5 +- .../HybridPICModel/HybridPICModel.H | 4 +- Source/Utils/WarpXAlgorithmSelection.H | 18 +- Source/Utils/WarpXAlgorithmSelection.cpp | 16 +- Source/WarpX.H | 10 +- Source/WarpX.cpp | 4 +- 13 files changed, 741 insertions(+), 675 deletions(-) diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 5f41ecbd706..835877f2de9 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -11,6 +11,8 @@ #include "PML_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" + #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralSolver.H" #endif @@ -131,8 +133,6 @@ private: amrex::Real dt_E = -1.e10; }; -enum struct PatchType : int; - class PML { public: diff --git a/Source/BoundaryConditions/PML_RZ.H b/Source/BoundaryConditions/PML_RZ.H index 3a5a51770f8..74d3da2e19b 100644 --- a/Source/BoundaryConditions/PML_RZ.H +++ b/Source/BoundaryConditions/PML_RZ.H @@ -10,6 +10,8 @@ #include "PML_RZ_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" + #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #endif @@ -25,8 +27,6 @@ #include #include -enum struct PatchType : int; - class PML_RZ { public: diff --git a/Source/BoundaryConditions/PML_fwd.H b/Source/BoundaryConditions/PML_fwd.H index 58532a89070..21ad81949f1 100644 --- a/Source/BoundaryConditions/PML_fwd.H +++ b/Source/BoundaryConditions/PML_fwd.H @@ -14,8 +14,6 @@ struct SigmaBox; class SigmaBoxFactory; class MultiSigmaBox; -enum struct PatchType; - class PML; #endif /* WARPX_PML_FWD_H */ diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index ebf3d1e39eb..0e50c560e7c 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -18,26 +18,72 @@ using namespace amrex::literals; using namespace amrex; +namespace +{ + /** Returns true if any field boundary is set to FieldBoundaryType FT, else returns false.*/ + template + [[nodiscard]] + bool isAnyBoundary (const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi) + { + const auto isFT = [](const auto& b){ + return b == FT;}; + return std::any_of(field_boundary_lo.begin(), field_boundary_lo.end(), isFT) || + std::any_of(field_boundary_hi.begin(), field_boundary_hi.end(), isFT); + } + + /** Returns true if any particle boundary is set to ParticleBoundaryType PT, else returns false.*/ + template + [[nodiscard]] + bool isAnyBoundary (const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi) + { + const auto isPT = [](const auto& b){ + return b == PT;}; + return std::any_of(particle_boundary_lo.begin(), particle_boundary_lo.end(), isPT) || + std::any_of(particle_boundary_hi.begin(), particle_boundary_hi.end(), isPT); + } + +} + void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) { + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoEfield( { getFieldPointer(FieldType::Efield_fp, lev, 0), - getFieldPointer(FieldType::Efield_fp, lev, 1), - getFieldPointer(FieldType::Efield_fp, lev, 2) }, lev, patch_type); - if (WarpX::isAnyBoundaryPML()) { + PEC::ApplyPECtoEfield( + {getFieldPointer(FieldType::Efield_fp, lev, 0), + getFieldPointer(FieldType::Efield_fp, lev, 1), + getFieldPointer(FieldType::Efield_fp, lev, 2)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { // apply pec on split E-fields in PML region const bool split_pml_field = true; - PEC::ApplyPECtoEfield( pml[lev]->GetE_fp(), lev, patch_type, split_pml_field); + PEC::ApplyPECtoEfield( + pml[lev]->GetE_fp(), + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, + split_pml_field); } } else { - PEC::ApplyPECtoEfield( { getFieldPointer(FieldType::Efield_cp, lev, 0), - getFieldPointer(FieldType::Efield_cp, lev, 1), - getFieldPointer(FieldType::Efield_cp, lev, 2) }, lev, patch_type); - if (WarpX::isAnyBoundaryPML()) { + PEC::ApplyPECtoEfield( + {getFieldPointer(FieldType::Efield_cp, lev, 0), + getFieldPointer(FieldType::Efield_cp, lev, 1), + getFieldPointer(FieldType::Efield_cp, lev, 2)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { // apply pec on split E-fields in PML region const bool split_pml_field = true; - PEC::ApplyPECtoEfield( pml[lev]->GetE_cp(), lev, patch_type, split_pml_field); + PEC::ApplyPECtoEfield( + pml[lev]->GetE_cp(), + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio, + split_pml_field); } } } @@ -57,15 +103,23 @@ void WarpX::ApplyEfieldBoundary(const int lev, PatchType patch_type) void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_dt_type) { - if (PEC::isAnyBoundaryPEC()) { + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { - PEC::ApplyPECtoBfield( { getFieldPointer(FieldType::Bfield_fp, lev, 0), - getFieldPointer(FieldType::Bfield_fp, lev, 1), - getFieldPointer(FieldType::Bfield_fp, lev, 2) }, lev, patch_type); + PEC::ApplyPECtoBfield( { + getFieldPointer(FieldType::Bfield_fp, lev, 0), + getFieldPointer(FieldType::Bfield_fp, lev, 1), + getFieldPointer(FieldType::Bfield_fp, lev, 2) }, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); } else { - PEC::ApplyPECtoBfield( { getFieldPointer(FieldType::Bfield_cp, lev, 0), - getFieldPointer(FieldType::Bfield_cp, lev, 1), - getFieldPointer(FieldType::Bfield_cp, lev, 2) }, lev, patch_type); + PEC::ApplyPECtoBfield( { + getFieldPointer(FieldType::Bfield_cp, lev, 0), + getFieldPointer(FieldType::Bfield_cp, lev, 1), + getFieldPointer(FieldType::Bfield_cp, lev, 2)}, + field_boundary_lo, field_boundary_hi, + get_ng_fieldgather(), Geom(lev), + lev, patch_type, ref_ratio); } } @@ -74,18 +128,11 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d // E and B are staggered in time, which is only true after the first half-push if (lev == 0) { if (a_dt_type == DtType::FirstHalf) { - bool applySilverMueller = false; - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( (WarpX::field_boundary_lo[idim] == FieldBoundaryType::Absorbing_SilverMueller) || - (WarpX::field_boundary_hi[idim] == FieldBoundaryType::Absorbing_SilverMueller) ) { - applySilverMueller = true; - } - } - if(applySilverMueller) { m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( - Efield_fp[lev], Bfield_fp[lev], - Geom(lev).Domain(), dt[lev], - WarpX::field_boundary_lo, - WarpX::field_boundary_hi); + if(::isAnyBoundary(field_boundary_lo, field_boundary_hi)){ + m_fdtd_solver_fp[0]->ApplySilverMuellerBoundary( + Efield_fp[lev], Bfield_fp[lev], + Geom(lev).Domain(), dt[lev], + field_boundary_lo, field_boundary_hi); } } } @@ -106,9 +153,14 @@ void WarpX::ApplyBfieldBoundary (const int lev, PatchType patch_type, DtType a_d void WarpX::ApplyRhofieldBoundary (const int lev, MultiFab* rho, PatchType patch_type) { - if (PEC::isAnyParticleBoundaryReflecting() || WarpX::isAnyParticleBoundaryThermal() || PEC::isAnyBoundaryPEC()) + if (::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { - PEC::ApplyReflectiveBoundarytoRhofield(rho, lev, patch_type); + PEC::ApplyReflectiveBoundarytoRhofield(rho, + field_boundary_lo, field_boundary_hi, + particle_boundary_lo, particle_boundary_hi, + Geom(lev), lev, patch_type, ref_ratio); } } @@ -116,9 +168,14 @@ void WarpX::ApplyJfieldBoundary (const int lev, amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, PatchType patch_type) { - if (PEC::isAnyParticleBoundaryReflecting() || WarpX::isAnyParticleBoundaryThermal() || PEC::isAnyBoundaryPEC()) + if (::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(particle_boundary_lo, particle_boundary_hi) || + ::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { - PEC::ApplyReflectiveBoundarytoJfield(Jx, Jy, Jz, lev, patch_type); + PEC::ApplyReflectiveBoundarytoJfield(Jx, Jy, Jz, + field_boundary_lo, field_boundary_hi, + particle_boundary_lo, particle_boundary_hi, + Geom(lev), lev, patch_type, ref_ratio); } } @@ -207,11 +264,12 @@ WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex void WarpX::ApplyElectronPressureBoundary (const int lev, PatchType patch_type) { - if (PEC::isAnyBoundaryPEC()) { + if (::isAnyBoundary(field_boundary_lo, field_boundary_hi)) { if (patch_type == PatchType::fine) { PEC::ApplyPECtoElectronPressure( - m_hybrid_pic_model->get_pointer_electron_pressure_fp(lev), lev, patch_type - ); + m_hybrid_pic_model->get_pointer_electron_pressure_fp(lev), + field_boundary_lo, field_boundary_hi, + Geom(lev), lev, patch_type, ref_ratio); } else { amrex::Abort(Utils::TextMsg::Err( "ApplyElectronPressureBoundary: Only one level implemented for hybrid solver.")); diff --git a/Source/BoundaryConditions/WarpX_PEC.H b/Source/BoundaryConditions/WarpX_PEC.H index b0d15a70321..a6af894beb4 100644 --- a/Source/BoundaryConditions/WarpX_PEC.H +++ b/Source/BoundaryConditions/WarpX_PEC.H @@ -1,16 +1,11 @@ #ifndef WARPX_PEC_KERNELS_H_ #define WARPX_PEC_KERNELS_H_ -#include "WarpX.H" #include "Utils/WarpXAlgorithmSelection.H" #include -#include -#include -#include -#include -#include -#include +#include +#include #include @@ -18,528 +13,113 @@ #include namespace PEC { -using namespace amrex; - /** - * \brief Determines if the field boundary condition stored in fboundary - * in direction, dir, is PEC. - * - * \param[in] fboundary Value containing boundary type - * \param[in] dir direction - * - * \returns 1 if the boundary type is PEC else 0 - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - bool is_boundary_PEC (amrex::GpuArray const& fboundary, int dir) { - return ( fboundary[dir] == FieldBoundaryType::PEC ); - } - - /** - * \brief Determines if the particle boundary condition stored in pboundary - * in direction, dir, is reflecting. - * - * \param[in] pboundary Value containing boundary type - * \param[in] dir direction - * - * \returns 1 if the boundary type is reflecting else 0 - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - bool is_boundary_reflecting (amrex::GpuArray const& pboundary, int dir) { - return ( pboundary[dir] == ParticleBoundaryType::Reflecting ); - } - - /** - * \brief Calculates the number of grid points the given index is pass the - * domain boundary i.e. a value of +1 means the current cell is - * outside of the simulation domain by 1 cell. Note that the high - * side domain boundary is between cell dom_hi and dom_hi+1 for cell - * centered grids and on cell dom_hi+1 for nodal grid. This is why - * (dom_hi[idim] + is_nodal[idim]) is used below. - * - * \param[in] dom_lo, dom_hi Domain boundaries - * \param[in] ijk_vec Cell coordinates - * \param[in] is_nodal Whether the field of interest is nodal - * \param[in] idim Dimension of interest - * \param[in] iside 0 for low and 1 for high - * - * \returns number of grid points to the boundary - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - int get_cell_count_to_boundary (const amrex::IntVect& dom_lo, - const amrex::IntVect& dom_hi, const amrex::IntVect& ijk_vec, - const amrex::IntVect& is_nodal, const int idim, const int iside) - { - return ((iside == 0) ? (dom_lo[idim] - ijk_vec[idim]) - : (ijk_vec[idim] - (dom_hi[idim] + is_nodal[idim]))); - } - - /** - * \brief Sets the electric field value tangential to the PEC boundary to zero. The - * tangential Efield components in the guard cells outside the - * domain boundary are set equal and opposite to the field in the valid cells - * at their mirrored locations. The normal Efield components in the guard cells - * are set equal to the field in the valid cells at their mirrored locations. - * The number or depth of guard cells updated is equal to the shape factor of - * particles in each dimension. - * For corner cells with mixed boundaries, the mirror location could be outside - * valid region, while still ensuring PEC condition is maintained across the - * PEC boundary, and the necessary sign change is accounted for depending on - * if the component, icomp, is tangential or normal to the PEC boundary. - * - * For 3D : - * x component is tangential to the y-boundary and z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary and y-boundary - * x component is normal to the x-boundary - * y component is normal to the y-boundary - * z component is normal to the z-boundary - * where, x-boundary is the yz-plane at x=xmin and x=xmax - * y-boundary is the xz-plane at y=ymin and y=ymax - * z-boundary is the xy-plane at z=zmin and z=zmax - * - * For 2D : WarpX uses X-Z as the two dimensions - * x component is tangential to the z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary - * x component is normal to the x-boundary - * y component is not normal to any boundary (Only xz dimensions in 2D) - * z component is normal to the z-boundary - * where, x-boundary is along the line z at x=xmin and x=xmax - * z-boundary is along the line x at z=zmin and z=zmax - * - * For 1D : WarpX uses Z as the only dimension - * x component is tangential to the z-boundary - * y component is tangential to the z-boundary - * z component is not tangential to the z-boundary - * x component is not normal to any boundary (Only z dimension in 1D) - * y component is not normal to any boundary (Only z dimension in 1D) - * z component is normal to the z-boundary - * where, z-boundary is a point at z=zmin and z=zmax - * - * For RZ : WarpX uses R-Z as the two dimensions - * r component is tangential to the z-boundary - * theta_component is tangential to the r-boundary and z-boundary - * z component is tangential to the r-boundary - * r component is normal to the r-boundary - * theta_component is not normal to any boundary (on RZ dimensions are modeled) - * z component is normal to the z-boundary - * where, r-boundary is along the line z at r=rmin and r=rmax - * z-boundary is along the line r at z=zmin and z=zmax - * - * - * \param[in] icomp component of the Efield being updated - * (0=x, 1=y, 2=z in Cartesian) - * (0=r, 1=theta, 2=z in RZ) - * \param[in] dom_lo index value of the lower domain boundary (cell-centered) - * \param[in] dom_hi index value of the higher domain boundary (cell-centered) - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 - * \param[in] n index of the MultiFab component being updated - * \param[in] Efield field data to be updated if (ijk) is at the boundary or a guard cell - * \param[in] is_nodal staggering of the field data being updated. - * \param[in] fbndry_lo Field boundary type at the lower boundaries - * \param[in] fbndry_hi Field boundary type at the upper boundaries - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetEfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, - const amrex::IntVect &dom_hi, - const amrex::IntVect &ijk_vec, const int n, - amrex::Array4 const& Efield, - const amrex::IntVect& is_nodal, - amrex::GpuArray const& fbndry_lo, - amrex::GpuArray const& fbndry_hi ) - { - // Tangential Efield components in guard cells set equal and opposite to cells - // in the mirror locations across the PEC boundary, whereas normal E-field - // components are set equal to values in the mirror locations across the PEC - // boundary. Here we just initialize it. - amrex::IntVect ijk_mirror = ijk_vec; - bool OnPECBoundary = false; - bool GuardCell = false; - amrex::Real sign = 1._rt; - // Loop over all the dimensions - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - // Loop over sides, iside = 0 (lo), iside = 1 (hi) - for (int iside = 0; iside < 2; ++iside) { - const bool isPECBoundary = ( (iside == 0) - ? is_boundary_PEC(fbndry_lo, idim) - : is_boundary_PEC(fbndry_hi, idim) ); -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - // For 2D : for icomp==1, (Ey in XZ, Etheta in RZ), - // icomp=1 is tangential to both x and z boundaries - // The logic below ensures that the flags are set right for 2D - const bool is_tangent_to_PEC = (icomp != AMREX_SPACEDIM*idim); -#elif (defined WARPX_DIM_1D_Z) - // For 1D : icomp=0 and icomp=1 (Ex and Ey are tangential to the z boundary) - // The logic below ensures that the flags are set right for 1D - const bool is_tangent_to_PEC = (icomp != idim+2); -#else - const bool is_tangent_to_PEC = (icomp != idim); -#endif - if (isPECBoundary) { - // grid point ijk_vec is ig number of points pass the - // domain boundary in direction, idim - const int ig = get_cell_count_to_boundary( - dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); - - if (ig == 0) { - if (is_tangent_to_PEC && is_nodal[idim] == 1) { - OnPECBoundary = true; - } - } else if (ig > 0) { - // Find mirror location across PEC boundary - ijk_mirror[idim] = ( ( iside == 0) - ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) - : (dom_hi[idim] + 1 - ig)); - GuardCell = true; - // tangential components are inverted across PEC boundary - if (is_tangent_to_PEC) { sign *= -1._rt; } -#if (defined WARPX_DIM_RZ) - if (icomp == 0 && idim == 0 && iside == 1) { - // Add radial scale so that drEr/dr = 0. - // This only works for the first guard cell and with - // Er cell centered in r. - const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - sign *= rmirror/rguard; - } -#endif - } - } // is PEC boundary - } // loop over iside - } // loop over dimensions - if (OnPECBoundary) { - // if ijk_vec is on a PEC boundary in any direction, set Etangential to 0. - Efield(ijk_vec,n) = 0._rt; - } else if (GuardCell) { - Efield(ijk_vec,n) = sign * Efield(ijk_mirror,n); - } - } - - - /** - * \brief Sets the magnetic field value normal to the PEC boundary to zero. The - * tangential (and normal) field value of the guard cells outside the - * domain boundary are set equal (and opposite) to the respective field components - * in the valid cells at their mirrored locations. - * The number or depth of guard cells updated is equal to the shape factor of - * particles in each dimension. - * - * For 3D : - * x component is tangential to the y-boundary and z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary and y-boundary - * x component is normal to the x-boundary - * y component is normal to the y-boundary - * z component is normal to the z-boundary - * where, x-boundary is the yz-plane at x=xmin and x=xmax - * y-boundary is the xz-plane at y=ymin and y=ymax - * z-boundary is the xy-plane at z=zmin and z=zmax - * - * For 2D : WarpX uses X-Z as the two dimensions - * x component is tangential to the z-boundary - * y component is tangential to the x-boundary and z-boundary - * z component is tangential to the x-boundary - * x component is normal to the x-boundary - * y component is not normal to any boundary (Only xz dimensions in 2D) - * z component is normal to the z-boundary - * where, x-boundary is along the line z at x=xmin and x=xmax - * z-boundary is along the line x at z=zmin and z=zmax - * - * For 1D : WarpX uses Z as the only dimension - * x component is tangential to the z-boundary - * y component is tangential to the z-boundary - * z component is not tangential to the z-boundary - * x component is not normal to any boundary (Only z dimension in 1D) - * y component is not normal to any boundary (Only z dimension in 1D) - * z component is normal to the z-boundary - * where, z-boundary is a point at z=zmin and z=zmax - * - * For RZ : WarpX uses R-Z as the two dimensions - * r component is tangential to the z-boundary - * theta_component is tangential to the r-boundary and z-boundary - * z component is tangential to the r-boundary - * r component is normal to the r-boundary - * theta_component is not normal to any boundary (on RZ dimensions are modeled) - * z component is normal to the z-boundary - * where, r-boundary is along the line z at r=rmin and r=rmax - * z-boundary is along the line r at z=zmin and z=zmax - * - * - * \param[in] icomp component of the Bfield being updated - * (0=x, 1=y, 2=z in Cartesian) - * (0=r, 1=theta, 2=z in RZ) - * \param[in] dom_lo index value of the lower domain boundary (cell-centered) - * \param[in] dom_hi index value of the higher domain boundary (cell-centered) - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 - * \param[in] n index of the MultiFab component being updated - * \param[in] Bfield field data to be updated if (ijk) is at the boundary - or a guard cell - * \param[in] is_nodal staggering of the field data being updated. - * \param[in] fbndry_lo Field boundary type at the lower boundaries - * \param[in] fbndry_hi Field boundary type at the upper boundaries - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetBfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, - const amrex::IntVect & dom_hi, - const amrex::IntVect & ijk_vec, const int n, - amrex::Array4 const& Bfield, - const amrex::IntVect & is_nodal, - amrex::GpuArray const& fbndry_lo, - amrex::GpuArray const& fbndry_hi ) - { - amrex::IntVect ijk_mirror = ijk_vec; - bool OnPECBoundary = false; - bool GuardCell = false; - amrex::Real sign = 1._rt; - // Loop over all dimensions - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - // Loop over sides, iside = 0 (lo), iside = 1 (hi) - for (int iside = 0; iside < 2; ++iside) { - const bool isPECBoundary = ( (iside == 0 ) - ? is_boundary_PEC(fbndry_lo, idim) - : is_boundary_PEC(fbndry_hi, idim) ); - if (isPECBoundary) { -#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) - // For 2D : for icomp==1, (By in XZ, Btheta in RZ), - // icomp=1 is not normal to x or z boundary - // The logic below ensures that the flags are set right for 2D - const bool is_normal_to_PEC = (icomp == (AMREX_SPACEDIM*idim)); -#elif (defined WARPX_DIM_1D_Z) - // For 1D : icomp=0 and icomp=1 (Bx and By are not normal to the z boundary) - // The logic below ensures that the flags are set right for 1D - const bool is_normal_to_PEC = (icomp == (idim+2)); -#else - const bool is_normal_to_PEC = (icomp == idim); -#endif - - // grid point ijk_vec is ig number of points pass the - // domain boundary in direction, idim - const int ig = get_cell_count_to_boundary( - dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); - - if (ig == 0) { - // Only normal component is set to 0 - if (is_normal_to_PEC && is_nodal[idim]==1) { - OnPECBoundary = true; - } - } else if ( ig > 0) { - // Mirror location inside the domain by "ig" number of cells - // across PEC boundary in direction, idim, and side, iside - ijk_mirror[idim] = ( (iside == 0) - ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) - : (dom_hi[idim] + 1 - ig)); - GuardCell = true; - // Sign of the normal component in guard cell is inverted - if (is_normal_to_PEC) { sign *= -1._rt; } -#if (defined WARPX_DIM_RZ) - if (icomp == 0 && idim == 0 && iside == 1) { - // Add radial scale so that drBr/dr = 0. - const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); - sign *= rmirror/rguard; - } -#endif - } - } // if PEC Boundary - } // loop over sides - } // loop of dimensions - - if (OnPECBoundary) { - // if ijk_vec is on a PEC boundary in any direction, set Bnormal to 0. - Bfield(ijk_vec,n) = 0._rt; - } else if (GuardCell) { - // Bnormal and Btangential is set opposite and equal to the value - // in the mirror location, respectively. - Bfield(ijk_vec,n) = sign * Bfield(ijk_mirror,n); - } - } - - - /** - * \brief Sets the rho or J field value in cells close to and on reflecting particle boundary - * or PEC field boundary. The charge/current density deposited - * in the guard cells are either reflected - * back into the simulation domain (if a reflecting particle - * boundary is used), or the opposite charge/current density is deposited - * back in the domain to capture the effect of an image charge. - * The charge/current density on the reflecting boundary is set to 0 while values - * in the guard cells are set equal (and opposite) to their mirror - * location inside the domain - representing image charges - in the - * normal (tangential) direction. - * - * \param[in] n index of the MultiFab component being updated - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 - * \param[in out] field field data to be updated - * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec - * \param[in] psign Whether the field value should be flipped across the boundary - * \param[in] is_reflective Whether the given particle boundary is reflecting or field boundary is pec - * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary - * \param[in] fabbox multifab box including ghost cells - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetRhoOrJfieldFromPEC (const int n, - const amrex::IntVect & ijk_vec, - amrex::Array4 const& field, - amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, - amrex::GpuArray, AMREX_SPACEDIM> const& psign, - amrex::GpuArray, AMREX_SPACEDIM> const& is_reflective, - amrex::GpuArray const& tangent_to_bndy, - amrex::Box const& fabbox) - { - // The boundary is handled in 2 steps: - // 1) The cells internal to the domain are updated using the - // current deposited in the guard cells - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int iside = 0; iside < 2; ++iside) - { - if (!is_reflective[idim][iside]) { continue; } - - // Get the mirror guard cell index - amrex::IntVect iv_mirror = ijk_vec; - iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - - // On the PEC boundary the charge/current density is set to 0 - if (ijk_vec == iv_mirror) { - field(ijk_vec, n) = 0._rt; - // otherwise update the internal cell if the mirror guard cell exists - } else if (fabbox.contains(iv_mirror)) { - field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); - } - } - } - // 2) The guard cells are updated with the appropriate image - // charge based on the charge/current in the valid cells - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int iside = 0; iside < 2; ++iside) - { - if (!is_reflective[idim][iside]) { continue; } - - amrex::IntVect iv_mirror = ijk_vec; - iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - if (ijk_vec != iv_mirror && fabbox.contains(iv_mirror)) - { - if (tangent_to_bndy[idim]) { - field(iv_mirror, n) = -field(ijk_vec, n); - } else { - field(iv_mirror, n) = field(ijk_vec, n); - } - } - } - } - } - - - /** - * \brief This function sets the given field value on a PEC boundary - * to enforce a Neumann boundary condition (zero derivative) in the - * normal direction. - * - * \param[in] n index of the MultiFab component being updated - * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 - * \param[in out] field field data to be updated - * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec - * \param[in] psign Whether the field value should be flipped across the boundary - * \param[in] is_pec Whether the given boundary is PEC - * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary - * \param[in] fabbox multifab box including ghost cells - */ - AMREX_GPU_DEVICE AMREX_FORCE_INLINE - void SetNeumannOnPEC (const int n, - const amrex::IntVect & ijk_vec, - amrex::Array4 const& field, - amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, - amrex::GpuArray, AMREX_SPACEDIM> const& is_pec, - amrex::Box const& fabbox ) - { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - for (int iside = 0; iside < 2; ++iside) - { - if (!is_pec[idim][iside]) { continue; } - - // Get the mirror guard cell index - amrex::IntVect iv_mirror = ijk_vec; - iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; - - // On the PEC boundary the field value is set equal to the - // first value in the domain (nodal fields) - if (ijk_vec == iv_mirror) { - iv_mirror[idim] += (iside == 0) ? 1 : -1; - if (fabbox.contains(iv_mirror)) { field(ijk_vec, n) = field(iv_mirror, n); } - } - // otherwise set the mirror guard cell equal to the internal cell value - else if (fabbox.contains(iv_mirror)) - { - field(iv_mirror, n) = field(ijk_vec, n); - } - } - } - } - - - /** Returns 1 if any domain boundary is set to PEC, else returns 0.*/ - bool isAnyBoundaryPEC(); - - /** Return true if any particle boundary is set to reflecting, else returns false*/ - bool isAnyParticleBoundaryReflecting (); /** * \brief Sets the tangential electric field at the PEC boundary to zero. * The guard cell values are set equal and opposite to the valid cell * field value at the respective mirror locations. * - * \param[in,out] Efield Boundary values of tangential Efield are set to zero - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine - * \param[in] split_pml_field whether pml the multifab is the regular Efield or - * split pml field + * \param[in,out] Efield Boundary values of tangential Efield are set to zero + * \param[in] field_boundary_lo Boundary types of the "low" boundaries + * \param[in] field_boundary_hi Boundary types of the "high" boundaries + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels + * \param[in] split_pml_field whether pml the multifab is the regular Efield or + * split pml field */ - void ApplyPECtoEfield ( std::array Efield, - int lev, PatchType patch_type, - bool split_pml_field = false); + void ApplyPECtoEfield ( + std::array Efield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios, + bool split_pml_field = false); /** * \brief Sets the normal component of the magnetic field at the PEC boundary to zero. * The guard cell values are set equal and opposite to the valid cell * field value at the respective mirror locations. * - * \param[in,out] Bfield Boundary values of normal Bfield are set to zero. - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] Bfield Boundary values of normal Bfield are set to zero. + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] ng_fieldgather number of guard cells used by field gather + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyPECtoBfield ( std::array Bfield, - int lev, PatchType patch_type); + void ApplyPECtoBfield ( + std::array Bfield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); /** * \brief Reflects charge density deposited over the PEC boundary back into * the simulation domain. * - * \param[in,out] rho Multifab containing the charge density - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] rho Multifab containing the charge density + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] particle_boundary_lo Boundary types of the "low" particle boundaries + * \param[in] particle_boundary_hi Boundary types of the "high" particle boundaries + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyReflectiveBoundarytoRhofield(amrex::MultiFab* rho, int lev, - PatchType patch_type); + void ApplyReflectiveBoundarytoRhofield( + amrex::MultiFab* rho, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); /** * \brief Reflects current density deposited over the PEC boundary back into * the simulation domain. * - * \param[in,out] Jx, Jy, Jz Multifabs containing the current density - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] Jx, Jy, Jz Multifabs containing the current density + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] particle_boundary_lo Boundary types of the "low" particle boundaries + * \param[in] particle_boundary_hi Boundary types of the "high" particle boundaries + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, - amrex::MultiFab* Jz, int lev, - PatchType patch_type); + void ApplyReflectiveBoundarytoJfield( + amrex::MultiFab* Jx, amrex::MultiFab* Jy, + amrex::MultiFab* Jz, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); /** * \brief Apply the PEC boundary to the electron pressure field. * - * \param[in,out] Pefield Multifab containing the electron pressure - * \param[in] lev level of the Multifab - * \param[in] patch_type coarse or fine + * \param[in,out] Pefield Multifab containing the electron pressure + * \param[in] field_boundary_lo Boundary types of the "low" field boundaries + * \param[in] field_boundary_hi Boundary types of the "high" field boundaries + * \param[in] geom geometry object of level "lev" + * \param[in] lev level of the Multifab + * \param[in] patch_type coarse or fine + * \param[in] ref_ratios vector containing the refinement ratios of the refinement levels */ - void ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, - int lev, PatchType patch_type); + void ApplyPECtoElectronPressure ( + amrex::MultiFab* Pefield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Geometry& geom, + int lev, PatchType patch_type, const amrex::Vector& ref_ratios); } #endif // WarpX_PEC_KERNELS_H_ diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index a3d92fa507d..a42873db555 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -1,60 +1,483 @@ #include "BoundaryConditions/WarpX_PEC.H" -#include "WarpX.H" #include +#include #include +#include +#include #include #include #include +#include #include +#include #include #include +#include #include -#include +using namespace amrex; using namespace amrex::literals; -bool -PEC::isAnyBoundaryPEC() { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC) { return true; } - if ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC) { return true; } +namespace +{ + /** + * \brief Calculates the number of grid points the given index is pass the + * domain boundary i.e. a value of +1 means the current cell is + * outside of the simulation domain by 1 cell. Note that the high + * side domain boundary is between cell dom_hi and dom_hi+1 for cell + * centered grids and on cell dom_hi+1 for nodal grid. This is why + * (dom_hi[idim] + is_nodal[idim]) is used below. + * + * \param[in] dom_lo, dom_hi Domain boundaries + * \param[in] ijk_vec Cell coordinates + * \param[in] is_nodal Whether the field of interest is nodal + * \param[in] idim Dimension of interest + * \param[in] iside 0 for low and 1 for high + * + * \returns number of grid points to the boundary + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + int get_cell_count_to_boundary (const amrex::IntVect& dom_lo, + const amrex::IntVect& dom_hi, const amrex::IntVect& ijk_vec, + const amrex::IntVect& is_nodal, const int idim, const int iside) + { + return ((iside == 0) ? (dom_lo[idim] - ijk_vec[idim]) + : (ijk_vec[idim] - (dom_hi[idim] + is_nodal[idim]))); } - return false; -} -bool -PEC::isAnyParticleBoundaryReflecting () { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) {return true;} - if (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) {return true;} + + /** + * \brief Sets the electric field value tangential to the PEC boundary to zero. The + * tangential Efield components in the guard cells outside the + * domain boundary are set equal and opposite to the field in the valid cells + * at their mirrored locations. The normal Efield components in the guard cells + * are set equal to the field in the valid cells at their mirrored locations. + * The number or depth of guard cells updated is equal to the shape factor of + * particles in each dimension. + * For corner cells with mixed boundaries, the mirror location could be outside + * valid region, while still ensuring PEC condition is maintained across the + * PEC boundary, and the necessary sign change is accounted for depending on + * if the component, icomp, is tangential or normal to the PEC boundary. + * + * For 3D : + * x component is tangential to the y-boundary and z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary and y-boundary + * x component is normal to the x-boundary + * y component is normal to the y-boundary + * z component is normal to the z-boundary + * where, x-boundary is the yz-plane at x=xmin and x=xmax + * y-boundary is the xz-plane at y=ymin and y=ymax + * z-boundary is the xy-plane at z=zmin and z=zmax + * + * For 2D : WarpX uses X-Z as the two dimensions + * x component is tangential to the z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary + * x component is normal to the x-boundary + * y component is not normal to any boundary (Only xz dimensions in 2D) + * z component is normal to the z-boundary + * where, x-boundary is along the line z at x=xmin and x=xmax + * z-boundary is along the line x at z=zmin and z=zmax + * + * For 1D : WarpX uses Z as the only dimension + * x component is tangential to the z-boundary + * y component is tangential to the z-boundary + * z component is not tangential to the z-boundary + * x component is not normal to any boundary (Only z dimension in 1D) + * y component is not normal to any boundary (Only z dimension in 1D) + * z component is normal to the z-boundary + * where, z-boundary is a point at z=zmin and z=zmax + * + * For RZ : WarpX uses R-Z as the two dimensions + * r component is tangential to the z-boundary + * theta_component is tangential to the r-boundary and z-boundary + * z component is tangential to the r-boundary + * r component is normal to the r-boundary + * theta_component is not normal to any boundary (on RZ dimensions are modeled) + * z component is normal to the z-boundary + * where, r-boundary is along the line z at r=rmin and r=rmax + * z-boundary is along the line r at z=zmin and z=zmax + * + * + * \param[in] icomp component of the Efield being updated + * (0=x, 1=y, 2=z in Cartesian) + * (0=r, 1=theta, 2=z in RZ) + * \param[in] dom_lo index value of the lower domain boundary (cell-centered) + * \param[in] dom_hi index value of the higher domain boundary (cell-centered) + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 + * \param[in] n index of the MultiFab component being updated + * \param[in] Efield field data to be updated if (ijk) is at the boundary or a guard cell + * \param[in] is_nodal staggering of the field data being updated. + * \param[in] fbndry_lo Field boundary type at the lower boundaries + * \param[in] fbndry_hi Field boundary type at the upper boundaries + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetEfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, + const amrex::IntVect &dom_hi, + const amrex::IntVect &ijk_vec, const int n, + amrex::Array4 const& Efield, + const amrex::IntVect& is_nodal, + amrex::GpuArray const& fbndry_lo, + amrex::GpuArray const& fbndry_hi ) + { + // Tangential Efield components in guard cells set equal and opposite to cells + // in the mirror locations across the PEC boundary, whereas normal E-field + // components are set equal to values in the mirror locations across the PEC + // boundary. Here we just initialize it. + amrex::IntVect ijk_mirror = ijk_vec; + bool OnPECBoundary = false; + bool GuardCell = false; + amrex::Real sign = 1._rt; + // Loop over all the dimensions + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + // Loop over sides, iside = 0 (lo), iside = 1 (hi) + for (int iside = 0; iside < 2; ++iside) { + const bool isPECBoundary = ( (iside == 0) + ? fbndry_lo[idim] == FieldBoundaryType::PEC + : fbndry_hi[idim] == FieldBoundaryType::PEC ); +#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) + // For 2D : for icomp==1, (Ey in XZ, Etheta in RZ), + // icomp=1 is tangential to both x and z boundaries + // The logic below ensures that the flags are set right for 2D + const bool is_tangent_to_PEC = (icomp != AMREX_SPACEDIM*idim); +#elif (defined WARPX_DIM_1D_Z) + // For 1D : icomp=0 and icomp=1 (Ex and Ey are tangential to the z boundary) + // The logic below ensures that the flags are set right for 1D + const bool is_tangent_to_PEC = (icomp != idim+2); +#else + const bool is_tangent_to_PEC = (icomp != idim); +#endif + if (isPECBoundary) { + // grid point ijk_vec is ig number of points pass the + // domain boundary in direction, idim + const int ig = ::get_cell_count_to_boundary( + dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); + + if (ig == 0) { + if (is_tangent_to_PEC && is_nodal[idim] == 1) { + OnPECBoundary = true; + } + } else if (ig > 0) { + // Find mirror location across PEC boundary + ijk_mirror[idim] = ( ( iside == 0) + ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) + : (dom_hi[idim] + 1 - ig)); + GuardCell = true; + // tangential components are inverted across PEC boundary + if (is_tangent_to_PEC) { sign *= -1._rt; } +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drEr/dr = 0. + // This only works for the first guard cell and with + // Er cell centered in r. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif + } + } // is PEC boundary + } // loop over iside + } // loop over dimensions + if (OnPECBoundary) { + // if ijk_vec is on a PEC boundary in any direction, set Etangential to 0. + Efield(ijk_vec,n) = 0._rt; + } else if (GuardCell) { + Efield(ijk_vec,n) = sign * Efield(ijk_mirror,n); + } + } + + + /** + * \brief Sets the magnetic field value normal to the PEC boundary to zero. The + * tangential (and normal) field value of the guard cells outside the + * domain boundary are set equal (and opposite) to the respective field components + * in the valid cells at their mirrored locations. + * The number or depth of guard cells updated is equal to the shape factor of + * particles in each dimension. + * + * For 3D : + * x component is tangential to the y-boundary and z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary and y-boundary + * x component is normal to the x-boundary + * y component is normal to the y-boundary + * z component is normal to the z-boundary + * where, x-boundary is the yz-plane at x=xmin and x=xmax + * y-boundary is the xz-plane at y=ymin and y=ymax + * z-boundary is the xy-plane at z=zmin and z=zmax + * + * For 2D : WarpX uses X-Z as the two dimensions + * x component is tangential to the z-boundary + * y component is tangential to the x-boundary and z-boundary + * z component is tangential to the x-boundary + * x component is normal to the x-boundary + * y component is not normal to any boundary (Only xz dimensions in 2D) + * z component is normal to the z-boundary + * where, x-boundary is along the line z at x=xmin and x=xmax + * z-boundary is along the line x at z=zmin and z=zmax + * + * For 1D : WarpX uses Z as the only dimension + * x component is tangential to the z-boundary + * y component is tangential to the z-boundary + * z component is not tangential to the z-boundary + * x component is not normal to any boundary (Only z dimension in 1D) + * y component is not normal to any boundary (Only z dimension in 1D) + * z component is normal to the z-boundary + * where, z-boundary is a point at z=zmin and z=zmax + * + * For RZ : WarpX uses R-Z as the two dimensions + * r component is tangential to the z-boundary + * theta_component is tangential to the r-boundary and z-boundary + * z component is tangential to the r-boundary + * r component is normal to the r-boundary + * theta_component is not normal to any boundary (on RZ dimensions are modeled) + * z component is normal to the z-boundary + * where, r-boundary is along the line z at r=rmin and r=rmax + * z-boundary is along the line r at z=zmin and z=zmax + * + * + * \param[in] icomp component of the Bfield being updated + * (0=x, 1=y, 2=z in Cartesian) + * (0=r, 1=theta, 2=z in RZ) + * \param[in] dom_lo index value of the lower domain boundary (cell-centered) + * \param[in] dom_hi index value of the higher domain boundary (cell-centered) + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of Efield Array4 + * \param[in] n index of the MultiFab component being updated + * \param[in] Bfield field data to be updated if (ijk) is at the boundary + or a guard cell + * \param[in] is_nodal staggering of the field data being updated. + * \param[in] fbndry_lo Field boundary type at the lower boundaries + * \param[in] fbndry_hi Field boundary type at the upper boundaries + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetBfieldOnPEC (const int icomp, const amrex::IntVect & dom_lo, + const amrex::IntVect & dom_hi, + const amrex::IntVect & ijk_vec, const int n, + amrex::Array4 const& Bfield, + const amrex::IntVect & is_nodal, + amrex::GpuArray const& fbndry_lo, + amrex::GpuArray const& fbndry_hi ) + { + amrex::IntVect ijk_mirror = ijk_vec; + bool OnPECBoundary = false; + bool GuardCell = false; + amrex::Real sign = 1._rt; + // Loop over all dimensions + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + // Loop over sides, iside = 0 (lo), iside = 1 (hi) + for (int iside = 0; iside < 2; ++iside) { + const bool isPECBoundary = ( (iside == 0) + ? fbndry_lo[idim] == FieldBoundaryType::PEC + : fbndry_hi[idim] == FieldBoundaryType::PEC ); + if (isPECBoundary) { +#if (defined WARPX_DIM_XZ) || (defined WARPX_DIM_RZ) + // For 2D : for icomp==1, (By in XZ, Btheta in RZ), + // icomp=1 is not normal to x or z boundary + // The logic below ensures that the flags are set right for 2D + const bool is_normal_to_PEC = (icomp == (AMREX_SPACEDIM*idim)); +#elif (defined WARPX_DIM_1D_Z) + // For 1D : icomp=0 and icomp=1 (Bx and By are not normal to the z boundary) + // The logic below ensures that the flags are set right for 1D + const bool is_normal_to_PEC = (icomp == (idim+2)); +#else + const bool is_normal_to_PEC = (icomp == idim); +#endif + + // grid point ijk_vec is ig number of points pass the + // domain boundary in direction, idim + const int ig = ::get_cell_count_to_boundary( + dom_lo, dom_hi, ijk_vec, is_nodal, idim, iside); + + if (ig == 0) { + // Only normal component is set to 0 + if (is_normal_to_PEC && is_nodal[idim]==1) { + OnPECBoundary = true; + } + } else if ( ig > 0) { + // Mirror location inside the domain by "ig" number of cells + // across PEC boundary in direction, idim, and side, iside + ijk_mirror[idim] = ( (iside == 0) + ? (dom_lo[idim] + ig - (1 - is_nodal[idim])) + : (dom_hi[idim] + 1 - ig)); + GuardCell = true; + // Sign of the normal component in guard cell is inverted + if (is_normal_to_PEC) { sign *= -1._rt; } +#if (defined WARPX_DIM_RZ) + if (icomp == 0 && idim == 0 && iside == 1) { + // Add radial scale so that drBr/dr = 0. + const amrex::Real rguard = ijk_vec[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + const amrex::Real rmirror = ijk_mirror[idim] + 0.5_rt*(1._rt - is_nodal[idim]); + sign *= rmirror/rguard; + } +#endif + } + } // if PEC Boundary + } // loop over sides + } // loop of dimensions + + if (OnPECBoundary) { + // if ijk_vec is on a PEC boundary in any direction, set Bnormal to 0. + Bfield(ijk_vec,n) = 0._rt; + } else if (GuardCell) { + // Bnormal and Btangential is set opposite and equal to the value + // in the mirror location, respectively. + Bfield(ijk_vec,n) = sign * Bfield(ijk_mirror,n); + } + } + + + /** + * \brief Sets the rho or J field value in cells close to and on reflecting particle boundary + * or PEC field boundary. The charge/current density deposited + * in the guard cells are either reflected + * back into the simulation domain (if a reflecting particle + * boundary is used), or the opposite charge/current density is deposited + * back in the domain to capture the effect of an image charge. + * The charge/current density on the reflecting boundary is set to 0 while values + * in the guard cells are set equal (and opposite) to their mirror + * location inside the domain - representing image charges - in the + * normal (tangential) direction. + * + * \param[in] n index of the MultiFab component being updated + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 + * \param[in out] field field data to be updated + * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec + * \param[in] psign Whether the field value should be flipped across the boundary + * \param[in] is_reflective Whether the given particle boundary is reflecting or field boundary is pec + * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary + * \param[in] fabbox multifab box including ghost cells + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetRhoOrJfieldFromPEC (const int n, + const amrex::IntVect & ijk_vec, + amrex::Array4 const& field, + amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, + amrex::GpuArray, AMREX_SPACEDIM> const& psign, + amrex::GpuArray, AMREX_SPACEDIM> const& is_reflective, + amrex::GpuArray const& tangent_to_bndy, + amrex::Box const& fabbox) + { + // The boundary is handled in 2 steps: + // 1) The cells internal to the domain are updated using the + // current deposited in the guard cells + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int iside = 0; iside < 2; ++iside) + { + if (!is_reflective[idim][iside]) { continue; } + + // Get the mirror guard cell index + amrex::IntVect iv_mirror = ijk_vec; + iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; + + // On the PEC boundary the charge/current density is set to 0 + if (ijk_vec == iv_mirror) { + field(ijk_vec, n) = 0._rt; + // otherwise update the internal cell if the mirror guard cell exists + } else if (fabbox.contains(iv_mirror)) { + field(ijk_vec,n) += psign[idim][iside] * field(iv_mirror,n); + } + } + } + // 2) The guard cells are updated with the appropriate image + // charge based on the charge/current in the valid cells + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int iside = 0; iside < 2; ++iside) + { + if (!is_reflective[idim][iside]) { continue; } + + amrex::IntVect iv_mirror = ijk_vec; + iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; + if (ijk_vec != iv_mirror && fabbox.contains(iv_mirror)) + { + if (tangent_to_bndy[idim]) { + field(iv_mirror, n) = -field(ijk_vec, n); + } else { + field(iv_mirror, n) = field(ijk_vec, n); + } + } + } + } + } + + + /** + * \brief This function sets the given field value on a PEC boundary + * to enforce a Neumann boundary condition (zero derivative) in the + * normal direction. + * + * \param[in] n index of the MultiFab component being updated + * \param[in] ijk_vec indices along the x(i), y(j), z(k) of the rho Array4 + * \param[in out] field field data to be updated + * \param[in] mirrorfac mirror cell is given by mirrorfac - ijk_vec + * \param[in] psign Whether the field value should be flipped across the boundary + * \param[in] is_pec Whether the given boundary is PEC + * \param[in] tangent_to_bndy Whether a given direction is perpendicular to the boundary + * \param[in] fabbox multifab box including ghost cells + */ + AMREX_GPU_DEVICE AMREX_FORCE_INLINE + void SetNeumannOnPEC (const int n, + const amrex::IntVect & ijk_vec, + amrex::Array4 const& field, + amrex::GpuArray, AMREX_SPACEDIM> const& mirrorfac, + amrex::GpuArray, AMREX_SPACEDIM> const& is_pec, + amrex::Box const& fabbox ) + { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + for (int iside = 0; iside < 2; ++iside) + { + if (!is_pec[idim][iside]) { continue; } + + // Get the mirror guard cell index + amrex::IntVect iv_mirror = ijk_vec; + iv_mirror[idim] = mirrorfac[idim][iside] - ijk_vec[idim]; + + // On the PEC boundary the field value is set equal to the + // first value in the domain (nodal fields) + if (ijk_vec == iv_mirror) { + iv_mirror[idim] += (iside == 0) ? 1 : -1; + if (fabbox.contains(iv_mirror)) { field(ijk_vec, n) = field(iv_mirror, n); } + } + // otherwise set the mirror guard cell equal to the internal cell value + else if (fabbox.contains(iv_mirror)) + { + field(iv_mirror, n) = field(ijk_vec, n); + } + } + } } - return false; } void -PEC::ApplyPECtoEfield (std::array Efield, const int lev, - PatchType patch_type, const bool split_pml_field) +PEC::ApplyPECtoEfield ( + std::array Efield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios, + const bool split_pml_field) { - auto& warpx = WarpX::GetInstance(); - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } const amrex::IntVect domain_lo = domain_box.smallEnd(); const amrex::IntVect domain_hi = domain_box.bigEnd(); - amrex::GpuArray fbndry_lo; - amrex::GpuArray fbndry_hi; + amrex::GpuArray fbndry_lo; + amrex::GpuArray fbndry_hi; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - fbndry_lo[idim] = WarpX::field_boundary_lo[idim]; - fbndry_hi[idim] = WarpX::field_boundary_hi[idim]; + fbndry_lo[idim] = field_boundary_lo[idim]; + fbndry_hi[idim] = field_boundary_hi[idim]; } const amrex::IntVect Ex_nodal = Efield[0]->ixType().toIntVect(); const amrex::IntVect Ey_nodal = Efield[1]->ixType().toIntVect(); const amrex::IntVect Ez_nodal = Efield[2]->ixType().toIntVect(); - const amrex::IntVect ng_fieldgather = warpx.get_ng_fieldgather(); // For each Efield multifab, apply PEC boundary condition to ncomponents // If not split E-field, the PEC is applied to the regular Efield used in Maxwell's eq. // If split_pml_field is true, then PEC is applied to all the split field components of the tangential field. @@ -95,7 +518,7 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 0; - PEC::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Ex, Ex_nodal, fbndry_lo, fbndry_hi); }, tey, nComp_y, @@ -108,7 +531,7 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 1; - PEC::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Ey, Ey_nodal, fbndry_lo, fbndry_hi); }, tez, nComp_z, @@ -121,7 +544,7 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 2; - PEC::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetEfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Ez, Ez_nodal, fbndry_lo, fbndry_hi); } ); @@ -130,27 +553,28 @@ PEC::ApplyPECtoEfield (std::array Efield, const int lev, void -PEC::ApplyPECtoBfield (std::array Bfield, const int lev, - PatchType patch_type) +PEC::ApplyPECtoBfield ( + std::array Bfield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::IntVect& ng_fieldgather, const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } const amrex::IntVect domain_lo = domain_box.smallEnd(); const amrex::IntVect domain_hi = domain_box.bigEnd(); - amrex::GpuArray fbndry_lo; - amrex::GpuArray fbndry_hi; + amrex::GpuArray fbndry_lo; + amrex::GpuArray fbndry_hi; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - fbndry_lo[idim] = WarpX::field_boundary_lo[idim]; - fbndry_hi[idim] = WarpX::field_boundary_hi[idim]; + fbndry_lo[idim] = field_boundary_lo[idim]; + fbndry_hi[idim] = field_boundary_hi[idim]; } const amrex::IntVect Bx_nodal = Bfield[0]->ixType().toIntVect(); const amrex::IntVect By_nodal = Bfield[1]->ixType().toIntVect(); const amrex::IntVect Bz_nodal = Bfield[2]->ixType().toIntVect(); - const amrex::IntVect ng_fieldgather = warpx.get_ng_fieldgather(); const int nComp_x = Bfield[0]->nComp(); const int nComp_y = Bfield[1]->nComp(); const int nComp_z = Bfield[2]->nComp(); @@ -186,7 +610,7 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 0; - PEC::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Bx, Bx_nodal, fbndry_lo, fbndry_hi); }, tby, nComp_y, @@ -199,7 +623,7 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 1; - PEC::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, By, By_nodal, fbndry_lo, fbndry_hi); }, tbz, nComp_z, @@ -212,7 +636,7 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, #endif const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); const int icomp = 2; - PEC::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, + ::SetBfieldOnPEC(icomp, domain_lo, domain_hi, iv, n, Bz, Bz_nodal, fbndry_lo, fbndry_hi); } ); @@ -231,14 +655,18 @@ PEC::ApplyPECtoBfield (std::array Bfield, const int lev, * location inside the domain - representing image charges. **/ void -PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, PatchType patch_type) +PEC::ApplyReflectiveBoundarytoRhofield ( + amrex::MultiFab* rho, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } domain_box.convert(rho->ixType()); @@ -257,12 +685,12 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat amrex::GpuArray, AMREX_SPACEDIM> psign; amrex::GpuArray, AMREX_SPACEDIM> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_reflective[idim][0] = ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) - || ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC); - is_reflective[idim][1] = ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) - || ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC); + is_reflective[idim][0] = ( particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_lo[idim] == FieldBoundaryType::PEC); + is_reflective[idim][1] = ( particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_hi[idim] == FieldBoundaryType::PEC); if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -270,11 +698,11 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat // components of the current density is_tangent_to_bndy[idim] = true; - psign[idim][0] = ( ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) + psign[idim][0] = ((particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; - psign[idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) + psign[idim][1] = ((particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; mirrorfac[idim][0] = 2*domain_lo[idim] - (1 - rho_nodal[idim]); mirrorfac[idim][1] = 2*domain_hi[idim] + (1 - rho_nodal[idim]); @@ -307,7 +735,7 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( + ::SetRhoOrJfieldFromPEC( n, iv, rho_array, mirrorfac, psign, is_reflective, is_tangent_to_bndy, fabbox ); @@ -317,16 +745,18 @@ PEC::ApplyReflectiveBoundarytoRhofield (amrex::MultiFab* rho, const int lev, Pat void -PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, - amrex::MultiFab* Jz, const int lev, - PatchType patch_type) +PEC::ApplyReflectiveBoundarytoJfield( + amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::MultiFab* Jz, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Vector& particle_boundary_lo, + const amrex::Vector& particle_boundary_hi, + const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } // Note: force domain box to be nodal to simplify the mirror cell @@ -355,12 +785,12 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, amrex::GpuArray, AMREX_SPACEDIM>, 3> psign; amrex::GpuArray, AMREX_SPACEDIM>, 3> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_reflective[idim][0] = ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) - || ( WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC); - is_reflective[idim][1] = ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) - || ( WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC); + is_reflective[idim][0] = ( particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_lo[idim] == FieldBoundaryType::PEC); + is_reflective[idim][1] = ( particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + || ( particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) + || ( field_boundary_hi[idim] == FieldBoundaryType::PEC); if (!is_reflective[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_reflective[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -381,19 +811,19 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, #endif if (is_tangent_to_bndy[icomp][idim]){ - psign[icomp][idim][0] = ( (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) + psign[icomp][idim][0] = ( (particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; - psign[icomp][idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) + psign[icomp][idim][1] = ( (particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) ? 1._rt : -1._rt; } else { - psign[icomp][idim][0] = ( (WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_lo[idim] == ParticleBoundaryType::Thermal) ) + psign[icomp][idim][0] = ( (particle_boundary_lo[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_lo[idim] == ParticleBoundaryType::Thermal)) ? -1._rt : 1._rt; - psign[icomp][idim][1] = ( (WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) - || ( WarpX::particle_boundary_hi[idim] == ParticleBoundaryType::Thermal) ) + psign[icomp][idim][1] = ( (particle_boundary_hi[idim] == ParticleBoundaryType::Reflecting) + ||(particle_boundary_hi[idim] == ParticleBoundaryType::Thermal)) ? -1._rt : 1._rt; } } @@ -437,7 +867,7 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( + ::SetRhoOrJfieldFromPEC( n, iv, Jx_array, mirrorfac[0], psign[0], is_reflective, is_tangent_to_bndy[0], fabbox ); @@ -472,7 +902,7 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( + ::SetRhoOrJfieldFromPEC( n, iv, Jy_array, mirrorfac[1], psign[1], is_reflective, is_tangent_to_bndy[1], fabbox ); @@ -507,7 +937,7 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetRhoOrJfieldFromPEC( + ::SetRhoOrJfieldFromPEC( n, iv, Jz_array, mirrorfac[2], psign[2], is_reflective, is_tangent_to_bndy[2], fabbox ); @@ -516,15 +946,16 @@ PEC::ApplyReflectiveBoundarytoJfield(amrex::MultiFab* Jx, amrex::MultiFab* Jy, } void -PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, - PatchType patch_type) +PEC::ApplyPECtoElectronPressure ( + amrex::MultiFab* Pefield, + const amrex::Vector& field_boundary_lo, + const amrex::Vector& field_boundary_hi, + const amrex::Geometry& geom, + const int lev, PatchType patch_type, const amrex::Vector& ref_ratios) { - auto& warpx = WarpX::GetInstance(); - - amrex::Box domain_box = warpx.Geom(lev).Domain(); - if (patch_type == PatchType::coarse) { - const amrex::IntVect ref_ratio = ( (lev > 0) ? WarpX::RefRatio(lev-1) : amrex::IntVect(1) ); - domain_box.coarsen(ref_ratio); + amrex::Box domain_box = geom.Domain(); + if (patch_type == PatchType::coarse && (lev > 0)) { + domain_box.coarsen(ref_ratios[lev-1]); } domain_box.convert(Pefield->ixType()); @@ -541,8 +972,8 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, amrex::GpuArray, AMREX_SPACEDIM> is_pec; amrex::GpuArray, AMREX_SPACEDIM> mirrorfac; for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - is_pec[idim][0] = WarpX::field_boundary_lo[idim] == FieldBoundaryType::PEC; - is_pec[idim][1] = WarpX::field_boundary_hi[idim] == FieldBoundaryType::PEC; + is_pec[idim][0] = field_boundary_lo[idim] == FieldBoundaryType::PEC; + is_pec[idim][1] = field_boundary_hi[idim] == FieldBoundaryType::PEC; if (!is_pec[idim][0]) { grown_domain_box.growLo(idim, ng_fieldgather[idim]); } if (!is_pec[idim][1]) { grown_domain_box.growHi(idim, ng_fieldgather[idim]); } @@ -577,7 +1008,7 @@ PEC::ApplyPECtoElectronPressure (amrex::MultiFab* Pefield, const int lev, // Store the array index const amrex::IntVect iv(AMREX_D_DECL(i,j,k)); - PEC::SetNeumannOnPEC(n, iv, Pe_array, mirrorfac, is_pec, fabbox); + ::SetNeumannOnPEC(n, iv, Pe_array, mirrorfac, is_pec, fabbox); }); } } diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp index 36be09696fd..5e75698903e 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ApplySilverMuellerBoundary.cpp @@ -39,8 +39,8 @@ void FiniteDifferenceSolver::ApplySilverMuellerBoundary ( std::array< std::unique_ptr, 3 >& Bfield, amrex::Box domain_box, amrex::Real const dt, - amrex::Vector field_boundary_lo, - amrex::Vector field_boundary_hi) { + amrex::Vector field_boundary_lo, + amrex::Vector field_boundary_hi) { // Ensure that we are using the Yee solver WARPX_ALWAYS_ASSERT_WITH_MESSAGE( diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index d045a30dd44..a7a4f10a713 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -10,6 +10,7 @@ #include "EmbeddedBoundary/WarpXFaceInfoBox_fwd.H" #include "FiniteDifferenceSolver_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" #include "BoundaryConditions/PML_fwd.H" #include "Evolve/WarpXDtType.H" @@ -89,8 +90,8 @@ class FiniteDifferenceSolver std::array< std::unique_ptr, 3 >& Bfield, amrex::Box domain_box, amrex::Real dt, - amrex::Vector field_boundary_lo, - amrex::Vector field_boundary_hi); + amrex::Vector field_boundary_lo, + amrex::Vector field_boundary_hi); void ComputeDivE ( const std::array,3>& Efield, amrex::MultiFab& divE ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 7d477b4e5fd..5c0c2bcc96a 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -12,16 +12,16 @@ #include "HybridPICModel_fwd.H" +#include "Utils/WarpXAlgorithmSelection.H" + #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" -#include "WarpX.H" #include #include - /** * \brief This class contains the parameters needed to evaluate hybrid field * solutions (kinetic ions with fluid electrons). diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 1da869b4ba0..9768b36cb84 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -55,6 +55,12 @@ struct GridType { }; }; +enum struct PatchType +{ + fine, + coarse +}; + struct ElectromagneticSolverAlgo { enum { None = 0, @@ -146,14 +152,13 @@ struct LoadBalanceCostsUpdateAlgo { /** Field boundary conditions at the domain boundary */ -struct FieldBoundaryType { - enum { +enum struct FieldBoundaryType { PML = 0, Periodic = 1, - PEC = 2, //!< perfect electric conductor (PEC) with E_tangential=0 + PEC = 2, //!< perfect electric conductor (PEC) with E_tangential=0 PMC = 3, //!< perfect magnetic conductor (PMC) with B_tangential=0 Damped = 4, // Fields in the guard cells are damped for PSATD - //in the moving window direction + //in the moving window direction Absorbing_SilverMueller = 5, // Silver-Mueller boundary condition Neumann = 6, // For electrostatic, the normal E is set to zero None = 7, // The fields values at the boundary are not updated. This is @@ -161,7 +166,6 @@ struct FieldBoundaryType { Open = 8 // Used in the Integrated Green Function Poisson solver // Note that the solver implicitely assumes open BCs: // no need to enforce them separately - }; }; /** Particle boundary conditions at the domain boundary @@ -190,7 +194,7 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ); /** Select BC Type for fields, if field=true * else select BCType for particles. */ -int +FieldBoundaryType GetFieldBCTypeInteger( std::string BCType ); /** Select BC Type for particles. @@ -201,6 +205,6 @@ GetParticleBCTypeInteger( std::string BCType ); /** Find the name associated with a BC type */ std::string -GetFieldBCTypeString( int fb_type ); +GetFieldBCTypeString( FieldBoundaryType fb_type ); #endif // UTILS_WARPXALGORITHMSELECTION_H_ diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 3386ce55b48..b1f93c2f6c8 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -123,7 +123,7 @@ const std::map MacroscopicSolver_algo_to_int = { {"default", MacroscopicSolverAlgo::BackwardEuler} }; -const std::map FieldBCType_algo_to_int = { +const std::map FieldBCType_algo_to_enum = { {"pml", FieldBoundaryType::PML}, {"periodic", FieldBoundaryType::Periodic}, {"pec", FieldBoundaryType::PEC}, @@ -222,21 +222,21 @@ GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ return algo_to_int[algo]; } -int +FieldBoundaryType GetFieldBCTypeInteger( std::string BCType ){ std::transform(BCType.begin(), BCType.end(), BCType.begin(), ::tolower); - if (FieldBCType_algo_to_int.count(BCType) == 0) { + if (FieldBCType_algo_to_enum.count(BCType) == 0) { std::string error_message = "Invalid string for field/particle BC. : " + BCType + "\nThe valid values are : \n"; - for (const auto &valid_pair : FieldBCType_algo_to_int) { + for (const auto &valid_pair : FieldBCType_algo_to_enum) { if (valid_pair.first != "default"){ error_message += " - " + valid_pair.first + "\n"; } } WARPX_ABORT_WITH_MESSAGE(error_message); } - // return FieldBCType_algo_to_int[BCType]; // This operator cannot be used for a const map - return FieldBCType_algo_to_int.at(BCType); + // return FieldBCType_algo_to_enum[BCType]; // This operator cannot be used for a const map + return FieldBCType_algo_to_enum.at(BCType); } ParticleBoundaryType @@ -257,9 +257,9 @@ GetParticleBCTypeInteger( std::string BCType ){ } std::string -GetFieldBCTypeString( int fb_type ) { +GetFieldBCTypeString( FieldBoundaryType fb_type ) { std::string boundary_name; - for (const auto &valid_pair : FieldBCType_algo_to_int) { + for (const auto &valid_pair : FieldBCType_algo_to_enum) { if ((valid_pair.second == fb_type)&&(valid_pair.first != "default")){ boundary_name = valid_pair.first; break; diff --git a/Source/WarpX.H b/Source/WarpX.H index cdd2447ed5e..9ca754ea1bb 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -75,12 +75,6 @@ #include #include -enum struct PatchType : int -{ - fine, - coarse -}; - enum struct FieldType : int { Efield_aux, @@ -215,12 +209,12 @@ public: * lower domain boundaries * (0 to 6 correspond to PML, Periodic, PEC, PMC, Damped, Absorbing Silver-Mueller, None) */ - static amrex::Vector field_boundary_lo; + static amrex::Vector field_boundary_lo; /** Integers that correspond to boundary condition applied to fields at the * upper domain boundaries * (0 to 6 correspond to PML, Periodic, PEC, PMC, Damped, Absorbing Silver-Mueller, None) */ - static amrex::Vector field_boundary_hi; + static amrex::Vector field_boundary_hi; /** Integers that correspond to boundary condition applied to particles at the * lower domain boundaries * (0 to 4 correspond to Absorbing, Open, Reflecting, Periodic, Thermal) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index f89a61e524b..9a9903f013f 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -137,8 +137,8 @@ amrex::IntVect WarpX::shared_tilesize(AMREX_D_DECL(1,1,1)); #endif int WarpX::shared_mem_current_tpb = 128; -amrex::Vector WarpX::field_boundary_lo(AMREX_SPACEDIM,0); -amrex::Vector WarpX::field_boundary_hi(AMREX_SPACEDIM,0); +amrex::Vector WarpX::field_boundary_lo(AMREX_SPACEDIM,FieldBoundaryType::PML); +amrex::Vector WarpX::field_boundary_hi(AMREX_SPACEDIM,FieldBoundaryType::PML); amrex::Vector WarpX::particle_boundary_lo(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); amrex::Vector WarpX::particle_boundary_hi(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); From 5640de35aec95dcd2247798d645a7625c3c5807a Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 9 Apr 2024 11:08:04 -0700 Subject: [PATCH 037/190] Refactor output of phi on the particles (#4839) * Refactor output of phi on the particles --- Source/Diagnostics/ParticleIO.cpp | 54 +++++++++++++++++++++++++++++ Source/Diagnostics/WarpXOpenPMD.cpp | 27 +-------------- Source/Particles/ParticleIO.H | 45 ++++-------------------- 3 files changed, 62 insertions(+), 64 deletions(-) diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index a8bb9303fe1..5cb18602ed9 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -235,3 +235,57 @@ MultiParticleContainer::WriteHeader (std::ostream& os) const allcontainers.at(i)->WriteHeader(os); } } + +void +storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, + int electrostatic_solver_id, bool is_full_diagnostic ) { + + using PinnedParIter = typename PinnedMemoryParticleContainer::ParIterType; + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame) || + (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic), + "Output of the electrostatic potential (phi) on the particles was requested, " + "but this is only available for `warpx.do_electrostatic=labframe` or `labframe-electromagnetostatic`."); + // When this is not a full diagnostic, the particles are not written at the same physical time (i.e. PIC iteration) + // that they were collected. This happens for diagnostics that use buffering (e.g. BackTransformed, BoundaryScraping). + // Here `phi` is gathered at the iteration when particles are written (not collected) and is thus mismatched. + // To avoid confusion, we raise an error in this case. + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + is_full_diagnostic, + "Output of the electrostatic potential (phi) on the particles was requested, " + "but this is only available with `diag_type = Full`."); + tmp.AddRealComp("phi"); + int const phi_index = tmp.getParticleComps().at("phi"); + auto& warpx = WarpX::GetInstance(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for (int lev=0; lev<=warpx.finestLevel(); lev++) { + const amrex::Geometry& geom = warpx.Geom(lev); + auto plo = geom.ProbLoArray(); + auto dxi = geom.InvCellSizeArray(); + amrex::MultiFab const& phi = warpx.getField( FieldType::phi_fp, lev, 0 ); + + for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { + + auto phi_grid = phi[pti].array(); + const auto getPosition = GetParticlePosition(pti); + amrex::ParticleReal* phi_particle_arr = pti.GetStructOfArrays().GetRealData(phi_index).dataPtr(); + + // Loop over the particles and update their position + amrex::ParallelFor( pti.numParticles(), + [=] AMREX_GPU_DEVICE (long ip) { + + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + int i, j, k; + amrex::Real W[AMREX_SPACEDIM][2]; + ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); + amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi_grid); + phi_particle_arr[ip] = phi_value; + } + ); + } + } +} diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 6882b962445..965eb82b31e 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -591,32 +591,7 @@ for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) { // Gather the electrostatic potential (phi) on the macroparticles if ( particle_diags[i].m_plot_phi ) { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrame) || - (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic), - "Output of the electrostatic potential (phi) on the particles was requested, " - "but this is only available for `warpx.do_electrostatic=labframe` or `labframe-electromagnetostatic`."); - // Using pinned PC indicates that the particles are not written at the same physical time (i.e. PIC iteration) - // that they were collected. This happens for diagnostics that use buffering (e.g. BackTransformed, BoundaryScraping). - // Here `phi` is gathered at the iteration when particles are written (not collected) and is thus mismatched. - // To avoid confusion, we raise an error in this case. - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - use_pinned_pc == false, - "Output of the electrostatic potential (phi) on the particles was requested, " - "but this is only available with `diag_type = Full`."); - tmp.AddRealComp("phi"); - int const phi_index = tmp.getParticleComps().at("phi"); - auto& warpx = WarpX::GetInstance(); -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for (int lev=0; lev<=warpx.finestLevel(); lev++) { - const amrex::Geometry& geom = warpx.Geom(lev); - auto plo = geom.ProbLoArray(); - auto dxi = geom.InvCellSizeArray(); - amrex::MultiFab const& phi = warpx.getField( FieldType::phi_fp, lev, 0 ); - storeFieldOnParticles( tmp, plo, dxi, phi, phi_index, lev ); - } + storePhiOnParticles( tmp, WarpX::electrostatic_solver_id, !use_pinned_pc ); } // names of amrex::Real and int particle attributes in SoA data diff --git a/Source/Particles/ParticleIO.H b/Source/Particles/ParticleIO.H index 1abda6ea9af..022ab471362 100644 --- a/Source/Particles/ParticleIO.H +++ b/Source/Particles/ParticleIO.H @@ -9,6 +9,7 @@ #include "Particles/WarpXParticleContainer.H" #include "Particles/Pusher/GetAndSetPosition.H" +#include "Particles/PinnedMemoryParticleContainer.H" #include @@ -80,47 +81,15 @@ particlesConvertUnits (ConvertDirection convert_direction, T_ParticleContainer* } } -/** Gathers the field from a MultiFab to the macroparticles - * and stores it into a runtime component of the particle container +/** Gathers phi (electrostatic potential) from a MultiFab to the macroparticles. + * Adds a runtime component of the particle container to store it. * - * @tparam T_ParticleContainer a WarpX particle container or AmrParticleContainer * @param tmp the particle container on which to store the gathered field - * @param plo the coordinates of the low bound of the box, along each dimension - * @param dxi the inverse of the cell size, along each dimension - * @param phi a multifab that contains the field to be gathered - * @param phi_index the index of the runtime component where to store the gathered field - * @param lev the MR level + * @param electrostatic_solver_id the type of electrostatic solver used + * @param is_full_diagnostic whether this diagnostic is a full diagnostic */ -template< typename T_ParticleContainer > void -storeFieldOnParticles ( T_ParticleContainer& tmp, - amrex::GpuArray const& plo, - amrex::GpuArray const& dxi, - amrex::MultiFab const& phi, - int const phi_index, int const lev ) { - - using PinnedParIter = typename T_ParticleContainer::ParIterType; - - for (PinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { - - auto phi_grid = phi[pti].array(); - const auto getPosition = GetParticlePosition(pti); - amrex::ParticleReal* phi_particle_arr = pti.GetStructOfArrays().GetRealData(phi_index).dataPtr(); - - // Loop over the particles and update their position - amrex::ParallelFor( pti.numParticles(), - [=] AMREX_GPU_DEVICE (long ip) { - - amrex::ParticleReal xp, yp, zp; - getPosition(ip, xp, yp, zp); - int i, j, k; - amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); - amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi_grid); - phi_particle_arr[ip] = phi_value; - } - ); - } -} +storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, + int electrostatic_solver_id, bool is_full_diagnostic ); #endif /* PARTICLEIO_H_ */ From a3974ff6b7cf6370c72a4f2c9372058bc1ce803b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 9 Apr 2024 15:29:02 -0700 Subject: [PATCH 038/190] Filter `rho`, when using relativistic electrostatic solver (#4823) * Filter/synchronize rho across levels * Allocate coarse version of `rho` * Remove filtering for now * Remove boundary treatment * Apply boundary and scale volume * Apply suggestions from code review * Update benchmark * Update benchmarks * Increase resolution to mitigate the effects of filtering * Increase resolution in relativistic test * Use filter in openbc test * Update benchmark * Update resolution * Update Examples/Tests/openbc_poisson_solver/inputs_3d * Update checksums --- .../Tests/openbc_poisson_solver/inputs_3d | 5 +- .../inputs_3d | 2 +- .../space_charge_initialization/inputs_3d | 2 +- .../benchmarks_json/BeamBeamCollision.json | 140 +++++++++--------- .../benchmarks_json/ElectrostaticSphere.json | 18 +-- .../ElectrostaticSphereRelNodal.json | 26 ++-- .../NodalElectrostaticSolver.json | 18 +-- .../openbc_poisson_solver.json | 30 ++-- ...ativistic_space_charge_initialization.json | 10 +- .../space_charge_initialization.json | 6 +- .../space_charge_initialization_2d.json | 4 +- Source/FieldSolver/ElectrostaticSolver.cpp | 17 ++- 12 files changed, 145 insertions(+), 133 deletions(-) diff --git a/Examples/Tests/openbc_poisson_solver/inputs_3d b/Examples/Tests/openbc_poisson_solver/inputs_3d index fd4299f360b..2ac17a06751 100644 --- a/Examples/Tests/openbc_poisson_solver/inputs_3d +++ b/Examples/Tests/openbc_poisson_solver/inputs_3d @@ -8,8 +8,8 @@ amr.n_cell = 128 128 128 amr.max_level = 0 amr.max_grid_size = 128 geometry.dims = 3 -geometry.prob_lo = -5*sigmax -5*sigmay -5*sigmaz -geometry.prob_hi = +5*sigmax +5*sigmay +5*sigmaz +geometry.prob_lo = -4*sigmax -4*sigmay -4*sigmaz +geometry.prob_hi = +4*sigmax +4*sigmay +4*sigmaz boundary.field_lo = open open open boundary.field_hi = open open open warpx.const_dt = 1e-14 @@ -18,7 +18,6 @@ warpx.poisson_solver = fft algo.field_gathering = momentum-conserving algo.particle_shape = 1 - particles.species_names = electron electron.charge = -q_e diff --git a/Examples/Tests/relativistic_space_charge_initialization/inputs_3d b/Examples/Tests/relativistic_space_charge_initialization/inputs_3d index 91cd6000547..c472f88c007 100644 --- a/Examples/Tests/relativistic_space_charge_initialization/inputs_3d +++ b/Examples/Tests/relativistic_space_charge_initialization/inputs_3d @@ -1,5 +1,5 @@ max_step = 1 -amr.n_cell = 64 64 64 +amr.n_cell = 128 128 128 amr.max_grid_size = 32 amr.max_level = 0 diff --git a/Examples/Tests/space_charge_initialization/inputs_3d b/Examples/Tests/space_charge_initialization/inputs_3d index a24b67ff02e..c8058fac519 100644 --- a/Examples/Tests/space_charge_initialization/inputs_3d +++ b/Examples/Tests/space_charge_initialization/inputs_3d @@ -1,5 +1,5 @@ max_step = 1 -amr.n_cell = 64 64 64 +amr.n_cell = 128 128 128 amr.max_grid_size = 32 amr.max_level = 0 diff --git a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json index 693faf43882..799692135ce 100644 --- a/Regression/Checksum/benchmarks_json/BeamBeamCollision.json +++ b/Regression/Checksum/benchmarks_json/BeamBeamCollision.json @@ -1,96 +1,96 @@ { "lev=0": { - "Bx": 970573570349.2142, - "By": 970493693614.8823, - "Bz": 18743533.08373785, - "Ex": 2.9100804648817213e+20, - "Ey": 2.910188719414779e+20, - "Ez": 1.309938057448976e+17, - "rho_beam1": 7.969171294602906e+16, - "rho_beam2": 7.969079911431987e+16, - "rho_ele1": 234837475722889.97, - "rho_ele2": 277285508564124.12, - "rho_pos1": 229824415763789.62, - "rho_pos2": 286513388076434.4 + "Bx": 958613893612.4355, + "By": 958606286084.2804, + "Bz": 50291310.73170665, + "Ex": 2.873993867215236e+20, + "Ey": 2.8740263458334176e+20, + "Ez": 1.3469564521662371e+17, + "rho_beam1": 7.96926761425663e+16, + "rho_beam2": 7.969234189119718e+16, + "rho_ele1": 325562788515568.7, + "rho_ele2": 301373974746269.7, + "rho_pos1": 314013278169396.75, + "rho_pos2": 311298336003435.56 }, "beam1": { - "particle_opticalDepthQSR": 104947.58821047074, - "particle_position_x": 0.0015001846430437182, - "particle_position_y": 0.0015002200838688795, - "particle_position_z": 0.004965556992831605, - "particle_momentum_x": 6.207861637949496e-15, - "particle_momentum_y": 6.16591876835476e-15, - "particle_momentum_z": 6.806755306915448e-12, - "particle_weight": 635859587.7915188 + "particle_opticalDepthQSR": 104848.69019384333, + "particle_position_x": 0.0015001641452988207, + "particle_position_y": 0.0015001296473841738, + "particle_position_z": 0.004965480036291212, + "particle_momentum_x": 6.203657034942546e-15, + "particle_momentum_y": 6.161790111190829e-15, + "particle_momentum_z": 6.806194292286189e-12, + "particle_weight": 635868088.3867786 }, "beam2": { - "particle_opticalDepthQSR": 104180.42997257302, - "particle_position_x": 0.0015001132145418016, - "particle_position_y": 0.001500162338878405, - "particle_position_z": 0.004965528365042041, - "particle_momentum_x": 6.201791193788119e-15, - "particle_momentum_y": 6.188946480983165e-15, - "particle_momentum_z": 6.796967564632488e-12, - "particle_weight": 635853166.1373858 + "particle_opticalDepthQSR": 104197.28107371755, + "particle_position_x": 0.0015001452558405398, + "particle_position_y": 0.0015001281739351966, + "particle_position_z": 0.0049656445643994716, + "particle_momentum_x": 6.202758467582172e-15, + "particle_momentum_y": 6.18910011814166e-15, + "particle_momentum_z": 6.7994521022372906e-12, + "particle_weight": 635874794.3085052 }, "ele1": { - "particle_opticalDepthQSR": 385.3959944370225, - "particle_position_x": 5.063867198487091e-06, - "particle_position_y": 5.323267522706671e-06, - "particle_position_z": 1.81974337459613e-05, - "particle_momentum_x": 5.220808698440275e-18, - "particle_momentum_y": 5.2833645967924376e-18, - "particle_momentum_z": 2.4960852396311436e-15, - "particle_weight": 1883777.3071500752 + "particle_opticalDepthQSR": 398.7154177999122, + "particle_position_x": 5.2166787076833645e-06, + "particle_position_y": 5.005755590473258e-06, + "particle_position_z": 1.856829463647771e-05, + "particle_momentum_x": 6.0736878569270085e-18, + "particle_momentum_y": 5.735020185191748e-18, + "particle_momentum_z": 2.827581034346608e-15, + "particle_weight": 2602683.4209351614 }, "ele2": { - "particle_opticalDepthQSR": 391.0032239817431, - "particle_position_x": 5.442400321293304e-06, - "particle_position_y": 5.3621200149906786e-06, - "particle_position_z": 1.716649458876156e-05, - "particle_momentum_x": 4.7056270741663116e-18, - "particle_momentum_y": 4.414438514376292e-18, - "particle_momentum_z": 2.3816287305827113e-15, - "particle_weight": 2255238.1204111334 + "particle_opticalDepthQSR": 328.6975869797729, + "particle_position_x": 4.984003903707884e-06, + "particle_position_y": 4.695016970410262e-06, + "particle_position_z": 1.606918799511055e-05, + "particle_momentum_x": 4.524294388810778e-18, + "particle_momentum_y": 4.193609622515901e-18, + "particle_momentum_z": 2.624217472737641e-15, + "particle_weight": 2432495.8168380223 }, "pho1": { - "particle_opticalDepthBW": 9912.332305353064, - "particle_position_x": 0.0001424857999095925, - "particle_position_y": 0.00014366766674682975, - "particle_position_z": 0.0004764478273708734, + "particle_opticalDepthBW": 10028.214317531058, + "particle_position_x": 0.00014200324200040716, + "particle_position_y": 0.00014310262095706036, + "particle_position_z": 0.00047470309948487784, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_weight": 61671336.44071695 + "particle_weight": 61455533.15171491 }, "pho2": { - "particle_opticalDepthBW": 10361.142481614075, - "particle_position_x": 0.00014718841339786984, - "particle_position_y": 0.00014538267635727008, - "particle_position_z": 0.00048768591004702896, + "particle_opticalDepthBW": 10261.48950301913, + "particle_position_x": 0.0001465092909391631, + "particle_position_y": 0.00014555115652303745, + "particle_position_z": 0.00048686081947093, "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 0.0, - "particle_weight": 61889405.34553001 + "particle_weight": 61924991.09906147 }, "pos1": { - "particle_opticalDepthQSR": 343.0370605565725, - "particle_position_x": 5.41949101836001e-06, - "particle_position_y": 5.5292865311090835e-06, - "particle_position_z": 1.7063207973991194e-05, - "particle_momentum_x": 4.76704175252458e-18, - "particle_momentum_y": 5.049007427826035e-18, - "particle_momentum_z": 2.533578387785534e-15, - "particle_weight": 1821809.4309160584 + "particle_opticalDepthQSR": 380.4787933889546, + "particle_position_x": 5.59226140958729e-06, + "particle_position_y": 5.199149983019462e-06, + "particle_position_z": 1.7261766049926983e-05, + "particle_momentum_x": 5.182944941041321e-18, + "particle_momentum_y": 4.665394338329992e-18, + "particle_momentum_z": 2.565450485567441e-15, + "particle_weight": 2523696.1743423166 }, "pos2": { - "particle_opticalDepthQSR": 390.0211924195479, - "particle_position_x": 5.053169658257247e-06, - "particle_position_y": 5.039302796539821e-06, - "particle_position_z": 1.8526669235892217e-05, - "particle_momentum_x": 5.755216173987019e-18, - "particle_momentum_y": 5.056618594918483e-18, - "particle_momentum_z": 2.52324147594341e-15, - "particle_weight": 2328558.5523011778 + "particle_opticalDepthQSR": 378.7526306435402, + "particle_position_x": 4.812490588954386e-06, + "particle_position_y": 4.351750384371962e-06, + "particle_position_z": 1.7621416174292307e-05, + "particle_momentum_x": 4.979887438720444e-18, + "particle_momentum_y": 4.8215630209506066e-18, + "particle_momentum_z": 2.193964301475807e-15, + "particle_weight": 2513162.277112609 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json index 4e9bde52a78..45441acf466 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphere.json @@ -1,17 +1,17 @@ { "lev=0": { - "Ex": 6.504803536004636, - "Ey": 6.504803536004637, - "Ez": 6.504803536004637, + "Ex": 6.504631859049922, + "Ey": 6.50463185904992, + "Ez": 6.504631859049919, "rho": 2.609256800833379e-10 }, "electron": { - "particle_momentum_x": 1.0084159184928337e-23, - "particle_momentum_y": 1.008415918492834e-23, - "particle_momentum_z": 1.0084159184928338e-23, - "particle_position_x": 518.418176976446, - "particle_position_y": 518.4181769764461, - "particle_position_z": 518.418176976446, + "particle_momentum_x": 9.882423407879068e-24, + "particle_momentum_y": 9.882423407879068e-24, + "particle_momentum_z": 9.882423407879066e-24, + "particle_position_x": 513.5229059338338, + "particle_position_y": 513.5229059338338, + "particle_position_z": 513.522905933834, "particle_weight": 6212.501525878906 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json index 8a79a510352..45441acf466 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereRelNodal.json @@ -1,17 +1,17 @@ { - "electron": { - "particle_momentum_x": 1.0084159184928337e-23, - "particle_momentum_y": 1.008415918492834e-23, - "particle_momentum_z": 1.0084159184928338e-23, - "particle_position_x": 518.418176976446, - "particle_position_y": 518.4181769764461, - "particle_position_z": 518.418176976446, - "particle_weight": 6212.501525878906 - }, "lev=0": { - "Ex": 6.504803536004636, - "Ey": 6.504803536004637, - "Ez": 6.504803536004637, + "Ex": 6.504631859049922, + "Ey": 6.50463185904992, + "Ez": 6.504631859049919, "rho": 2.609256800833379e-10 + }, + "electron": { + "particle_momentum_x": 9.882423407879068e-24, + "particle_momentum_y": 9.882423407879068e-24, + "particle_momentum_z": 9.882423407879066e-24, + "particle_position_x": 513.5229059338338, + "particle_position_y": 513.5229059338338, + "particle_position_z": 513.522905933834, + "particle_weight": 6212.501525878906 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json b/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json index 2c60799425a..0692546f6cf 100644 --- a/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json +++ b/Regression/Checksum/benchmarks_json/NodalElectrostaticSolver.json @@ -1,15 +1,15 @@ { "lev=0": { - "Bx": 4966241349161.254, - "By": 4966241349161.246, - "Bz": 6.831134328962481e-11, - "Ex": 1.4888417010888241e+21, - "Ey": 1.4888417010888241e+21, - "Ez": 8138626456.10672, - "g_beam_p": 175096250199.0888, + "Bx": 4964230908893.931, + "By": 4964230908893.93, + "Bz": 7.391430009205359e-11, + "Ex": 1.4882389862594233e+21, + "Ey": 1.4882389862594225e+21, + "Ez": 8130823640.561069, + "g_beam_p": 175096250199.08884, "rho_beam_p": 8.331476384415223e+17, - "vx_beam_p": 3.9150198762467515e-10, - "vy_beam_p": 3.915019770559888e-10, + "vx_beam_p": 3.912093052231238e-10, + "vy_beam_p": 3.9120930706063816e-10, "vz_beam_p": 715792.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json index 50b402b4bab..e4ff1fc68a8 100644 --- a/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json +++ b/Regression/Checksum/benchmarks_json/openbc_poisson_solver.json @@ -1,20 +1,20 @@ { "lev=0": { - "Bx": 67876594.52390265, - "By": 112770930.05280717, - "Bz": 1.1060167389533003e-13, - "Ex": 3.380787637240387e+16, - "Ey": 2.034889235345363e+16, - "Ez": 2601470.9380206033, - "rho": 5630115646680.221 + "Bx": 100915933.44993827, + "By": 157610622.1855512, + "Bz": 9.717358898362187e-14, + "Ex": 4.7250652706211096e+16, + "Ey": 3.0253948990559976e+16, + "Ez": 3276573.9514776524, + "rho": 10994013582437.193 }, "electron": { - "particle_momentum_x": 6.378073505535043e-19, - "particle_momentum_y": 3.838951101276526e-19, - "particle_momentum_z": 1.1454327682972433e-10, - "particle_position_x": 21.64260864000591, - "particle_position_y": 0.3229614080035443, - "particle_position_z": 12582.912000000004, - "particle_weight": 19972794743.418434 + "particle_momentum_x": 5.701277606050295e-19, + "particle_momentum_y": 3.6504516641520437e-19, + "particle_momentum_z": 1.145432768297242e-10, + "particle_position_x": 17.314086912497864, + "particle_position_y": 0.2583691267187796, + "particle_position_z": 10066.329600000008, + "particle_weight": 19969036501.910976 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json b/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json index d97f8b58053..adb4477378b 100644 --- a/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json +++ b/Regression/Checksum/benchmarks_json/relativistic_space_charge_initialization.json @@ -1,11 +1,11 @@ { "lev=0": { - "By": 3.176136185686860e-05, - "Ex": 9.522292819119155e+03, - "Ey": 9.516582886012895e+03, - "Ez": 8.118976924626002, + "By": 0.00025366943932442746, + "Ex": 76051.98704866154, + "Ey": 76009.35086584378, + "Ez": 64.90079149285842, "jx": 0.0, "jy": 0.0, - "jz": 785848.6496458045 + "jz": 6286789.197166436 } } diff --git a/Regression/Checksum/benchmarks_json/space_charge_initialization.json b/Regression/Checksum/benchmarks_json/space_charge_initialization.json index e0a157f2a81..07916316a06 100644 --- a/Regression/Checksum/benchmarks_json/space_charge_initialization.json +++ b/Regression/Checksum/benchmarks_json/space_charge_initialization.json @@ -1,8 +1,8 @@ { "lev=0": { - "Ex": 7869.6596181638515, - "Ey": 7863.981043503829, - "Ez": 7869.780361756294, + "Ex": 62838.484914953624, + "Ey": 62796.0798671538, + "Ez": 62839.82091026324, "jx": 0.0, "jy": 0.0, "jz": 0.0 diff --git a/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json b/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json index f37add38914..6ca98880d4a 100644 --- a/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json +++ b/Regression/Checksum/benchmarks_json/space_charge_initialization_2d.json @@ -1,8 +1,8 @@ { "lev=0": { - "Ex": 7438.125125466449, + "Ex": 29702.801592163793, "Ey": 0.0, - "Ez": 7438.329203373256, + "Ez": 29704.478300339702, "jx": 0.0, "jy": 0.0, "jz": 0.0 diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 8f022c308af..d0818959b30 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -164,6 +164,7 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) // Allocate fields for charge and potential const int num_levels = max_level + 1; Vector > rho(num_levels); + Vector > rho_coarse(num_levels); // Used in order to interpolate between levels Vector > phi(num_levels); // Use number of guard cells used for local deposition of rho const amrex::IntVect ng = guard_cells.ng_depos_rho; @@ -174,16 +175,28 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) rho[lev]->setVal(0.); phi[lev] = std::make_unique(nba, DistributionMap(lev), 1, 1); phi[lev]->setVal(0.); + if (lev > 0) { + // For MR levels: allocated the coarsened version of rho + BoxArray cba = nba; + cba.coarsen(refRatio(lev-1)); + rho_coarse[lev] = std::make_unique(cba, DistributionMap(lev), 1, ng); + rho_coarse[lev]->setVal(0.); + } } // Deposit particle charge density (source of Poisson solver) - bool const local = false; + // The options below are identical to those in MultiParticleContainer::DepositCharge + bool const local = true; bool const reset = false; bool const apply_boundary_and_scale_volume = true; + bool const interpolate_across_levels = false; if ( !pc.do_not_deposit) { - pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume); + pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, + interpolate_across_levels); } + SyncRho(rho, rho_coarse, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels + // Get the particle beta vector bool const local_average = false; // Average across all MPI ranks std::array beta_pr = pc.meanParticleVelocity(local_average); From e19d66b511f965728b12724441e4a4a6459ad13c Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Tue, 9 Apr 2024 19:38:09 -0700 Subject: [PATCH 039/190] Fix out of bounds access to `distance_to_eb` (#4831) * Fix out of bounds access to `distance_to_eb` * Fix clang-tidy error * specify level for redistribute and set `nGrow=1` * Set `local=0` in order to avoid unnecessary communication in the second `Redistribute` call. * Switch `local` back to `1`. * move second redistribute inside EB-only block --- Source/Particles/WarpXParticleContainer.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index fd1bc7917ab..ddbfda57c71 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -295,13 +295,16 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, ); } + // Move particles to their appropriate tiles + Redistribute(); + // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + // Call (local) redistribute again to remove particles with invalid ids + Redistribute(0, -1, 0, 1, true); #endif - - Redistribute(); } /* \brief Current Deposition for thread thread_num From 833813d7fefd23487d865c21a6d75a71f0a00283 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 10 Apr 2024 18:10:47 +0200 Subject: [PATCH 040/190] Doc: LUMI (CSC) Update (#4810) * Doc: LUMI (CSC) Update Update the documentation for LUMI (CSC). * LUMI: `-DWarpX_QED_TABLES_GEN_OMP=OFF` Compile issues. --- Docs/source/install/hpc/lumi.rst | 47 +++++-- .../machines/lumi-csc/install_dependencies.sh | 124 +++++++++++++----- .../lumi-csc/{submit.sh => lumi.sbatch} | 2 +- .../lumi-csc/lumi_warpx.profile.example | 27 ++-- 4 files changed, 142 insertions(+), 58 deletions(-) rename Tools/machines/lumi-csc/{submit.sh => lumi.sbatch} (97%) diff --git a/Docs/source/install/hpc/lumi.rst b/Docs/source/install/hpc/lumi.rst index ee65d5b421b..2bae4d21599 100644 --- a/Docs/source/install/hpc/lumi.rst +++ b/Docs/source/install/hpc/lumi.rst @@ -3,7 +3,7 @@ LUMI (CSC) ========== -The `LUMI cluster `_ is located at CSC (Finland). +The `LUMI cluster `__ is located at CSC (Finland). Each node contains 4 AMD MI250X GPUs, each with 2 Graphics Compute Dies (GCDs) for a total of 8 GCDs per node. You can think of the 8 GCDs as 8 separate GPUs, each having 64 GB of high-bandwidth memory (HBM2E). @@ -12,8 +12,11 @@ Introduction If you are new to this system, **please see the following resources**: -* `Lumi user guide `_ -* Batch system: `Slurm `_ +* `Lumi user guide `__ + + * `Project Maintainance `__ and `SSH Key management `__ + * `Quotas and projects `__ +* Batch system: `Slurm `__ * `Data analytics and visualization `__ * `Production directories `__: @@ -48,9 +51,16 @@ Create it now: .. literalinclude:: ../../../../Tools/machines/lumi-csc/lumi_warpx.profile.example :language: bash -Edit the 2nd line of this script, which sets the ``export proj=""`` variable using a text editor -such as ``nano``, ``emacs``, or ``vim`` (all available by default on -LUMI login nodes). +Edit the 2nd line of this script, which sets the ``export proj="project_..."`` variable using a text editor +such as ``nano``, ``emacs``, or ``vim`` (all available by default on LUMI login nodes). +You can find out your project name by running ``lumi-ldap-userinfo`` on LUMI. +For example, if you are member of the project ``project_465000559``, then run ``nano $HOME/lumi_impactx.profile`` and edit line 2 to read: + +.. code-block:: bash + + export proj="project_465000559" + +Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). .. important:: @@ -88,7 +98,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_lumi - cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_lumi/bin/``. @@ -98,7 +108,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_lumi_py - cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi_py -j 16 --target pip_install @@ -143,11 +153,26 @@ Running MI250X GPUs (2x64 GB) ^^^^^^^^^^^^^^^^^^^^^ -In non-interactive runs: +The GPU partition on the supercomputer LUMI at CSC has up to `2978 nodes `__, each with 8 Graphics Compute Dies (GCDs). +WarpX runs one MPI rank per Graphics Compute Die. + +For interactive runs, simply use the aliases ``getNode`` or ``runNode ...``. -.. literalinclude:: ../../../../Tools/machines/lumi-csc/submit.sh +The batch script below can be used to run a WarpX simulation on multiple nodes (change ``-N`` accordingly). +Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` or the concete inputs file. +Copy the executable or point to it via ``EXE`` and adjust the path for the ``INPUTS`` variable accordingly. + +.. literalinclude:: ../../../../Tools/machines/lumi-csc/lumi.sbatch :language: bash - :caption: You can copy this file from ``Tools/machines/lumi-csc/submit.sh``. + :caption: You can copy this file from ``Tools/machines/lumi-csc/lumi.sbatch``. + +To run a simulation, copy the lines above to a file ``lumi.sbatch`` and run + +.. code-block:: bash + + sbatch lumi.sbatch + +to submit the job. .. _post-processing-lumi: diff --git a/Tools/machines/lumi-csc/install_dependencies.sh b/Tools/machines/lumi-csc/install_dependencies.sh index 0e466f1f57f..814e134bad4 100755 --- a/Tools/machines/lumi-csc/install_dependencies.sh +++ b/Tools/machines/lumi-csc/install_dependencies.sh @@ -14,15 +14,17 @@ set -eu -o pipefail # Check: ###################################################################### # -# Was perlmutter_gpu_warpx.profile sourced and configured correctly? +# Was lumi_warpx.profile sourced and configured correctly? if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lumi_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi # Remove old dependencies ##################################################### # +SRC_DIR="${HOME}/src" SW_DIR="${HOME}/sw/lumi/gpu" rm -rf ${SW_DIR} mkdir -p ${SW_DIR} +mkdir -p ${SRC_DIR} # remove common user mistakes in python, located in .local instead of a venv python3 -m pip uninstall -qq -y pywarpx @@ -33,63 +35,113 @@ python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true # General extra dependencies ################################################## # +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + # BLAS++ (for PSATD+RZ) -if [ -d $HOME/src/blaspp ] +if [ -d ${SRC_DIR}/blaspp ] then - cd $HOME/src/blaspp + cd ${SRC_DIR}/blaspp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi -rm -rf $HOME/src/blaspp-lumi-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-lumi-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build $HOME/src/blaspp-lumi-gpu-build --target install --parallel 16 -rm -rf $HOME/src/blaspp-lumi-gpu-build +rm -rf ${build_dir}/blaspp-lumi-gpu-build +CXX=$(which CC) \ +cmake -S ${SRC_DIR}/blaspp \ + -B ${build_dir}/blaspp-lumi-gpu-build \ + -Duse_openmp=OFF \ + -Dgpu_backend=hip \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-lumi-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-lumi-gpu-build # LAPACK++ (for PSATD+RZ) -if [ -d $HOME/src/lapackpp ] +if [ -d ${SRC_DIR}/lapackpp ] then - cd $HOME/src/lapackpp + cd ${SRC_DIR}/lapackpp git fetch --prune git checkout master git pull cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi -rm -rf $HOME/src/lapackpp-lumi-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-lumi-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -cmake --build $HOME/src/lapackpp-lumi-gpu-build --target install --parallel 16 -rm -rf $HOME/src/lapackpp-lumi-gpu-build +rm -rf ${build_dir}/lapackpp-lumi-gpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" \ +cmake -S ${SRC_DIR}/lapackpp \ + -B ${build_dir}/lapackpp-lumi-gpu-build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-lumi-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-lumi-gpu-build -# c-blosc (I/O compression, for OpenPMD) -if [ -d $HOME/src/c-blosc ] +# c-blosc (I/O compression, for openPMD) +if [ -d ${SRC_DIR}/c-blosc ] then - # git repository is already there - : + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.1 + cd - else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git $HOME/src/c-blosc + git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc fi -rm -rf $HOME/src/c-blosc-lu-build -cmake -S $HOME/src/c-blosc -B $HOME/src/c-blosc-lu-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/c-blosc-1.21.1 -cmake --build $HOME/src/c-blosc-lu-build --target install --parallel 16 -rm -rf $HOME/src/c-blosc-lu-build +rm -rf ${build_dir}/c-blosc-lu-build +cmake -S ${SRC_DIR}/c-blosc \ + -B ${build_dir}/c-blosc-lu-build \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DDEACTIVATE_AVX2=OFF \ + -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/c-blosc-1.21.1 +cmake --build ${build_dir}/c-blosc-lu-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-lu-build -# ADIOS2 v. 2.8.3 (for OpenPMD) -if [ -d $HOME/src/adios2 ] +# HDF5 (for openPMD) +if [ -d ${SRC_DIR}/hdf5 ] then - # git repository is already there - : + cd ${SRC_DIR}/hdf5 + git fetch --prune + git checkout hdf5-1_14_1-2 + cd - +else + git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git ${SRC_DIR}/hdf5 +fi +rm -rf ${build_dir}/hdf5-lu-build +cmake -S ${SRC_DIR}/hdf5 \ + -B ${build_dir}/hdf5-lu-build \ + -DBUILD_TESTING=OFF \ + -DHDF5_ENABLE_PARALLEL=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 +cmake --build ${build_dir}/hdf5-lu-build --target install --parallel 10 +rm -rf ${build_dir}/hdf5-lu-build + +# ADIOS2 (for openPMD) +if [ -d ${SRC_DIR}/adios2 ] +then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.8.3 + cd - else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 + git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 fi -rm -rf $HOME/src/adios2-lu-build -cmake -S $HOME/src/adios2 -B $HOME/src/adios2-lu-build -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/adios2-2.8.3 -cmake --build $HOME/src/adios2-lu-build --target install -j 16 -rm -rf $HOME/src/adios2-lu-build +rm -rf ${build_dir}/adios2-lu-build +cmake -S ${SRC_DIR}/adios2 \ + -B ${build_dir}/adios2-lu-build \ + -DADIOS2_USE_Blosc=ON \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_HDF5=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${HOME}/sw/lumi/gpu/adios2-2.8.3 +cmake --build ${build_dir}/adios2-lu-build --target install -j 16 +rm -rf ${build_dir}/adios2-lu-build # Python ###################################################################### @@ -114,9 +166,9 @@ python3 -m pip install --upgrade openpmd-api python3 -m pip install --upgrade matplotlib python3 -m pip install --upgrade yt # install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt # optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt +python3 -m pip install -r ${SRC_DIR}/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2 -#python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt +#python3 -m pip install -r ${SRC_DIR}/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/lumi-csc/submit.sh b/Tools/machines/lumi-csc/lumi.sbatch similarity index 97% rename from Tools/machines/lumi-csc/submit.sh rename to Tools/machines/lumi-csc/lumi.sbatch index d784471acd5..6d4ce1ee37f 100644 --- a/Tools/machines/lumi-csc/submit.sh +++ b/Tools/machines/lumi-csc/lumi.sbatch @@ -20,7 +20,7 @@ date export FI_MR_CACHE_MONITOR=memhooks # alternative cache monitor # Seen since August 2023 seen on OLCF (not yet seen on LUMI?) -# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors +# OLCFDEV-1597: OFI Poll Failed UNDELIVERABLE Errors # https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#olcfdev-1597-ofi-poll-failed-undeliverable-errors #export MPICH_SMP_SINGLE_COPY_MODE=NONE #export FI_CXI_RX_MATCH_MODE=software diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example index 2cb44035ce4..33aff1946c2 100644 --- a/Tools/machines/lumi-csc/lumi_warpx.profile.example +++ b/Tools/machines/lumi-csc/lumi_warpx.profile.example @@ -1,5 +1,5 @@ # please set your project account -#export proj= +#export proj="project_..." # required dependencies module load LUMI/23.09 partition/G @@ -10,23 +10,30 @@ module load buildtools/23.09 module load nano # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/lumi/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +SW_DIR="${HOME}/sw/lumi/gpu" +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load Boost/1.82.0-cpeCray-23.09 # optional: for openPMD support -module load cray-hdf5/1.12.2.7 -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export PATH=${HOME}/sw/lumi/gpu/adios2-2.8.3/bin:${PATH} +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH +export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} +export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for Python bindings or libEnsemble module load cray-python/3.10.10 +if [ -d "${SW_DIR}/venvs/warpx-lumi" ] +then + source ${SW_DIR}/venvs/warpx-lumi/bin/activate +fi + # an alias to request an interactive batch node for one hour # for paralle execution, start on the batch node: srun alias getNode="salloc -A $proj -J warpx -t 01:00:00 -p dev-g -N 1 --ntasks-per-node=8 --gpus-per-task=1 --gpu-bind=closest" @@ -50,4 +57,4 @@ export CXX=$(which amdclang++) export FC=$(which amdflang) export CFLAGS="-I${ROCM_PATH}/include" export CXXFLAGS="-I${ROCM_PATH}/include -Wno-pass-failed" -export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64" +export LDFLAGS="-L${ROCM_PATH}/lib -lamdhip64 ${PE_MPICH_GTL_DIR_amd_gfx90a} -lmpi_gtl_hsa" From d8fab99e05fd30420eab1a43b5166f45aff11fbc Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 11 Apr 2024 00:14:41 +0200 Subject: [PATCH 041/190] pybind11: v2.12.0 (#4844) Update pybinde11 to v2.12.0: https://github.com/pybind/pybind11/releases/tag/v2.12.0 --- cmake/dependencies/pybind11.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/dependencies/pybind11.cmake b/cmake/dependencies/pybind11.cmake index e65cd206f34..0a7ec260493 100644 --- a/cmake/dependencies/pybind11.cmake +++ b/cmake/dependencies/pybind11.cmake @@ -37,7 +37,7 @@ function(find_pybind11) mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11) endif() else() - find_package(pybind11 2.11.1 CONFIG REQUIRED) + find_package(pybind11 2.12.0 CONFIG REQUIRED) message(STATUS "pybind11: Found version '${pybind11_VERSION}'") endif() endfunction() @@ -52,7 +52,7 @@ option(WarpX_pybind11_internal "Download & build pybind11" ON) set(WarpX_pybind11_repo "https://github.com/pybind/pybind11.git" CACHE STRING "Repository URI to pull and build pybind11 from if(WarpX_pybind11_internal)") -set(WarpX_pybind11_branch "v2.11.1" +set(WarpX_pybind11_branch "v2.12.0" CACHE STRING "Repository branch for WarpX_pybind11_repo if(WarpX_pybind11_internal)") From 7fca90521b74f546b47ca74c7c5b596d0f38ff05 Mon Sep 17 00:00:00 2001 From: "S. Eric Clark" <25495882+clarkse@users.noreply.github.com> Date: Wed, 10 Apr 2024 16:08:29 -0700 Subject: [PATCH 042/190] Add ohms law hybrid multifab redistribution (#4848) * Initial MF redistribution for load balancing in Hybrid Ohms law solver. * Redistributing temp current multifab since it stores information between steps. * Adding rho_temp multifab to be redistruted. --- Source/Initialization/WarpXInitData.cpp | 6 ------ Source/Parallelization/WarpXRegrid.cpp | 11 +++++++++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index f0cf82e04a9..5134a3aaa10 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1297,12 +1297,6 @@ void WarpX::CheckKnownIssues() ablastr::warn_manager::WarnPriority::low); } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - !load_balance_intervals.isActivated(), - "The hybrid-PIC algorithm involves multifabs that are not yet " - "properly redistributed during load balancing events." - ); - const bool external_particle_field_used = ( mypc->m_B_ext_particle_s != "none" || mypc->m_E_ext_particle_s != "none" ); diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 7b0f135f5c8..22979912a84 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -11,6 +11,7 @@ #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "EmbeddedBoundary/WarpXFaceInfoBox.H" +#include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Initialization/ExternalField.H" #include "Particles/MultiParticleContainer.H" #include "Particles/ParticleBoundaryBuffer.H" @@ -206,6 +207,11 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(Efield_avg_fp[lev][idim], dm, true ,lev); RemakeMultiFab(Bfield_avg_fp[lev][idim], dm, true ,lev); } + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { + RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], dm, true, lev); + RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], dm, false, lev); + RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim], dm, true, lev); + } #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { RemakeMultiFab(m_edge_lengths[lev][idim], dm, false ,lev); @@ -228,6 +234,11 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi // the last step as the initial guess for the next solve RemakeMultiFab(phi_fp[lev], dm, true ,lev); + if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { + RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], dm, true, lev); + RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], dm, false, lev); + } + #ifdef AMREX_USE_EB RemakeMultiFab(m_distance_to_eb[lev], dm, false ,lev); From 2009612b0b308e2d59ac979ee05131b74f9cb78b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 10 Apr 2024 17:12:31 -0700 Subject: [PATCH 043/190] Use function to remove invalid particles without MPI exchange (#4849) * Use function to remove particles * Fix compilation * Make deleteInvalidParticles a separate function * Apply suggestions from code review --- Source/Particles/WarpXParticleContainer.H | 7 +++++++ Source/Particles/WarpXParticleContainer.cpp | 17 +++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 4b01f09e27e..61a87a12830 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -322,6 +322,13 @@ public: int nattr_int, amrex::Vector> const & attr_int, int uniqueparticles, amrex::Long id=-1); + /** Remove particles with invalid ID + * + * This is a local operation (no MPI communication) and is thus preferable over `ReDistribute` + * (which also removes invalid particles, but involves MPI communications at the same time) + */ + void deleteInvalidParticles (); + virtual void ReadHeader (std::istream& is) = 0; virtual void WriteHeader (std::ostream& os) const = 0; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index ddbfda57c71..142e973d1b2 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -302,11 +302,24 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); - // Call (local) redistribute again to remove particles with invalid ids - Redistribute(0, -1, 0, 1, true); + deleteInvalidParticles(); #endif } +void +WarpXParticleContainer::deleteInvalidParticles () { + const int nLevels = finestLevel(); + for (int lev = 0; lev <= nLevels; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) { + ParticleTileType& ptile = ParticlesAt(lev, pti); + removeInvalidParticles( ptile ); + } + } +} + /* \brief Current Deposition for thread thread_num * \param pti Particle iterator * \param wp Array of particle weights From e5b9b5247297dcf1a2765eb680c2f574f75665bb Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 10 Apr 2024 18:13:42 -0700 Subject: [PATCH 044/190] Use `BinaryCollision` class for DSMC collisions (#4744) * use `BinaryCollision` class for DSMC collisions * comment out explicit delete of `ScatteringProcess` constructor * fix first GPU compilation issue * fix RZ issue * avoid copying `binary_collision_functor` * fix merge error * do not keep `ScatteringProcess` instances in `DSMCFunc` * only keep array of `ScatteringProcess::Executor`s * fix clang-tidy errors and reset benchmark * fix default collision type issue * copy cross-section data to `DSMCFunc` * update checksum after fixing issue with cross-section arrays * move logic to set up cross-section interpolators to `DSMCFunc.cpp` file * use internal `struct`s in collision filter functors * clean-up and fixing clang-tidy issues * second clang-tidy fix attempt * Only call redistribute if collision type has product species * fix clang-tidy error * another clang-tidy error fix * Appropriately set `local` parameter in `Redistribute` call. * set `Redistribute` arguments appropriately * check if weight is numerically zero * Apply suggestions from code review Co-authored-by: Remi Lehe * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add docstring for `CollisionPairFilter` function arguments * Use `insert` rather than `push_back` & `rotate` Co-authored-by: Remi Lehe --------- Co-authored-by: Remi Lehe Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../capacitive_discharge/PICMI_inputs_1d.py | 5 +- .../BinaryCollision/BinaryCollision.H | 47 +- .../BinaryCollision/BinaryCollisionUtils.H | 2 + .../BinaryCollision/BinaryCollisionUtils.cpp | 220 +++++----- .../Coulomb/PairWiseCoulombCollisionFunc.H | 70 +-- .../BinaryCollision/DSMC/CMakeLists.txt | 3 +- .../DSMC/CollisionFilterFunc.H | 190 ++------- .../Collision/BinaryCollision/DSMC/DSMC.H | 86 ---- .../Collision/BinaryCollision/DSMC/DSMC.cpp | 300 ------------- .../Collision/BinaryCollision/DSMC/DSMCFunc.H | 197 +++++++++ .../BinaryCollision/DSMC/DSMCFunc.cpp | 78 ++++ .../BinaryCollision/DSMC/Make.package | 3 +- .../DSMC/SplitAndScatterFunc.H | 403 +++++++++++------- .../DSMC/SplitAndScatterFunc.cpp | 43 ++ .../NuclearFusion/NuclearFusionFunc.H | 122 +++--- .../Particles/Collision/CollisionHandler.cpp | 16 +- 16 files changed, 890 insertions(+), 895 deletions(-) delete mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H delete mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp create mode 100644 Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index d54800debd6..0b59516ccc7 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -183,11 +183,10 @@ def __init__(self, n=0, test=False, pythonsolver=False, dsmc=False): self.max_steps = 50 self.diag_steps = 5 self.mcc_subcycling_steps = 2 + self.rng = np.random.default_rng(23094290) else: self.mcc_subcycling_steps = None - - if self.dsmc: - self.rng = np.random.default_rng(23094290) + self.rng = np.random.default_rng() self.ion_density_array = np.zeros(self.nz + 1) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index a9e081d44b4..b36c28db662 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -8,6 +8,7 @@ #define WARPX_PARTICLES_COLLISION_BINARYCOLLISION_H_ #include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" +#include "Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" #include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" @@ -58,14 +59,14 @@ /** * \brief This class performs generic binary collisions. * - * \tparam CollisionFunctorType the type of the specific binary collision functor that acts on a + * \tparam CollisionFunctor the specific binary collision functor that acts on a * single cell - * \tparam CopyTransformFunctorType the type of the second functor used in the case of + * \tparam CopyTransformFunctor the second functor used in the case of * particle creation * */ -template +template class BinaryCollision final : public CollisionBase { @@ -92,18 +93,27 @@ public: WARPX_ABORT_WITH_MESSAGE("Binary collision " + collision_name + " must have exactly two species."); } + const CollisionType collision_type = BinaryCollisionUtils::get_collision_type(collision_name, mypc); + m_isSameSpecies = (m_species_names[0] == m_species_names[1]); - m_binary_collision_functor = CollisionFunctorType(collision_name, mypc, m_isSameSpecies); + m_binary_collision_functor = CollisionFunctor(collision_name, mypc, m_isSameSpecies); const amrex::ParmParse pp_collision_name(collision_name); pp_collision_name.queryarr("product_species", m_product_species); + + // if DSMC the colliding species are also product species + // Therefore, we insert the colliding species at the beginning of `m_product_species` + if (collision_type == CollisionType::DSMC) { + m_product_species.insert( m_product_species.begin(), m_species_names.begin(), m_species_names.end() ); + } m_have_product_species = !m_product_species.empty(); - if ((std::is_same::value) & (m_have_product_species)) { + + if ((std::is_same::value) && (m_have_product_species)) { WARPX_ABORT_WITH_MESSAGE( "Binary collision " + collision_name + " does not produce species. Thus, `product_species` should not be specified in the input script." ); } - m_copy_transform_functor = CopyTransformFunctorType(collision_name, mypc); + m_copy_transform_functor = CopyTransformFunctor(collision_name, mypc); } ~BinaryCollision () override = default; @@ -111,7 +121,6 @@ public: BinaryCollision ( BinaryCollision const &) = default; BinaryCollision& operator= ( BinaryCollision const & ) = default; - BinaryCollision ( BinaryCollision&& ) = delete; BinaryCollision& operator= ( BinaryCollision&& ) = delete; @@ -157,8 +166,8 @@ public: amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, copy_species2.begin(), copy_species2.end(), device_copy_species2.begin()); amrex::Gpu::streamSynchronize(); - auto copy_species1_data = device_copy_species1.data(); - auto copy_species2_data = device_copy_species2.data(); + auto *copy_species1_data = device_copy_species1.data(); + auto *copy_species2_data = device_copy_species2.data(); #else auto *copy_species1_data = copy_species1.data(); auto *copy_species2_data = copy_species2.data(); @@ -190,7 +199,7 @@ public: auto wt = static_cast(amrex::second()); doCollisionsWithinTile( dt, lev, mfi, species1, species2, product_species_vector, - copy_species1_data, copy_species2_data); + copy_species1_data, copy_species2_data); if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { @@ -199,6 +208,15 @@ public: amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); } } + + if (m_have_product_species) { + // The fact that there are product species indicates that particles of + // the colliding species (`species1` and `species2`) may be removed + // (i.e., marked as invalid) in the process of creating new product particles. + // Call (local) redistribute to remove invalid particles + species1.Redistribute(lev, lev, 0, 1, true); + species2.Redistribute(lev, lev, 0, 1, true); + } } } @@ -224,7 +242,7 @@ public: using namespace ParticleUtils; using namespace amrex::literals; - CollisionFunctorType binary_collision_functor = m_binary_collision_functor; + const auto& binary_collision_functor = m_binary_collision_functor.executor(); const bool have_product_species = m_have_product_species; // Store product species data in vectors @@ -522,7 +540,6 @@ public: } ); - // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, @@ -550,9 +567,9 @@ private: bool m_have_product_species; amrex::Vector m_product_species; // functor that performs collisions within a cell - CollisionFunctorType m_binary_collision_functor; + CollisionFunctor m_binary_collision_functor; // functor that creates new particles and initializes their parameters - CopyTransformFunctorType m_copy_transform_functor; + CopyTransformFunctor m_copy_transform_functor; }; diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index f6aa8e6e303..fdc88a275a0 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -19,6 +19,8 @@ enum struct CollisionType { DeuteriumTritiumToNeutronHeliumFusion, DeuteriumDeuteriumToNeutronHeliumFusion, DeuteriumHeliumToProtonHeliumFusion, ProtonBoronToAlphasFusion, + DSMC, + PairwiseCoulomb, Undefined }; enum struct NuclearFusionType { diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp index 812be089462..05756583554 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.cpp @@ -17,123 +17,129 @@ namespace BinaryCollisionUtils{ + CollisionType get_collision_type (const std::string& collision_name, + MultiParticleContainer const * const mypc) + { + const amrex::ParmParse pp_collision_name(collision_name); + // For legacy, pairwisecoulomb is the default + std::string type = "pairwisecoulomb"; + pp_collision_name.query("type", type); + if (type == "pairwisecoulomb") { + return CollisionType::PairwiseCoulomb; + } + else if (type == "nuclearfusion") { + const NuclearFusionType fusion_type = get_nuclear_fusion_type(collision_name, mypc); + return nuclear_fusion_type_to_collision_type(fusion_type); + } + else if (type == "dsmc") { + return CollisionType::DSMC; + } + return CollisionType::Undefined; + } + NuclearFusionType get_nuclear_fusion_type (const std::string& collision_name, MultiParticleContainer const * const mypc) - { - const amrex::ParmParse pp_collision_name(collision_name); - amrex::Vector species_names; - pp_collision_name.getarr("species", species_names); - auto& species1 = mypc->GetParticleContainerFromName(species_names[0]); - auto& species2 = mypc->GetParticleContainerFromName(species_names[1]); - amrex::Vector product_species_name; - pp_collision_name.getarr("product_species", product_species_name); + { + const amrex::ParmParse pp_collision_name(collision_name); + amrex::Vector species_names; + pp_collision_name.getarr("species", species_names); + auto& species1 = mypc->GetParticleContainerFromName(species_names[0]); + auto& species2 = mypc->GetParticleContainerFromName(species_names[1]); + amrex::Vector product_species_name; + pp_collision_name.getarr("product_species", product_species_name); - if ((species1.AmIA() && species2.AmIA()) - || - (species1.AmIA() && species2.AmIA()) - ) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 2u, - "ERROR: Deuterium-tritium fusion must contain exactly two product species"); - auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); - auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (product_species1.AmIA() && product_species2.AmIA()) - || - (product_species1.AmIA() && product_species2.AmIA()), - "ERROR: Product species of deuterium-tritium fusion must be of type neutron and helium4"); - return NuclearFusionType::DeuteriumTritiumToNeutronHelium; - } - else if (species1.AmIA() && species2.AmIA()) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 2u, - "ERROR: Deuterium-deuterium fusion must contain exactly two product species"); - auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); - auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); - if ( - (product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA())){ - return NuclearFusionType::DeuteriumDeuteriumToNeutronHelium; - } else if ( - (product_species1.AmIA() && product_species2.AmIA()) - ||(product_species1.AmIA() && product_species2.AmIA())){ - return NuclearFusionType::DeuteriumDeuteriumToProtonTritium; - } else { - WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or hydrogen3 and hydrogen1"); - } - } - else if ((species1.AmIA() && species2.AmIA()) + if ((species1.AmIA() && species2.AmIA()) + || + (species1.AmIA() && species2.AmIA()) + ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 2u, + "ERROR: Deuterium-tritium fusion must contain exactly two product species"); + auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); + auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (product_species1.AmIA() && product_species2.AmIA()) || - (species1.AmIA() && species2.AmIA()) - ) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 2u, - "ERROR: Deuterium-helium fusion must contain exactly two product species"); - auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); - auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - (product_species1.AmIA() && product_species2.AmIA()) - || - (product_species1.AmIA() && product_species2.AmIA()), - "ERROR: Product species of deuterium-helium fusion must be of type hydrogen1 and helium4"); - return NuclearFusionType::DeuteriumHeliumToProtonHelium; + (product_species1.AmIA() && product_species2.AmIA()), + "ERROR: Product species of deuterium-tritium fusion must be of type neutron and helium4"); + return NuclearFusionType::DeuteriumTritiumToNeutronHelium; + } + else if (species1.AmIA() && species2.AmIA()) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 2u, + "ERROR: Deuterium-deuterium fusion must contain exactly two product species"); + auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); + auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); + if ( + (product_species1.AmIA() && product_species2.AmIA()) + ||(product_species1.AmIA() && product_species2.AmIA())){ + return NuclearFusionType::DeuteriumDeuteriumToNeutronHelium; + } else if ( + (product_species1.AmIA() && product_species2.AmIA()) + ||(product_species1.AmIA() && product_species2.AmIA())){ + return NuclearFusionType::DeuteriumDeuteriumToProtonTritium; + } else { + WARPX_ABORT_WITH_MESSAGE("ERROR: Product species of deuterium-deuterium fusion must be of type helium3 and neutron, or hydrogen3 and hydrogen1"); } - else if ((species1.AmIA() && species2.AmIA()) + } + else if ((species1.AmIA() && species2.AmIA()) + || + (species1.AmIA() && species2.AmIA()) + ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 2u, + "ERROR: Deuterium-helium fusion must contain exactly two product species"); + auto& product_species1 = mypc->GetParticleContainerFromName(product_species_name[0]); + auto& product_species2 = mypc->GetParticleContainerFromName(product_species_name[1]); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + (product_species1.AmIA() && product_species2.AmIA()) || - (species1.AmIA() && species2.AmIA()) - ) - { - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species_name.size() == 1, - "ERROR: Proton-boron must contain exactly one product species"); - auto& product_species = mypc->GetParticleContainerFromName(product_species_name[0]); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - product_species.AmIA(), - "ERROR: Product species of proton-boron fusion must be of type helium4"); - return NuclearFusionType::ProtonBoronToAlphas; - } - WARPX_ABORT_WITH_MESSAGE("Binary nuclear fusion not implemented between species " + - species_names[0] + " of type " + species1.getSpeciesTypeName() + - " and species " + species_names[1] + " of type " + - species2.getSpeciesTypeName()); - return NuclearFusionType::Undefined; + (product_species1.AmIA() && product_species2.AmIA()), + "ERROR: Product species of deuterium-helium fusion must be of type hydrogen1 and helium4"); + return NuclearFusionType::DeuteriumHeliumToProtonHelium; } - - CollisionType get_collision_type (const std::string& collision_name, - MultiParticleContainer const * const mypc) + else if ((species1.AmIA() && species2.AmIA()) + || + (species1.AmIA() && species2.AmIA()) + ) { - const amrex::ParmParse pp_collision_name(collision_name); - std::string type; - pp_collision_name.get("type", type); - if (type == "nuclearfusion") { - const NuclearFusionType fusion_type = get_nuclear_fusion_type(collision_name, mypc); - return nuclear_fusion_type_to_collision_type(fusion_type); - } - WARPX_ABORT_WITH_MESSAGE(type + " is not a valid type of collision that creates new particles"); - return CollisionType::Undefined; + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species_name.size() == 1, + "ERROR: Proton-boron must contain exactly one product species"); + auto& product_species = mypc->GetParticleContainerFromName(product_species_name[0]); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + product_species.AmIA(), + "ERROR: Product species of proton-boron fusion must be of type helium4"); + return NuclearFusionType::ProtonBoronToAlphas; } + WARPX_ABORT_WITH_MESSAGE("Binary nuclear fusion not implemented between species " + + species_names[0] + " of type " + species1.getSpeciesTypeName() + + " and species " + species_names[1] + " of type " + + species2.getSpeciesTypeName()); + return NuclearFusionType::Undefined; + } CollisionType nuclear_fusion_type_to_collision_type (const NuclearFusionType fusion_type) - { - if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) { - return CollisionType::DeuteriumTritiumToNeutronHeliumFusion; - } - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) { - return CollisionType::DeuteriumDeuteriumToProtonTritiumFusion; - } - if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) { - return CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion; - } - if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) { - return CollisionType::DeuteriumHeliumToProtonHeliumFusion; - } - if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { - return CollisionType::ProtonBoronToAlphasFusion; - } - WARPX_ABORT_WITH_MESSAGE("Invalid nuclear fusion type"); - return CollisionType::Undefined; + { + if (fusion_type == NuclearFusionType::DeuteriumTritiumToNeutronHelium) { + return CollisionType::DeuteriumTritiumToNeutronHeliumFusion; + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToProtonTritium) { + return CollisionType::DeuteriumDeuteriumToProtonTritiumFusion; + } + if (fusion_type == NuclearFusionType::DeuteriumDeuteriumToNeutronHelium) { + return CollisionType::DeuteriumDeuteriumToNeutronHeliumFusion; + } + if (fusion_type == NuclearFusionType::DeuteriumHeliumToProtonHelium) { + return CollisionType::DeuteriumHeliumToProtonHeliumFusion; + } + if (fusion_type == NuclearFusionType::ProtonBoronToAlphas) { + return CollisionType::ProtonBoronToAlphasFusion; } + WARPX_ABORT_WITH_MESSAGE("Invalid nuclear fusion type"); + return CollisionType::Undefined; + } } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index d1a92ee9716..b31fa3922b9 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -58,37 +58,41 @@ public: utils::parser::queryWithParser( pp_collision_name, "CoulombLog", CoulombLog); m_CoulombLog = CoulombLog; + + m_exe.m_CoulombLog = m_CoulombLog; + m_exe.m_isSameSpecies = m_isSameSpecies; } - /** - * \brief operator() of the PairWiseCoulombCollisionFunc class. Performs Coulomb collisions - * at the cell level by calling ElasticCollisionPerez. - * - * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). - * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). - * @param[in] I1,I2 index arrays. They determine all elements that will be used. - * @param[in,out] soa_1,soa_2 contain the struct of array data of the two species. - * @param[in] q1,q2 are charges. - * @param[in] m1,m2 are masses. - * @param[in] dt is the time step length between two collision calls. - * @param[in] dV is the volume of the corresponding cell. - * @param[in] engine the random engine. - */ - AMREX_GPU_HOST_DEVICE AMREX_INLINE - void operator() ( - index_type const I1s, index_type const I1e, - index_type const I2s, index_type const I2e, - index_type const* AMREX_RESTRICT I1, - index_type const* AMREX_RESTRICT I2, - SoaData_type soa_1, SoaData_type soa_2, - GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, - amrex::ParticleReal const q1, amrex::ParticleReal const q2, - amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, - index_type const /*cell_start_pair*/, index_type* /*p_mask*/, - index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, - amrex::ParticleReal* /*p_pair_reaction_weight*/, - amrex::RandomEngine const& engine) const + struct Executor { + /** + * \brief Executor of the PairWiseCoulombCollisionFunc class. Performs Coulomb collisions + * at the cell level by calling ElasticCollisionPerez. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in,out] soa_1,soa_2 contain the struct of array data of the two species. + * @param[in] q1,q2 are charges. + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] engine the random engine. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void operator() ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + const SoaData_type& soa_1, const SoaData_type& soa_2, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const q1, amrex::ParticleReal const q2, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, + index_type const /*cell_start_pair*/, index_type* /*p_mask*/, + index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, + amrex::ParticleReal* /*p_pair_reaction_weight*/, + amrex::RandomEngine const& engine) const { using namespace amrex::literals; @@ -99,9 +103,17 @@ public: dt, m_CoulombLog, dV, engine, m_isSameSpecies); } + amrex::ParticleReal m_CoulombLog; + bool m_isSameSpecies; + }; + + [[nodiscard]] Executor const& executor () const { return m_exe; } + private: amrex::ParticleReal m_CoulombLog; bool m_isSameSpecies; + + Executor m_exe; }; #endif // PAIRWISE_COULOMB_COLLISION_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt index 3575d53ad29..60da82ee48f 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CMakeLists.txt @@ -2,6 +2,7 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) target_sources(lib_${SD} PRIVATE - DSMC.cpp + DSMCFunc.cpp + SplitAndScatterFunc.cpp ) endforeach() diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H index 64f05650d1f..bc11e5e4a99 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -18,59 +18,69 @@ * \brief This function determines whether a collision occurs for a given * pair of particles. * - * @param[in] u1x,u1y,u1z momenta of the first colliding particle - * @param[in] u2x,u2y,u2z momenta of the second colliding particle - * @param[in] m1,m2 masses - * @param[in] w1,w2 effective weight of the colliding particles + * @param[in] u1x,u1y,u1z momenta of the first colliding particle. + * @param[in] u2x,u2y,u2z momenta of the second colliding particle. + * @param[in] m1,m2 masses. + * @param[in] w1,w2 effective weight of the colliding particles. * @param[in] dt is the time step length between two collision calls. * @param[in] dV is the volume of the corresponding cell. - * @param[in] pair_index is the index of the colliding pair + * @param[in] pair_index is the index of the colliding pair. * @param[out] p_mask is a mask that will be set to a non-zero integer if a * collision occurs. The integer encodes the scattering process. - * @param[out] p_pair_reaction_weight stores the weight of the product particles - * @param[in] process_count number of scattering processes to consider - * @param[in] scattering processes an array of scattering processes included for consideration + * @param[out] p_pair_reaction_weight stores the weight of the product particles. + * @param[in] multiplier factor by which the collision probability is increased to + * account for all other possible binary collision partners. + * @param[in] process_count number of scattering processes to consider. + * @param[in] scattering processes an array of scattering processes included for consideration. * @param[in] engine the random engine. */ template AMREX_GPU_HOST_DEVICE AMREX_INLINE -void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleReal& u1y, - const amrex::ParticleReal& u1z, const amrex::ParticleReal& u2x, - const amrex::ParticleReal& u2y, const amrex::ParticleReal& u2z, - const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, - amrex::ParticleReal w1, amrex::ParticleReal w2, - const amrex::Real& dt, const amrex::ParticleReal& dV, const int& pair_index, +void CollisionPairFilter (const amrex::ParticleReal u1x, const amrex::ParticleReal u1y, + const amrex::ParticleReal u1z, const amrex::ParticleReal u2x, + const amrex::ParticleReal u2y, const amrex::ParticleReal u2z, + const amrex::ParticleReal m1, const amrex::ParticleReal m2, + const amrex::ParticleReal w1, const amrex::ParticleReal w2, + const amrex::Real dt, const amrex::ParticleReal dV, const int pair_index, index_type* AMREX_RESTRICT p_mask, amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, - const int& multiplier_ratio, - int const process_count, - ScatteringProcess::Executor* scattering_processes, + const int multiplier, + const int process_count, + const ScatteringProcess::Executor* scattering_processes, const amrex::RandomEngine& engine) { - using namespace amrex::literals; - amrex::ParticleReal E_coll, v_coll, lab_to_COM_factor; - const amrex::ParticleReal w_min = amrex::min(w1, w2); - const amrex::ParticleReal w_max = amrex::max(w1, w2); - BinaryCollisionUtils::get_collision_parameters( u1x, u1y, u1z, u2x, u2y, u2z, m1, m2, E_coll, v_coll, lab_to_COM_factor); - // convert E_coll to eV + using namespace amrex::literals; + + const amrex::ParticleReal w_min = amrex::min(w1, w2); + const amrex::ParticleReal w_max = amrex::max(w1, w2); + + // convert E_coll from Joule to eV E_coll /= PhysConst::q_e; - amrex::ParticleReal sigma_tot = 0._prt; + // Evaluate the cross-section for each scattering process to determine + // the total collision probability. + AMREX_ASSERT_WITH_MESSAGE( + (process_count < 4), "Too many scattering processes in DSMC routine." + ); + int coll_type[4] = {0, 0, 0, 0}; + amrex::ParticleReal sigma_sums[4] = {0._prt, 0._prt, 0._prt, 0._prt}; for (int ii = 0; ii < process_count; ii++) { - auto const& scattering_process = *(scattering_processes + ii); - sigma_tot += scattering_process.getCrossSection(E_coll); + auto const& scattering_process = scattering_processes[ii]; + coll_type[ii] = int(scattering_process.m_type); + const amrex::ParticleReal sigma = scattering_process.getCrossSection(E_coll); + sigma_sums[ii] = sigma + ((ii == 0) ? 0._prt : sigma_sums[ii-1]); } + const auto sigma_tot = sigma_sums[process_count-1]; // calculate total collision probability const amrex::ParticleReal exponent = ( - lab_to_COM_factor * multiplier_ratio * w_max - * sigma_tot * v_coll * dt / dV + lab_to_COM_factor * multiplier * w_max * sigma_tot * v_coll * dt / dV ); // Compute actual collision probability that is always between zero and one @@ -84,13 +94,10 @@ void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleR if (amrex::Random(engine) < probability) { const amrex::ParticleReal random_number = amrex::Random(engine); - amrex::ParticleReal sigma = 0._prt; for (int ii = 0; ii < process_count; ii++) { - auto const& scattering_process = *(scattering_processes + ii); - sigma += scattering_process.getCrossSection(E_coll); - if (random_number <= sigma / sigma_tot) + if (random_number <= sigma_sums[ii] / sigma_tot) { - p_mask[pair_index] = int(scattering_process.m_type); + p_mask[pair_index] = coll_type[ii]; p_pair_reaction_weight[pair_index] = w_min; break; } @@ -102,121 +109,4 @@ void CollisionPairFilter (const amrex::ParticleReal& u1x, const amrex::ParticleR } } -/** - * \brief Function that determines if a collision occurs and if so, what - * type. - * - * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). - * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). - * @param[in] I1,I2 index arrays. They determine all elements that will be used. - * @param[in] ptd_1,ptd_2 contain the particle data of the two species - * @param[in] m1,m2 are masses. - * @param[in] dt is the time step length between two collision calls. - * @param[in] dV is the volume of the corresponding cell. - * @param[in] cell_start_pair is the start index of the pairs in that cell. - * @param[out] p_mask is a mask that will be set to a non-zero integer if a - * collision occurs. The integer encodes the scattering process. - * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the - * particles of a given pair. They are only needed here to store information that will be used - * later on when actually creating the product particles. - * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only - * needed here to store information that will be used later on when actually creating the - * product particles. - * @param[in] process_count number of scattering processes to consider - * @param[in] scattering processes an array of scattering processes included for consideration - * @param[in] engine the random engine. - */ -template -AMREX_GPU_HOST_DEVICE AMREX_INLINE -void CollisionFilter ( - index_type const I1s, index_type const I1e, - index_type const I2s, index_type const I2e, - index_type const* AMREX_RESTRICT I1, - index_type const* AMREX_RESTRICT I2, - PData ptd_1, PData ptd_2, - amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, - index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, - index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, - amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, - int const process_count, - ScatteringProcess::Executor* scattering_processes, - amrex::RandomEngine const& engine) -{ - - amrex::ParticleReal * const AMREX_RESTRICT w1 = ptd_1.m_rdata[PIdx::w]; - amrex::ParticleReal * const AMREX_RESTRICT u1x = ptd_1.m_rdata[PIdx::ux]; - amrex::ParticleReal * const AMREX_RESTRICT u1y = ptd_1.m_rdata[PIdx::uy]; - amrex::ParticleReal * const AMREX_RESTRICT u1z = ptd_1.m_rdata[PIdx::uz]; - - amrex::ParticleReal * const AMREX_RESTRICT w2 = ptd_2.m_rdata[PIdx::w]; - amrex::ParticleReal * const AMREX_RESTRICT u2x = ptd_2.m_rdata[PIdx::ux]; - amrex::ParticleReal * const AMREX_RESTRICT u2y = ptd_2.m_rdata[PIdx::uy]; - amrex::ParticleReal * const AMREX_RESTRICT u2z = ptd_2.m_rdata[PIdx::uz]; - - // Number of macroparticles of each species - const int NI1 = I1e - I1s; - const int NI2 = I2e - I2s; - const int max_N = amrex::max(NI1,NI2); - - int i1 = I1s; - int i2 = I2s; - int pair_index = cell_start_pair; - - // Because the number of particles of each species is not always equal (NI1 != NI2 - // in general), some macroparticles will be paired with multiple macroparticles of the - // other species and we need to decrease their weight accordingly. - // c1 corresponds to the minimum number of times a particle of species 1 will be paired - // with a particle of species 2. Same for c2. - const int c1 = amrex::max(NI2/NI1,1); - const int c2 = amrex::max(NI1/NI2,1); - -#if (defined WARPX_DIM_RZ) - amrex::ParticleReal * const AMREX_RESTRICT theta1 = ptd_1.m_rdata[PIdx::theta]; - amrex::ParticleReal * const AMREX_RESTRICT theta2 = ptd_2.m_rdata[PIdx::theta]; -#endif - - for (int k = 0; k < max_N; ++k) - { - // c1k : how many times the current particle of species 1 is paired with a particle - // of species 2. Same for c2k. - const int c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; - const int c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; - -#if (defined WARPX_DIM_RZ) - /* In RZ geometry, macroparticles can collide with other macroparticles - * in the same *cylindrical* cell. For this reason, collisions between macroparticles - * are actually not local in space. In this case, the underlying assumption is that - * particles within the same cylindrical cell represent a cylindrically-symmetry - * momentum distribution function. Therefore, here, we temporarily rotate the - * momentum of one of the macroparticles in agreement with this cylindrical symmetry. - * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; - * there is a corresponding assert statement at initialization.) */ - amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; - amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; - u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); - u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); -#endif - - CollisionPairFilter( - u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], - u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], - m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, - dt, dV, pair_index, p_mask, p_pair_reaction_weight, - max_N, process_count, scattering_processes, engine); - -#if (defined WARPX_DIM_RZ) - amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; - u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); - u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); -#endif - - p_pair_indices_1[pair_index] = I1[i1]; - p_pair_indices_2[pair_index] = I2[i2]; - ++i1; if ( i1 == static_cast(I1e) ) { i1 = I1s; } - ++i2; if ( i2 == static_cast(I2e) ) { i2 = I2s; } - ++pair_index; - } -} - #endif // COLLISION_FILTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H deleted file mode 100644 index aa656a3d92e..00000000000 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.H +++ /dev/null @@ -1,86 +0,0 @@ -/* Copyright 2023 The WarpX Community - * - * This file is part of WarpX. - * - * Authors: Roelof Groenewald (TAE Technologies) - * - * License: BSD-3-Clause-LBNL - */ -#ifndef DSMC_H_ -#define DSMC_H_ - -#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" -#include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" -#include "Particles/Collision/CollisionBase.H" -#include "Particles/Collision/ScatteringProcess.H" -#include "Particles/MultiParticleContainer.H" -#include "Particles/ParticleCreation/SmartCopy.H" -#include "Particles/ParticleCreation/SmartUtils.H" -#include "Particles/WarpXParticleContainer.H" -#include "Utils/Parser/ParserUtils.H" -#include "Utils/ParticleUtils.H" -#include "Utils/WarpXProfilerWrapper.H" - -#include -#include -#include - - -/** - * \brief This class performs DSMC (direct simulation Monte Carlo) collisions - * within a cell. Particles are paired up and for each pair a stochastic process - * determines whether a collision occurs. The algorithm is similar to the one - * used for binary Coulomb collisions and the nuclear fusion module. - */ -class DSMC final - : public CollisionBase -{ - // Define shortcuts for frequently-used type names - using ParticleType = WarpXParticleContainer::ParticleType; - using ParticleTileType = WarpXParticleContainer::ParticleTileType; - using ParticleTileDataType = ParticleTileType::ParticleTileDataType; - using ParticleBins = amrex::DenseBins; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - using index_type = ParticleBins::index_type; - -public: - - /** - * \brief Constructor of the DSMC class - * - * @param[in] collision_name the name of the collision - */ - DSMC (const std::string& collision_name); - - /** Perform the collisions - * - * @param cur_time Current time - * @param dt Time step size - * @param mypc Container of species involved - * - */ - void doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) override; - - /** Perform all binary collisions within a tile - * - * \param[in] lev the mesh-refinement level - * \param[in] mfi iterator for multifab - * \param species_1 first species container - * \param species_2 second species container - * \param copy_species1 SmartCopy for species_1 - * \param copy_species2 SmartCopy for species_2 - * - */ - void doCollisionsWithinTile ( - amrex::Real dt, int lev, amrex::MFIter const& mfi, - WarpXParticleContainer& species_1, - WarpXParticleContainer& species_2, - SmartCopy& copy_species1, - SmartCopy& copy_species2 ); - -private: - amrex::Vector m_scattering_processes; - amrex::Gpu::DeviceVector m_scattering_processes_exe; -}; - -#endif // DSMC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp deleted file mode 100644 index 8f309c3610a..00000000000 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMC.cpp +++ /dev/null @@ -1,300 +0,0 @@ -/* Copyright 2023 The WarpX Community - * - * This file is part of WarpX. - * - * Authors: Roelof Groenewald (TAE Technologies) - * - * License: BSD-3-Clause-LBNL - */ -#include "CollisionFilterFunc.H" -#include "DSMC.H" -#include "SplitAndScatterFunc.H" - - -DSMC::DSMC (const std::string& collision_name) - : CollisionBase{collision_name} -{ - using namespace amrex::literals; - const amrex::ParmParse pp_collision_name(collision_name); - -#if defined WARPX_DIM_RZ - amrex::Abort("DSMC collisions are only implemented for Cartesian coordinates."); -#endif - - if(m_species_names.size() != 2) - { - amrex::Abort("DSMC collision " + collision_name + " must have exactly two species."); - } - - // query for a list of collision processes - // these could be elastic, excitation, charge_exchange, back, etc. - amrex::Vector scattering_process_names; - pp_collision_name.queryarr("scattering_processes", scattering_process_names); - - // create a vector of ScatteringProcess objects from each scattering - // process name - for (const auto& scattering_process : scattering_process_names) { - const std::string kw_cross_section = scattering_process + "_cross_section"; - std::string cross_section_file; - pp_collision_name.query(kw_cross_section.c_str(), cross_section_file); - - // if the scattering process is excitation or ionization get the - // energy associated with that process - amrex::ParticleReal energy = 0._prt; - if (scattering_process.find("excitation") != std::string::npos || - scattering_process.find("ionization") != std::string::npos) { - const std::string kw_energy = scattering_process + "_energy"; - utils::parser::getWithParser( - pp_collision_name, kw_energy.c_str(), energy); - } - - ScatteringProcess process(scattering_process, cross_section_file, energy); - - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, - "Cannot add an unknown scattering process type"); - - m_scattering_processes.push_back(std::move(process)); - } - -#ifdef AMREX_USE_GPU - amrex::Gpu::HostVector h_scattering_processes_exe; - for (auto const& p : m_scattering_processes) { - h_scattering_processes_exe.push_back(p.executor()); - } - m_scattering_processes_exe.resize(h_scattering_processes_exe.size()); - amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scattering_processes_exe.begin(), - h_scattering_processes_exe.end(), m_scattering_processes_exe.begin()); - amrex::Gpu::streamSynchronize(); -#else - for (auto const& p : m_scattering_processes) { - m_scattering_processes_exe.push_back(p.executor()); - } -#endif -} - -void -DSMC::doCollisions (amrex::Real /*cur_time*/, amrex::Real dt, MultiParticleContainer* mypc) -{ - WARPX_PROFILE("DSMC::doCollisions()"); - - auto& species1 = mypc->GetParticleContainerFromName(m_species_names[0]); - auto& species2 = mypc->GetParticleContainerFromName(m_species_names[1]); - - // SmartCopy objects are created that will facilitate the particle splitting - // operation involved in DSMC collisions between particles with arbitrary - // weights. - const auto copy_factory_species1 = SmartCopyFactory{species1, species1}; - const auto copy_factory_species2 = SmartCopyFactory{species2, species2}; - auto copy_species1 = copy_factory_species1.getSmartCopy(); - auto copy_species2 = copy_factory_species2.getSmartCopy(); - - species1.defineAllParticleTiles(); - species2.defineAllParticleTiles(); - - // Enable tiling - amrex::MFItInfo info; - if (amrex::Gpu::notInLaunchRegion()) { info.EnableTiling(WarpXParticleContainer::tile_size); } - - // Loop over refinement levels - for (int lev = 0; lev <= species1.finestLevel(); ++lev){ - - amrex::LayoutData* cost = WarpX::getCosts(lev); - - // Loop over all grids/tiles at this level -#ifdef AMREX_USE_OMP - info.SetDynamic(true); -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for (amrex::MFIter mfi = species1.MakeMFIter(lev, info); mfi.isValid(); ++mfi){ - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - amrex::Gpu::synchronize(); - } - auto wt = static_cast(amrex::second()); - - doCollisionsWithinTile(dt, lev, mfi, species1, species2, - copy_species1, copy_species2); - - if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) - { - amrex::Gpu::synchronize(); - wt = static_cast(amrex::second()) - wt; - amrex::HostDevice::Atomic::Add( &(*cost)[mfi.index()], wt); - } - } - - // Call redistribute to remove particles with negative ids - species1.Redistribute(lev, lev, 0, true, true); - species2.Redistribute(lev, lev, 0, true, true); - } -} - -void -DSMC::doCollisionsWithinTile( - amrex::Real dt, int const lev, amrex::MFIter const& mfi, - WarpXParticleContainer& species_1, - WarpXParticleContainer& species_2, - SmartCopy& copy_species1, - SmartCopy& copy_species2) -{ - using namespace ParticleUtils; - using namespace amrex::literals; - - // get collision processes - auto *scattering_processes = m_scattering_processes_exe.data(); - int const process_count = static_cast(m_scattering_processes_exe.size()); - - // Extract particles in the tile that `mfi` points to - ParticleTileType& ptile_1 = species_1.ParticlesAt(lev, mfi); - ParticleTileType& ptile_2 = species_2.ParticlesAt(lev, mfi); - - // Find the particles that are in each cell of this tile - ParticleBins bins_1 = findParticlesInEachCell( lev, mfi, ptile_1 ); - ParticleBins bins_2 = findParticlesInEachCell( lev, mfi, ptile_2 ); - - // Extract low-level data - int const n_cells = static_cast(bins_1.numBins()); - - // - Species 1 - index_type* AMREX_RESTRICT indices_1 = bins_1.permutationPtr(); - index_type const* AMREX_RESTRICT cell_offsets_1 = bins_1.offsetsPtr(); - const amrex::ParticleReal m1 = species_1.getMass(); - const auto ptd_1 = ptile_1.getParticleTileData(); - - // - Species 2 - index_type* AMREX_RESTRICT indices_2 = bins_2.permutationPtr(); - index_type const* AMREX_RESTRICT cell_offsets_2 = bins_2.offsetsPtr(); - const amrex::ParticleReal m2 = species_2.getMass(); - const auto ptd_2 = ptile_2.getParticleTileData(); - - amrex::Geometry const& geom = WarpX::GetInstance().Geom(lev); -#if defined WARPX_DIM_1D_Z - auto dV = geom.CellSize(0); -#elif defined WARPX_DIM_XZ - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined WARPX_DIM_RZ - amrex::Box const& cbx = mfi.tilebox(amrex::IntVect::TheZeroVector()); //Cell-centered box - const auto lo = lbound(cbx); - const auto hi = ubound(cbx); - const int nz = hi.y-lo.y+1; - auto dr = geom.CellSize(0); - auto dz = geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif - - // In the following we set up the "mask" used for creating new particles - // (from splitting events). There is a mask index for every collision - // pair. Below we find the size of the mask based on the greater of the - // number of each species' particle in each cell. - amrex::Gpu::DeviceVector n_pairs_in_each_cell(n_cells); - index_type* AMREX_RESTRICT p_n_pairs_in_each_cell = n_pairs_in_each_cell.dataPtr(); - - // Compute how many pairs in each cell and store in n_pairs_in_each_cell array - // For different species, the number of pairs in a cell is the number of particles of - // the species that has the most particles in that cell - amrex::ParallelFor( n_cells, - [=] AMREX_GPU_DEVICE (int i_cell) noexcept - { - const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; - const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; - // Particular case: no pair if a species has no particle in that cell - if (n_part_in_cell_1 == 0 || n_part_in_cell_2 == 0) - { - p_n_pairs_in_each_cell[i_cell] = 0; - } - else - { - p_n_pairs_in_each_cell[i_cell] = amrex::max(n_part_in_cell_1,n_part_in_cell_2); - } - } - ); - - // Start indices of the pairs in a cell. Will be used for particle creation - amrex::Gpu::DeviceVector pair_offsets(n_cells); - const index_type n_total_pairs = (n_cells == 0) ? 0: - amrex::Scan::ExclusiveSum(n_cells, - p_n_pairs_in_each_cell, pair_offsets.data()); - index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); - - // Now we create the mask. In the DSMC scheme the mask will serve two - // purposes, 1) record whether a given pair should collide and 2) record - // the scattering process that should occur. - amrex::Gpu::DeviceVector mask(n_total_pairs); - index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); - - // Will be filled with the index of the first particle of a given pair - amrex::Gpu::DeviceVector pair_indices_1(n_total_pairs); - index_type* AMREX_RESTRICT p_pair_indices_1 = pair_indices_1.dataPtr(); - // Will be filled with the index of the second particle of a given pair - amrex::Gpu::DeviceVector pair_indices_2(n_total_pairs); - index_type* AMREX_RESTRICT p_pair_indices_2 = pair_indices_2.dataPtr(); - - // How much weight should be given to the produced particle - based on the - // weight of the collision partner which is not split - amrex::Gpu::DeviceVector pair_reaction_weight(n_total_pairs); - amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight = - pair_reaction_weight.dataPtr(); - - // Loop over cells - amrex::ParallelForRNG( n_cells, - [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept - { - // The particles from species1 that are in the cell `i_cell` are - // given by the `indices_1[cell_start_1:cell_stop_1]` - index_type const cell_start_1 = cell_offsets_1[i_cell]; - index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; - // Same for species 2 - index_type const cell_start_2 = cell_offsets_2[i_cell]; - index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; - // Same but for the pairs - index_type const cell_start_pair = p_pair_offsets[i_cell]; - - // ux from species1 can be accessed like this: - // ux_1[ indices_1[i] ], where i is between - // cell_start_1 (inclusive) and cell_start_2 (exclusive) - - // Do not collide if one species is missing in the cell - if ( cell_stop_1 - cell_start_1 < 1 || - cell_stop_2 - cell_start_2 < 1 ) { return; } - - // shuffle - ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); - ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); -#if defined WARPX_DIM_RZ - const int ri = (i_cell - i_cell%nz) / nz; - auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; -#endif - // Call the function in order to perform collisions - // If there are product species, p_mask, p_pair_indices_1/2, and - // p_pair_reaction_weight are filled here - CollisionFilter( - cell_start_1, cell_stop_1, cell_start_2, cell_stop_2, - indices_1, indices_2, - ptd_1, ptd_2, - m1, m2, dt, dV, - cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight, - process_count, scattering_processes, engine ); - } - ); - - const auto num_p_tile1 = ptile_1.numParticles(); - const auto num_p_tile2 = ptile_2.numParticles(); - - // Create the new product particles and define their initial values - // num_added: how many particles of each product species have been created - const int num_added = splitScatteringParticles( - n_total_pairs, - ptile_1, ptile_2, - p_mask, - copy_species1, copy_species2, - m1, m2, - p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight); - - if (num_added > 0) { - setNewParticleIDs(ptile_1, num_p_tile1, num_added); - setNewParticleIDs(ptile_2, num_p_tile2, num_added); - } -} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H new file mode 100644 index 00000000000..1039ee96969 --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -0,0 +1,197 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef DSMC_FUNC_H_ +#define DSMC_FUNC_H_ + +#include "CollisionFilterFunc.H" + +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" +#include "Particles/Collision/BinaryCollision/ShuffleFisherYates.H" +#include "Particles/Collision/CollisionBase.H" +#include "Particles/Collision/ScatteringProcess.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleCreation/SmartCopy.H" +#include "Particles/ParticleCreation/SmartUtils.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include +#include + +/** + * \brief This class performs DSMC (direct simulation Monte Carlo) collisions + * within a cell. Particles are paired up and for each pair a stochastic process + * determines whether a collision occurs. The algorithm is similar to the one + * used for binary Coulomb collisions and the nuclear fusion module. + */ +class DSMCFunc +{ + // Define shortcuts for frequently-used type names + using ParticleType = WarpXParticleContainer::ParticleType; + using ParticleTileType = WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = ParticleBins::index_type; + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; + +public: + + /** + * \brief Constructor of the DSMCFunc class. + */ + DSMCFunc () = default; + + /** + * \brief Constructor of the DSMCFunc class + * + * @param[in] collision_name the name of the collision + * @param[in] mypc pointer to the MultiParticleContainer + * @param[in] isSameSpecies whether the two colliding species are the same + */ + DSMCFunc ( const std::string& collision_name, + MultiParticleContainer const * mypc, + bool isSameSpecies ); + + struct Executor { + /** + * \brief Executor of the DSMCFunc class. Performs DSMC collisions at the cell level. + * Note that this function does not yet create the product particles, but + * instead fills an array p_mask that stores which pairs result in a collision event. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in] soa_1,soa_2 contain the struct of array data of the two species + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] cell_start_pair is the start index of the pairs in that cell. + * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given + * pair. It is only needed here to store information that will be used later on when actually + * creating the product particles. + * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the + * particles of a given pair. They are only needed here to store information that will be used + * later on when actually creating the product particles. + * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only + * needed here to store information that will be used later on when actually creating the + * product particles. + * @param[in] engine the random engine. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void operator() ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + const SoaData_type& soa_1, const SoaData_type& soa_2, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, + index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, + index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + amrex::RandomEngine const& engine) const + { + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u1y = soa_1.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u1z = soa_1.m_rdata[PIdx::uz]; + + amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT u2x = soa_2.m_rdata[PIdx::ux]; + amrex::ParticleReal * const AMREX_RESTRICT u2y = soa_2.m_rdata[PIdx::uy]; + amrex::ParticleReal * const AMREX_RESTRICT u2z = soa_2.m_rdata[PIdx::uz]; + + // Number of macroparticles of each species + const index_type NI1 = I1e - I1s; + const index_type NI2 = I2e - I2s; + const index_type max_N = amrex::max(NI1,NI2); + + index_type i1 = I1s; + index_type i2 = I2s; + index_type pair_index = cell_start_pair; + + // Because the number of particles of each species is not always equal (NI1 != NI2 + // in general), some macroparticles will be paired with multiple macroparticles of the + // other species and we need to decrease their weight accordingly. + // c1 corresponds to the minimum number of times a particle of species 1 will be paired + // with a particle of species 2. Same for c2. + // index_type(1): https://github.com/AMReX-Codes/amrex/pull/3684 + const index_type c1 = amrex::max(NI2/NI1, index_type(1)); + const index_type c2 = amrex::max(NI1/NI2, index_type(1)); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; + amrex::ParticleReal * const AMREX_RESTRICT theta2 = soa_2.m_rdata[PIdx::theta]; +#endif + + for (index_type k = 0; k < max_N; ++k) + { + // c1k : how many times the current particle of species 1 is paired with a particle + // of species 2. Same for c2k. + const index_type c1k = (k%NI1 < max_N%NI1) ? c1 + 1: c1; + const index_type c2k = (k%NI2 < max_N%NI2) ? c2 + 1: c2; + +#if (defined WARPX_DIM_RZ) + /* In RZ geometry, macroparticles can collide with other macroparticles + * in the same *cylindrical* cell. For this reason, collisions between macroparticles + * are actually not local in space. In this case, the underlying assumption is that + * particles within the same cylindrical cell represent a cylindrically-symmetry + * momentum distribution function. Therefore, here, we temporarily rotate the + * momentum of one of the macroparticles in agreement with this cylindrical symmetry. + * (This is technically only valid if we use only the m=0 azimuthal mode in the simulation; + * there is a corresponding assert statement at initialization.) + */ + amrex::ParticleReal const theta = theta2[I2[i2]]-theta1[I1[i1]]; + amrex::ParticleReal const u1xbuf = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf*std::cos(theta) - u1y[I1[i1]]*std::sin(theta); + u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); +#endif + + CollisionPairFilter( + u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], + u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], + m1, m2, w1[ I1[i1] ]/c1k, w2[ I2[i2] ]/c2k, + dt, dV, static_cast(pair_index), p_mask, + p_pair_reaction_weight, static_cast(max_N), + m_process_count, m_scattering_processes_data, engine); + +#if (defined WARPX_DIM_RZ) + amrex::ParticleReal const u1xbuf_new = u1x[I1[i1]]; + u1x[I1[i1]] = u1xbuf_new*std::cos(-theta) - u1y[I1[i1]]*std::sin(-theta); + u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); +#endif + + p_pair_indices_1[pair_index] = I1[i1]; + p_pair_indices_2[pair_index] = I2[i2]; + ++i1; if ( i1 == I1e ) { i1 = I1s; } + ++i2; if ( i2 == I2e ) { i2 = I2s; } + ++pair_index; + } + } + + int m_process_count; + ScatteringProcess::Executor* m_scattering_processes_data; + }; + + [[nodiscard]] Executor const& executor () const { return m_exe; } + +private: + amrex::Vector m_scattering_processes; + amrex::Gpu::DeviceVector m_scattering_processes_exe; + + Executor m_exe; +}; + +#endif // DSMC_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp new file mode 100644 index 00000000000..e7a4eb722fc --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp @@ -0,0 +1,78 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#include "DSMCFunc.H" + +/** + * \brief Constructor of the DSMCFunc class + * + * @param[in] collision_name the name of the collision + * @param[in] mypc pointer to the MultiParticleContainer + * @param[in] isSameSpecies whether the two colliding species are the same + */ +DSMCFunc::DSMCFunc ( + const std::string& collision_name, + [[maybe_unused]] MultiParticleContainer const * const mypc, + [[maybe_unused]] const bool isSameSpecies ) +{ + using namespace amrex::literals; + + const amrex::ParmParse pp_collision_name(collision_name); + + // query for a list of collision processes + // these could be elastic, excitation, charge_exchange, back, etc. + amrex::Vector scattering_process_names; + pp_collision_name.queryarr("scattering_processes", scattering_process_names); + + // create a vector of ScatteringProcess objects from each scattering + // process name + for (const auto& scattering_process : scattering_process_names) { + const std::string kw_cross_section = scattering_process + "_cross_section"; + std::string cross_section_file; + pp_collision_name.query(kw_cross_section.c_str(), cross_section_file); + + // if the scattering process is excitation or ionization get the + // energy associated with that process + amrex::ParticleReal energy = 0._prt; + if (scattering_process.find("excitation") != std::string::npos || + scattering_process.find("ionization") != std::string::npos) { + const std::string kw_energy = scattering_process + "_energy"; + utils::parser::getWithParser( + pp_collision_name, kw_energy.c_str(), energy); + } + + ScatteringProcess process(scattering_process, cross_section_file, energy); + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(process.type() != ScatteringProcessType::INVALID, + "Cannot add an unknown scattering process type"); + + m_scattering_processes.push_back(std::move(process)); + } + + const int process_count = static_cast(m_scattering_processes.size()); + + // Store ScatteringProcess::Executor(s). +#ifdef AMREX_USE_GPU + amrex::Gpu::HostVector h_scattering_processes_exe; + for (auto const& p : m_scattering_processes) { + h_scattering_processes_exe.push_back(p.executor()); + } + m_scattering_processes_exe.resize(process_count); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, h_scattering_processes_exe.begin(), + h_scattering_processes_exe.end(), m_scattering_processes_exe.begin()); + amrex::Gpu::streamSynchronize(); +#else + for (auto const& p : m_scattering_processes) { + m_scattering_processes_exe.push_back(p.executor()); + } +#endif + + // Link executor to appropriate ScatteringProcess executors + m_exe.m_scattering_processes_data = m_scattering_processes_exe.data(); + m_exe.m_process_count = process_count; +} diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/Make.package b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package index b4cfab89c64..087044e63cd 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/Make.package +++ b/Source/Particles/Collision/BinaryCollision/DSMC/Make.package @@ -1,3 +1,4 @@ -CEXE_sources += DSMC.cpp +CEXE_sources += DSMCFunc.cpp +CEXE_sources += SplitAndScatterFunc.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Collision/BinaryCollision/DSMC diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index f684b60da78..074226d8227 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -1,4 +1,4 @@ -/* Copyright 2023 The WarpX Community +/* Copyright 2023-2024 The WarpX Community * * This file is part of WarpX. * @@ -9,173 +9,286 @@ #ifndef SPLIT_AND_SCATTER_FUNC_H_ #define SPLIT_AND_SCATTER_FUNC_H_ +#include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Particles/Collision/ScatteringProcess.H" -#include "Particles/NamedComponentParticleContainer.H" - -#include +#include "Particles/ParticleCreation/SmartCopy.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/ParticleUtils.H" /** - * \brief Function that performs the particle scattering and injection due - * to binary collisions. - * - * \return num_added the number of particles added to each species. + * \brief This class defines an operator to create product particles from DSMC + * collisions and sets the particle properties (position, momentum, weight). */ -template ::value, int> foo = 0> -int splitScatteringParticles ( - const index_type& n_total_pairs, - Tile& ptile1, Tile& ptile2, - const Index* AMREX_RESTRICT mask, - CopyFunc&& copy1, CopyFunc&& copy2, - const amrex::ParticleReal m1, const amrex::ParticleReal m2, - const index_type* AMREX_RESTRICT p_pair_indices_1, - const index_type* AMREX_RESTRICT p_pair_indices_2, - const amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight ) noexcept +class SplitAndScatterFunc { - using namespace amrex; + // Define shortcuts for frequently-used type names + using ParticleType = typename WarpXParticleContainer::ParticleType; + using ParticleTileType = typename WarpXParticleContainer::ParticleTileType; + using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; + using ParticleBins = amrex::DenseBins; + using index_type = typename ParticleBins::index_type; + using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - if (n_total_pairs == 0) { - return 0; - } - Gpu::DeviceVector offsets(n_total_pairs); - Index* AMREX_RESTRICT p_offsets = offsets.dataPtr(); +public: + /** + * \brief Default constructor of the SplitAndScatterFunc class. + */ + SplitAndScatterFunc () = default; - // The following is used to calculate the appropriate offsets. Note that - // a standard cummulative sum is not appropriate since the mask is also - // used to specify the type of collision and can therefore have values >1 - auto const num_added = amrex::Scan::PrefixSum(n_total_pairs, - [=] AMREX_GPU_DEVICE (Index i) -> Index { return mask[i] ? 1 : 0; }, - [=] AMREX_GPU_DEVICE (Index i, Index s) { p_offsets[i] = s; }, - amrex::Scan::Type::exclusive, amrex::Scan::retSum - ); + /** + * \brief Constructor of the SplitAndScatterFunc class + * + * @param[in] collision_name the name of the collision + * @param[in] mypc pointer to the MultiParticleContainer + */ + SplitAndScatterFunc (const std::string& collision_name, MultiParticleContainer const * mypc); - const auto ptile1_index = ptile1.numParticles(); - const auto ptile2_index = ptile2.numParticles(); - ptile1.resize(ptile1_index + num_added); - ptile2.resize(ptile2_index + num_added); + /** + * \brief Function that performs the particle scattering and injection due + * to binary collisions. + * + * \return num_added the number of particles added to each species. + */ + AMREX_INLINE + amrex::Vector operator() ( + const index_type& n_total_pairs, + // Tile& ptile1, Tile& ptile2, + const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + const amrex::Vector& pc_products, + ParticleTileType** AMREX_RESTRICT tile_products, + const amrex::ParticleReal m1, const amrex::ParticleReal m2, + const amrex::Vector& /*products_mass*/, + const index_type* AMREX_RESTRICT mask, + const amrex::Vector& products_np, + const SmartCopy* AMREX_RESTRICT copy_species1, + const SmartCopy* AMREX_RESTRICT copy_species2, + const index_type* AMREX_RESTRICT p_pair_indices_1, + const index_type* AMREX_RESTRICT p_pair_indices_2, + const amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight ) const + { + using namespace amrex::literals; - const auto ptile1_data = ptile1.getParticleTileData(); - const auto ptile2_data = ptile2.getParticleTileData(); + if (n_total_pairs == 0) { return amrex::Vector(m_num_product_species, 0); } - ParallelForRNG(n_total_pairs, - [=] AMREX_GPU_DEVICE (int i, RandomEngine const& engine) noexcept - { - if (mask[i]) + amrex::Gpu::DeviceVector offsets(n_total_pairs); + index_type* AMREX_RESTRICT offsets_data = offsets.data(); + const index_type* AMREX_RESTRICT p_offsets = offsets.dataPtr(); + + // The following is used to calculate the appropriate offsets. Note that + // a standard cummulative sum is not appropriate since the mask is also + // used to specify the type of collision and can therefore have values >1 + auto const total = amrex::Scan::PrefixSum(n_total_pairs, + [=] AMREX_GPU_DEVICE (index_type i) -> index_type { return mask[i] ? 1 : 0; }, + [=] AMREX_GPU_DEVICE (index_type i, index_type s) { offsets_data[i] = s; }, + amrex::Scan::Type::exclusive, amrex::Scan::retSum + ); + + amrex::Vector num_added_vec(m_num_product_species); + for (int i = 0; i < m_num_product_species; i++) { - // First, make copies of the colliding particles - copy1(ptile1_data, ptile1_data, p_pair_indices_1[i], p_offsets[i] + ptile1_index, engine); - copy2(ptile2_data, ptile2_data, p_pair_indices_2[i], p_offsets[i] + ptile2_index, engine); - - // Now we adjust the properties of the original and child particles, - // starting with the parent particles - auto& w1 = ptile1_data.m_rdata[PIdx::w][p_pair_indices_1[i]]; - auto& w2 = ptile2_data.m_rdata[PIdx::w][p_pair_indices_2[i]]; - uint64_t* AMREX_RESTRICT idcpu1 = ptile1_data.m_idcpu; - uint64_t* AMREX_RESTRICT idcpu2 = ptile2_data.m_idcpu; - - // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX - // to replace the following lambda. - auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + // How many particles of product species i are created. + const index_type num_added = total * m_num_products_host[i]; + num_added_vec[i] = static_cast(num_added); + tile_products[i]->resize(products_np[i] + num_added); + } + + // this works for DSMC since the colliding particles are also products + const auto soa_1 = tile_products[0]->getParticleTileData(); + const auto soa_2 = tile_products[1]->getParticleTileData(); + + amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; + uint64_t* AMREX_RESTRICT idcpu2 = soa_2.m_idcpu; + + // Create necessary GPU vectors, that will be used in the kernel below + amrex::Vector soa_products; + for (int i = 0; i < m_num_product_species; i++) + { + soa_products.push_back(tile_products[i]->getParticleTileData()); + } +#ifdef AMREX_USE_GPU + amrex::Gpu::DeviceVector device_soa_products(m_num_product_species); + amrex::Gpu::DeviceVector device_products_np(m_num_product_species); + + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, soa_products.begin(), + soa_products.end(), + device_soa_products.begin()); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, products_np.begin(), + products_np.end(), + device_products_np.begin()); + + amrex::Gpu::streamSynchronize(); + SoaData_type* AMREX_RESTRICT soa_products_data = device_soa_products.data(); + const index_type* AMREX_RESTRICT products_np_data = device_products_np.data(); +#else + SoaData_type* AMREX_RESTRICT soa_products_data = soa_products.data(); + const index_type* AMREX_RESTRICT products_np_data = products_np.data(); +#endif + + const int* AMREX_RESTRICT p_num_products_device = m_num_products_device.data(); + + amrex::ParallelForRNG(n_total_pairs, + [=] AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept + { + if (mask[i]) { + // for now we ignore the possibility of having actual reaction + // products - only duplicating (splitting) of the colliding + // particles is supported. + + const auto product1_index = products_np_data[0] + + (p_offsets[i]*p_num_products_device[0] + 0); + // Make a copy of the particle from species 1 + copy_species1[0](soa_products_data[0], soa_1, static_cast(p_pair_indices_1[i]), + static_cast(product1_index), engine); + // Set the weight of the new particles to p_pair_reaction_weight[i] + soa_products_data[0].m_rdata[PIdx::w][product1_index] = p_pair_reaction_weight[i]; + + const auto product2_index = products_np_data[1] + + (p_offsets[i]*p_num_products_device[1] + 0); + // Make a copy of the particle from species 2 + copy_species2[1](soa_products_data[1], soa_2, static_cast(p_pair_indices_2[i]), + static_cast(product2_index), engine); + // Set the weight of the new particles to p_pair_reaction_weight[i] + soa_products_data[1].m_rdata[PIdx::w][product2_index] = p_pair_reaction_weight[i]; + + // Remove p_pair_reaction_weight[i] from the colliding particles' weights + amrex::Gpu::Atomic::AddNoRet(&w1[p_pair_indices_1[i]], + -p_pair_reaction_weight[i]); + amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], + -p_pair_reaction_weight[i]); + + // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX + // to replace the following lambda. + auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + { #if defined(AMREX_USE_OMP) #pragma omp atomic write - idcpu = amrex::ParticleIdCpus::Invalid; + idcpu = amrex::ParticleIdCpus::Invalid; #else - amrex::Gpu::Atomic::Exch( - (unsigned long long *)&idcpu, - (unsigned long long)amrex::ParticleIdCpus::Invalid - ); + amrex::Gpu::Atomic::Exch( + (unsigned long long *)&idcpu, + (unsigned long long)amrex::ParticleIdCpus::Invalid + ); #endif - }; - - // Remove p_pair_reaction_weight[i] from the colliding particles' weights. - // If the colliding particle weight decreases to zero, remove particle by - // setting its id to -1. - Gpu::Atomic::AddNoRet(&w1, -p_pair_reaction_weight[i]); - if (w1 <= 0._prt) { - atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); - } + }; - Gpu::Atomic::AddNoRet(&w2, -p_pair_reaction_weight[i]); - if (w2 <= 0._prt) { - atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); - } + // If the colliding particle weight decreases to zero, remove particle by + // setting its id to invalid + if (w1[p_pair_indices_1[i]] <= std::numeric_limits::min()) + { + atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + } + if (w2[p_pair_indices_2[i]] <= std::numeric_limits::min()) + { + atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); + } - // Set the child particle properties appropriately - ptile1_data.m_rdata[PIdx::w][p_offsets[i] + ptile1_index] = p_pair_reaction_weight[i]; - ptile2_data.m_rdata[PIdx::w][p_offsets[i] + ptile2_index] = p_pair_reaction_weight[i]; - - auto& ux1 = ptile1_data.m_rdata[PIdx::ux][p_offsets[i] + ptile1_index]; - auto& uy1 = ptile1_data.m_rdata[PIdx::uy][p_offsets[i] + ptile1_index]; - auto& uz1 = ptile1_data.m_rdata[PIdx::uz][p_offsets[i] + ptile1_index]; - auto& ux2 = ptile2_data.m_rdata[PIdx::ux][p_offsets[i] + ptile2_index]; - auto& uy2 = ptile2_data.m_rdata[PIdx::uy][p_offsets[i] + ptile2_index]; - auto& uz2 = ptile2_data.m_rdata[PIdx::uz][p_offsets[i] + ptile2_index]; - - // for simplicity (for now) we assume non-relativistic particles - // and simply calculate the center-of-momentum velocity from the - // rest masses - auto const uCOM_x = (m1 * ux1 + m2 * ux2) / (m1 + m2); - auto const uCOM_y = (m1 * uy1 + m2 * uy2) / (m1 + m2); - auto const uCOM_z = (m1 * uz1 + m2 * uz2) / (m1 + m2); - - // transform to COM frame - ux1 -= uCOM_x; - uy1 -= uCOM_y; - uz1 -= uCOM_z; - ux2 -= uCOM_x; - uy2 -= uCOM_y; - uz2 -= uCOM_z; - - if (mask[i] == int(ScatteringProcessType::ELASTIC)) { - // randomly rotate the velocity vector for the first particle - ParticleUtils::RandomizeVelocity( - ux1, uy1, uz1, std::sqrt(ux1*ux1 + uy1*uy1 + uz1*uz1), engine - ); - // set the second particles velocity so that the total momentum - // is zero - ux2 = -ux1 * m1 / m2; - uy2 = -uy1 * m1 / m2; - uz2 = -uz1 * m1 / m2; - } else if (mask[i] == int(ScatteringProcessType::BACK)) { - // reverse the velocity vectors of both particles - ux1 *= -1.0_prt; - uy1 *= -1.0_prt; - uz1 *= -1.0_prt; - ux2 *= -1.0_prt; - uy2 *= -1.0_prt; - uz2 *= -1.0_prt; - } else if (mask[i] == int(ScatteringProcessType::CHARGE_EXCHANGE)) { - if (m1 == m2) { - auto const temp_ux = ux1; - auto const temp_uy = uy1; - auto const temp_uz = uz1; - ux1 = ux2; - uy1 = uy2; - uz1 = uz2; - ux2 = temp_ux; - uy2 = temp_uy; - uz2 = temp_uz; + // Set the child particle properties appropriately + auto& ux1 = soa_products_data[0].m_rdata[PIdx::ux][product1_index]; + auto& uy1 = soa_products_data[0].m_rdata[PIdx::uy][product1_index]; + auto& uz1 = soa_products_data[0].m_rdata[PIdx::uz][product1_index]; + auto& ux2 = soa_products_data[1].m_rdata[PIdx::ux][product2_index]; + auto& uy2 = soa_products_data[1].m_rdata[PIdx::uy][product2_index]; + auto& uz2 = soa_products_data[1].m_rdata[PIdx::uz][product2_index]; + + // for simplicity (for now) we assume non-relativistic particles + // and simply calculate the center-of-momentum velocity from the + // rest masses + auto const uCOM_x = (m1 * ux1 + m2 * ux2) / (m1 + m2); + auto const uCOM_y = (m1 * uy1 + m2 * uy2) / (m1 + m2); + auto const uCOM_z = (m1 * uz1 + m2 * uz2) / (m1 + m2); + + // transform to COM frame + ux1 -= uCOM_x; + uy1 -= uCOM_y; + uz1 -= uCOM_z; + ux2 -= uCOM_x; + uy2 -= uCOM_y; + uz2 -= uCOM_z; + + if (mask[i] == int(ScatteringProcessType::ELASTIC)) { + // randomly rotate the velocity vector for the first particle + ParticleUtils::RandomizeVelocity( + ux1, uy1, uz1, std::sqrt(ux1*ux1 + uy1*uy1 + uz1*uz1), engine + ); + // set the second particles velocity so that the total momentum + // is zero + ux2 = -ux1 * m1 / m2; + uy2 = -uy1 * m1 / m2; + uz2 = -uz1 * m1 / m2; + } else if (mask[i] == int(ScatteringProcessType::BACK)) { + // reverse the velocity vectors of both particles + ux1 *= -1.0_prt; + uy1 *= -1.0_prt; + uz1 *= -1.0_prt; + ux2 *= -1.0_prt; + uy2 *= -1.0_prt; + uz2 *= -1.0_prt; + } else if (mask[i] == int(ScatteringProcessType::CHARGE_EXCHANGE)) { + if (std::abs(m1 - m2) < 1e-28) { + auto const temp_ux = ux1; + auto const temp_uy = uy1; + auto const temp_uz = uz1; + ux1 = ux2; + uy1 = uy2; + uz1 = uz2; + ux2 = temp_ux; + uy2 = temp_uy; + uz2 = temp_uz; + } + else { + amrex::Abort("Uneven mass charge-exchange not implemented yet."); + } } else { - Abort("Uneven mass charge-exchange not implemented yet."); + amrex::Abort("Unknown scattering process."); } + // transform back to labframe + ux1 += uCOM_x; + uy1 += uCOM_y; + uz1 += uCOM_z; + ux2 += uCOM_x; + uy2 += uCOM_y; + uz2 += uCOM_z; } - else { - Abort("Unknown scattering process."); - } - // transform back to labframe - ux1 += uCOM_x; - uy1 += uCOM_y; - uz1 += uCOM_z; - ux2 += uCOM_x; - uy2 += uCOM_y; - uz2 += uCOM_z; + }); + + // Initialize the user runtime components + for (int i = 0; i < m_num_product_species; i++) + { + const int start_index = int(products_np[i]); + const int stop_index = int(products_np[i] + num_added_vec[i]); + ParticleCreation::DefaultInitializeRuntimeAttributes(*tile_products[i], + 0, 0, + pc_products[i]->getUserRealAttribs(), pc_products[i]->getUserIntAttribs(), + pc_products[i]->getParticleComps(), pc_products[i]->getParticleiComps(), + pc_products[i]->getUserRealAttribParser(), + pc_products[i]->getUserIntAttribParser(), +#ifdef WARPX_QED + false, // do not initialize QED quantities, since they were initialized + // when calling the SmartCopy functors + pc_products[i]->get_breit_wheeler_engine_ptr(), + pc_products[i]->get_quantum_sync_engine_ptr(), +#endif + pc_products[i]->getIonizationInitialLevel(), + start_index, stop_index); } - }); - Gpu::synchronize(); - return static_cast(num_added); -} + amrex::Gpu::synchronize(); + return num_added_vec; + } + +private: + // How many different type of species the collision produces + int m_num_product_species; + // Vectors of size m_num_product_species storing how many particles of a given species are + // produced by a collision event. These vectors are duplicated (one version for host and one + // for device) which is necessary with GPUs but redundant on CPU. + amrex::Gpu::DeviceVector m_num_products_device; + amrex::Gpu::HostVector m_num_products_host; + CollisionType m_collision_type; +}; #endif // SPLIT_AND_SCATTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp new file mode 100644 index 00000000000..de8de9b505d --- /dev/null +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.cpp @@ -0,0 +1,43 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#include "SplitAndScatterFunc.H" + +SplitAndScatterFunc::SplitAndScatterFunc (const std::string& collision_name, + MultiParticleContainer const * const mypc): + m_collision_type{BinaryCollisionUtils::get_collision_type(collision_name, mypc)} +{ + const amrex::ParmParse pp_collision_name(collision_name); + + if (m_collision_type == CollisionType::DSMC) + { + // here we can add logic to deal with cases where products are created, + // for example with impact ionization + m_num_product_species = 2; + m_num_products_host.push_back(1); + m_num_products_host.push_back(1); +#ifndef AMREX_USE_GPU + // On CPU, the device vector can be filled immediately + m_num_products_device.push_back(1); + m_num_products_device.push_back(1); +#endif + } + else + { + WARPX_ABORT_WITH_MESSAGE("Unknown collision type in SplitAndScatterFunc"); + } + +#ifdef AMREX_USE_GPU + m_num_products_device.resize(m_num_product_species); + amrex::Gpu::copyAsync(amrex::Gpu::hostToDevice, m_num_products_host.begin(), + m_num_products_host.end(), + m_num_products_device.begin()); + amrex::Gpu::streamSynchronize(); +#endif +} diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index 8f19364baec..b3826114b09 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -77,61 +77,67 @@ public: utils::parser::queryWithParser( pp_collision_name, "fusion_probability_target_value", m_probability_target_value); + + m_exe.m_fusion_multiplier = m_fusion_multiplier; + m_exe.m_probability_threshold = m_probability_threshold; + m_exe.m_probability_target_value = m_probability_target_value; + m_exe.m_fusion_type = m_fusion_type; + m_exe.m_isSameSpecies = m_isSameSpecies; } - /** - * \brief operator() of the NuclearFusionFunc class. Performs nuclear fusions at the cell level - * using the algorithm described in Higginson et al., Journal of Computational Physics 388, - * 439-453 (2019). Note that this function does not yet create the product particles, but - * instead fills an array p_mask that stores which collisions result in a fusion event. - * - * Also note that there are three main differences between this implementation and the - * algorithm described in Higginson's paper: - * - 1) The transformation from the lab frame to the center of mass frame is nonrelativistic - * in Higginson's paper. Here, we implement a relativistic generalization. - * - 2) The behaviour when the estimated fusion probability is greater than one is not - * specified in Higginson's paper. Here, we provide an implementation using two runtime - * dependent parameters (fusion probability threshold and fusion probability target value). See - * documentation for more details. - * - 3) Here, we divide the weight of a particle by the number of times it is paired with other - * particles. This was not explicitly specified in Higginson's paper. - * - * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). - * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). - * @param[in] I1,I2 index arrays. They determine all elements that will be used. - * @param[in] soa_1,soa_2 contain the struct of array data of the two species - * @param[in] m1,m2 are masses. - * @param[in] dt is the time step length between two collision calls. - * @param[in] dV is the volume of the corresponding cell. - * @param[in] cell_start_pair is the start index of the pairs in that cell. - * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given - * pair. It is only needed here to store information that will be used later on when actually - * creating the product particles. - * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the - * particles of a given pair. They are only needed here to store information that will be used - * later on when actually creating the product particles. - * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only - * needed here to store information that will be used later on when actually creating the - * product particles. - * @param[in] engine the random engine. - */ - AMREX_GPU_HOST_DEVICE AMREX_INLINE - void operator() ( - index_type const I1s, index_type const I1e, - index_type const I2s, index_type const I2e, - index_type const* AMREX_RESTRICT I1, - index_type const* AMREX_RESTRICT I2, - SoaData_type soa_1, SoaData_type soa_2, - GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, - amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, - amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, - index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, - index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, - amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, - amrex::RandomEngine const& engine) const + struct Executor { + /** + * \brief Executor of the NuclearFusionFunc class. Performs nuclear fusions at the cell level + * using the algorithm described in Higginson et al., Journal of Computational Physics 388, + * 439-453 (2019). Note that this function does not yet create the product particles, but + * instead fills an array p_mask that stores which collisions result in a fusion event. + * + * Also note that there are three main differences between this implementation and the + * algorithm described in Higginson's paper: + * - 1) The transformation from the lab frame to the center of mass frame is nonrelativistic + * in Higginson's paper. Here, we implement a relativistic generalization. + * - 2) The behaviour when the estimated fusion probability is greater than one is not + * specified in Higginson's paper. Here, we provide an implementation using two runtime + * dependent parameters (fusion probability threshold and fusion probability target value). See + * documentation for more details. + * - 3) Here, we divide the weight of a particle by the number of times it is paired with other + * particles. This was not explicitly specified in Higginson's paper. + * + * @param[in] I1s,I2s is the start index for I1,I2 (inclusive). + * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). + * @param[in] I1,I2 index arrays. They determine all elements that will be used. + * @param[in] soa_1,soa_2 contain the struct of array data of the two species + * @param[in] m1,m2 are masses. + * @param[in] dt is the time step length between two collision calls. + * @param[in] dV is the volume of the corresponding cell. + * @param[in] cell_start_pair is the start index of the pairs in that cell. + * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given + * pair. It is only needed here to store information that will be used later on when actually + * creating the product particles. + * @param[out] p_pair_indices_1,p_pair_indices_2 arrays that store the indices of the + * particles of a given pair. They are only needed here to store information that will be used + * later on when actually creating the product particles. + * @param[out] p_pair_reaction_weight stores the weight of the product particles. It is only + * needed here to store information that will be used later on when actually creating the + * product particles. + * @param[in] engine the random engine. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void operator() ( + index_type const I1s, index_type const I1e, + index_type const I2s, index_type const I2e, + index_type const* AMREX_RESTRICT I1, + index_type const* AMREX_RESTRICT I2, + const SoaData_type& soa_1, const SoaData_type& soa_2, + GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, + amrex::ParticleReal const m1, amrex::ParticleReal const m2, + amrex::Real const dt, amrex::Real const dV, + index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, + index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, + amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, + amrex::RandomEngine const& engine) const { - amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; amrex::ParticleReal * const AMREX_RESTRICT u1y = soa_1.m_rdata[PIdx::uy]; @@ -213,9 +219,17 @@ public: ++i2; if ( i2 == I2e ) { i2 = I2s; } ++pair_index; } - } + amrex::ParticleReal m_fusion_multiplier; + amrex::ParticleReal m_probability_threshold; + amrex::ParticleReal m_probability_target_value; + NuclearFusionType m_fusion_type; + bool m_isSameSpecies; + }; + + [[nodiscard]] Executor const& executor () const { return m_exe; } + private: // Factor used to increase the number of fusion reaction by decreasing the weight of the // produced particles @@ -230,6 +244,8 @@ private: amrex::ParticleReal m_probability_target_value; NuclearFusionType m_fusion_type; bool m_isSameSpecies; + + Executor m_exe; }; #endif // NUCLEAR_FUSION_FUNC_H_ diff --git a/Source/Particles/Collision/CollisionHandler.cpp b/Source/Particles/Collision/CollisionHandler.cpp index 5abee34c86a..d723a1017d8 100644 --- a/Source/Particles/Collision/CollisionHandler.cpp +++ b/Source/Particles/Collision/CollisionHandler.cpp @@ -8,9 +8,10 @@ #include "Particles/Collision/BackgroundMCC/BackgroundMCCCollision.H" #include "Particles/Collision/BackgroundStopping/BackgroundStopping.H" -#include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" #include "Particles/Collision/BinaryCollision/BinaryCollision.H" -#include "Particles/Collision/BinaryCollision/DSMC/DSMC.H" +#include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" +#include "Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H" +#include "Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" #include "Utils/TextMsg.H" @@ -45,7 +46,8 @@ CollisionHandler::CollisionHandler(MultiParticleContainer const * const mypc) if (type == "pairwisecoulomb") { allcollisions[i] = std::make_unique>( - collision_names[i], mypc); + collision_names[i], mypc + ); } else if (type == "background_mcc") { allcollisions[i] = std::make_unique(collision_names[i]); @@ -54,12 +56,16 @@ CollisionHandler::CollisionHandler(MultiParticleContainer const * const mypc) allcollisions[i] = std::make_unique(collision_names[i]); } else if (type == "dsmc") { - allcollisions[i] = std::make_unique(collision_names[i]); + allcollisions[i] = + std::make_unique>( + collision_names[i], mypc + ); } else if (type == "nuclearfusion") { allcollisions[i] = std::make_unique>( - collision_names[i], mypc); + collision_names[i], mypc + ); } else{ WARPX_ABORT_WITH_MESSAGE("Unknown collision type."); From c8b05b86c277ac93958bdc48169676b3b8bfe170 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 11 Apr 2024 00:06:43 -0700 Subject: [PATCH 045/190] Use local deleteInvalidParticles (instead of Redistribute) in binary collisions (#4851) --- Source/Particles/Collision/BinaryCollision/BinaryCollision.H | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index b36c28db662..807dc827b6b 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -213,9 +213,8 @@ public: // The fact that there are product species indicates that particles of // the colliding species (`species1` and `species2`) may be removed // (i.e., marked as invalid) in the process of creating new product particles. - // Call (local) redistribute to remove invalid particles - species1.Redistribute(lev, lev, 0, 1, true); - species2.Redistribute(lev, lev, 0, 1, true); + species1.deleteInvalidParticles(); + species2.deleteInvalidParticles(); } } } From 8443bce82431a247dea46254b8cb9f41041eef72 Mon Sep 17 00:00:00 2001 From: Ryan Sandberg Date: Fri, 12 Apr 2024 13:49:00 -0700 Subject: [PATCH 046/190] Update optimas/libensemble documentation (#4854) * add openpmd-viewer to conda install instructions * add optimas example to documentation --- Docs/source/install/dependencies.rst | 4 +- Docs/source/usage/workflows.rst | 2 +- Docs/source/usage/workflows/libensemble.rst | 144 ----------- Docs/source/usage/workflows/optimas.rst | 11 + Tools/LibEnsemble/all_machine_specs.py | 26 -- Tools/LibEnsemble/plot_results.py | 128 ---------- Tools/LibEnsemble/read_sim_output.py | 115 --------- Tools/LibEnsemble/requirements.txt | 7 - Tools/LibEnsemble/run_libensemble_on_warpx.py | 224 ------------------ Tools/LibEnsemble/sim/inputs | 215 ----------------- Tools/LibEnsemble/summit_submit_mproc.sh | 56 ----- Tools/LibEnsemble/warpx_simf.py | 110 --------- Tools/LibEnsemble/write_sim_input.py | 118 --------- 13 files changed, 14 insertions(+), 1146 deletions(-) delete mode 100644 Docs/source/usage/workflows/libensemble.rst create mode 100644 Docs/source/usage/workflows/optimas.rst delete mode 100644 Tools/LibEnsemble/all_machine_specs.py delete mode 100755 Tools/LibEnsemble/plot_results.py delete mode 100644 Tools/LibEnsemble/read_sim_output.py delete mode 100644 Tools/LibEnsemble/requirements.txt delete mode 100644 Tools/LibEnsemble/run_libensemble_on_warpx.py delete mode 100644 Tools/LibEnsemble/sim/inputs delete mode 100755 Tools/LibEnsemble/summit_submit_mproc.sh delete mode 100644 Tools/LibEnsemble/warpx_simf.py delete mode 100644 Tools/LibEnsemble/write_sim_input.py diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index 4f3544d5c0d..ce9f9dca520 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -80,7 +80,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv + conda create -n warpx-cpu-mpich-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp "openpmd-api=*=mpi_mpich*" openpmd-viewer python make numpy pandas scipy yt "fftw=*=mpi_mpich*" pkg-config matplotlib mamba mpich mpi4py ninja pip virtualenv conda activate warpx-cpu-mpich-dev # compile WarpX with -DWarpX_MPI=ON @@ -90,7 +90,7 @@ Conda (Linux/macOS/Windows) .. code-block:: bash - conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api python make numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv + conda create -n warpx-cpu-dev -c conda-forge blaspp boost ccache cmake compilers git lapackpp openpmd-api openpmd-viewer python make numpy pandas scipy yt fftw pkg-config matplotlib mamba ninja pip virtualenv conda activate warpx-cpu-dev # compile WarpX with -DWarpX_MPI=OFF diff --git a/Docs/source/usage/workflows.rst b/Docs/source/usage/workflows.rst index 039be199257..5c5329e18b8 100644 --- a/Docs/source/usage/workflows.rst +++ b/Docs/source/usage/workflows.rst @@ -13,8 +13,8 @@ This section collects typical user workflows and best practices for WarpX. workflows/plot_distribution_mapping workflows/debugging workflows/generate_lookup_tables_with_tools - workflows/libensemble workflows/plot_timestep_duration workflows/psatd_stencil workflows/archiving workflows/ml_dataset_training + workflows/optimas diff --git a/Docs/source/usage/workflows/libensemble.rst b/Docs/source/usage/workflows/libensemble.rst deleted file mode 100644 index af42ae6adf9..00000000000 --- a/Docs/source/usage/workflows/libensemble.rst +++ /dev/null @@ -1,144 +0,0 @@ -.. _libensemble: - -Run LibEnsemble on WarpX -======================== - -`LibEnsemble `__ is a library to coordinate the concurrent evaluation of dynamic ensembles of calculations. -While a WarpX simulation can provide insight in some physics, it remains a single point evaluation in the space of parameters. -If you have a simulation ready for use, but would like to (i) scan over some input parameters uniformly for, e.g., a tolerance study, or (ii) have a random evaluation of the space of input parameters within a given span or (iii) tune some input parameters to optimize an output parameter, e.g., beam emittance, energy spread, etc., LibEnsemble provides these capabilities and will take care of tasks monitoring with fault tolerance on multiple platforms (LibEnsemble targets modern HPC platforms like Summit). - -Scripts to run LibEnsemble on WarpX simulations can be found in ``WarpX/Tools/LibEnsemble/``. -This documentation does not aim at giving a training on LibEnsemble, so please refer to the `LibEnsemble documentation `__ for technical details. - -WarpX example problem for LibEnsemble study -------------------------------------------- - -The WarpX example is built on a 2D input file (so that 1 simulation take < 1 min) of a 2-stage laser-wakefield simulation in a boosted frame. -It aims at optimizing emittance preservation in the coupling between two consecutive plasma accelerator stages. -Each stage accelerates an externally-injected electron beam, which charge has been tuned to show a decent amount of Ez field flattening due to longitudinal beam loading. -Each stage has a parabolic transverse profile to guide the laser pulse, and a uniform longitudinal profile with cos-shape ramps and the entrance and at the exit. -A fresh laser pulse is introduced at the entrance of each stage and deleted at the exit. -The beam transverse distribution is matched to the first stage and the beam is injected at the beginning of the plateau of the first stage, so that the emittance is conserved in the first stage. -The two stages are separated by a few-cm gap, and a focusing lens is placed in-between. -Note that this is a very low resolution simulation to show an example, so it is **not** close to numerical convergence. - -In this example, we allow LibEnsemble to tune four **input parameters**: - - - Length of the downramp of the first stage - - - Longitudinal position of the focusing lens (between the two stages) - - - Strength of the focusing lens - - - Length of the downramp of the second stage - -The **output parameter** that LibEnsemble minimizes is the beam emittance at the exit of the second stage, while making sure the charge loss is small. - -The scripts provided can run on a local machine or on the Summit supercomputer at OLCF. -Two options are available: random sampling of parameter space or optimization on the output parameter. -For the latter, we are using the Asynchronously Parallel Optimization Solver for finding Multiple Minima `APOSMM `__ method provided by LibEnsemble. - -Install LibEnsemble -------------------- - -Besides a working WarpX executable, you have to install libEnsemble and its dependencies. - -You can either install all packages via `conda` (recommended), - -.. code-block:: sh - - conda install -c conda-forge libensemble matplotlib numpy scipy yt - -or try to install the same dependencies via `pip` (pick one *or* the other; note our :ref:`installation details on Summit `): - -.. literalinclude:: ../../../../Tools/LibEnsemble/requirements.txt - - -What's in ``Tools/LibEnsemble``? --------------------------------- - -See the `LibEnsemble User Guide `__ for an overview of LibEnsemble concepts. -In a nutshell, a user needs to define - - - A generator function ``gen_f`` that will generate inputs for the simulation, which can be done uniformly, randomly or using an optimizer. - The generator output, i.e., the simulation input, is called ``'x'``. - The generator is provided by LibEnsemble. - When the generator is an optimizer, it takes the simulation output called ``'f'`` as an input. - - - A simulation function ``sim_f`` that will take ``'x'`` as an input and return a single output value ``'f'``. - In our example, ``sim_f`` modifies the WarpX input file depending on ``'x'``, launches a WarpX simulation and reads the simulation output plotfile to extract ``'f'``. - - - An allocator function ``alloc_f`` that will feed available workers with tasks. - This is provided by LibEnsemble. - -The files in ``Tools/LibEnsemble/`` are: - -``run_libensemble_on_warpx.py`` - This is the main LibEnsemble script. - It imports ``gen_f`` and ``alloc_f`` from LibEnsemble, ``sim_f`` from file ``warpx_simf.py`` (see below), defines dictionaries for parameters of each of these objects (``gen_specs`` includes lower and upper bound of each element in the input array ``'x'``, ``alloc_specs`` and ``sim_specs`` respectively) and runs LibEnsemble. - -``warpx_simf.py`` - defines the ``sim_f`` function called ``run_warpx``: - - .. doxygenfunction:: run_warpx - -``sim/inputs`` - WarpX input file. Some of its parameters are modified in ``run_warpx``. - -``write_sim_input.py`` - (util) update one WarpX input file depending on values in ``'x'`` for this run. - - .. doxygenfunction:: write_sim_input - -``read_sim_output.py`` - (util) Read WarpX plotfile and return ``'f'``. - - .. doxygenfunction:: read_sim_output - -``plot_results.py`` - (util) Read LibEnsemble output files ``.npy`` and ``.pickle`` and plot output ``'f'`` (and other output, just for convenience) as a function of input from all simulations. - -``all_machine_specs.py`` - (util) dictionaries of machine-specific parameters. - For convenience, the maximum number of WarpX runs is defined there. - -``summit_submit_mproc.sh`` - Submission script for LibEnsemble+WarpX on Summit. - Make sure to edit this file and add your project ID for allocations. - -Run the example ---------------- - -On Summit or for a local run, LibEnsemble can run with a Random Number Generator (easiest, good as a first try) or with an optimizer (requires python package ``nlopt``). -This is set by the variable ``generator_type`` in ``run_libensemble_on_warpx.py``. -We hereafter assume that all Python modules required are installed and that a WarpX executable is available. - -Run locally -^^^^^^^^^^^ - -Adjust the ``local_specs`` dictionary in ``all_machine_specs.py`` to fix the path to the WarpX executable (and optionally change the number of cores and OpenMP threads), and run - -.. code-block:: sh - - python run_libensemble_on_warpx.py --comms local --nworkers 3 - -This is adapted to a 4-core machine, as it will use: - -- 1 process to run LibEnsemble - -- 1 process (among the 3 workers) to run the generator - -- 2 processes to run 2 concurrent simulations - -Run on Summit at OLCF -^^^^^^^^^^^^^^^^^^^^^ - -- ``cp -r $HOME/warpx/Tools/LibEnsemble/* sim_directory`` -- modify ``run_libensemble_on_warpx.py`` to have ``machine = 'summit'`` -- modify ``all_machine_specs.py`` to put the right path to the WarpX executable -- modify ``summit_submit_mproc.sh`` to set ``LIBE_PLOTS`` to ``false`` and set the project ID -- ``bsub summit_submit_mproc.sh``: - -.. code-block:: sh - - bsub summit_submit_mproc.sh diff --git a/Docs/source/usage/workflows/optimas.rst b/Docs/source/usage/workflows/optimas.rst new file mode 100644 index 00000000000..58e8143b251 --- /dev/null +++ b/Docs/source/usage/workflows/optimas.rst @@ -0,0 +1,11 @@ +.. _optimas: + +Optimizing with Optimas +======================== + +`optimas `__ is an open-source Python library that enables highly scalable parallel optimization, from a typical laptop to exascale HPC systems. +While a WarpX simulation can provide insight in some physics, it remains a single point evaluation in the space of parameters. +If you have a simulation ready for use, but would like to (i) scan over some input parameters uniformly for, e.g., a tolerance study, or (ii) have a random evaluation of the space of input parameters within a given span or (iii) tune some input parameters to optimize an output parameter, e.g., beam emittance, energy spread, etc., optimas provides these capabilities and will take care of tasks monitoring with fault tolerance on multiple platforms (optimas targets modern HPC platforms like Perlmutter and Frontier). + +A more detailed description of optimas is provided in the `optimas documentation `__. +In particular, the online optimas documentation provides `an example optimization with optimas that runs WarpX simulations `__. diff --git a/Tools/LibEnsemble/all_machine_specs.py b/Tools/LibEnsemble/all_machine_specs.py deleted file mode 100644 index 8dbc2879f61..00000000000 --- a/Tools/LibEnsemble/all_machine_specs.py +++ /dev/null @@ -1,26 +0,0 @@ -import os - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It contains a dictionary for machine-specific elements. -""" - - -local_specs = { - 'name': 'local', # Machine name - 'cores': 1, # Number of cores per simulation - 'sim_app': os.environ['HOME'] + '/warpx/Bin/main2d.gnu.TPROF.MPI.OMP.ex', - 'extra_args': '', # extra arguments passed to mpirun/mpiexec at execution - 'OMP_NUM_THREADS': '2', - 'sim_max': 3 # Maximum number of simulations -} - - -summit_specs = { - 'name': 'summit', # Machine name - 'sim_app': os.environ['HOME'] + '/warpx/Bin/main2d.gnu.TPROF.MPI.CUDA.ex', - # extra arguments passed to jsrun at execution - 'extra_args': '-n 1 -a 1 -g 1 -c 1 --bind=packed:1 --smpiargs="-gpu"', - 'OMP_NUM_THREADS': '1', - 'sim_max': 400 # Maximum number of simulations -} diff --git a/Tools/LibEnsemble/plot_results.py b/Tools/LibEnsemble/plot_results.py deleted file mode 100755 index 6c294fe3675..00000000000 --- a/Tools/LibEnsemble/plot_results.py +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env python - -""" -Plotting script for LibEnsemble run on WarpX simulations. -input: A directory which which the script will read the -latest .npy and .pickle files generated by LibEnsemble -""" - -import glob -import os -import pickle -import sys - -import matplotlib.pyplot as plt -import numpy as np - -# 'aposmm' or 'random' -generator_type = 'random' - -print("Reading latest .npy and .pickle files in " + sys.argv[1]) - -# Read latest .npy file, and store the result in results -# * means all if need specific format then *.csv -list_of_files = glob.glob(sys.argv[1] + '*npy') -latest_file = max(list_of_files, key=os.path.getctime) -results = np.load(latest_file) - -# Read latest .pickle file, and store the result in pickle_data -# * means all if need specific format then *.csv -list_of_files = glob.glob(sys.argv[1] + '*pickle') -latest_file = max(list_of_files, key=os.path.getctime) -with open(latest_file, 'rb') as f: - pickle_data = pickle.load(f) -if generator_type == 'aposmm': - nbatches = len(pickle_data[1]['run_order']) - -# Remove un-returned (i.e., not submitted) simulations -results = results[results['returned'] == 1] - -# Re-organize results into a dictionary. Each key is a field in libE_output, -# each value is an array with the value of this field for all WarpX simulations -names = results.dtype.names -results_dict = {} -for i in range(len(names)): - results_dict[names[i]] = np.array([task[i] for task in results]) - -# Sanity checks: exit if a simulation has null charge, as the script -# doesn't handle this (shouldn't be too hard to add though). -if np.min(results_dict['charge'] == 0.0): - sys.exit() -if np.min(results_dict['f'] == 0.0): - sys.exit() - -# Custom: which output parameters are plotted -# as a function of which input parameters -plot_output = ['f', 'emittance', 'charge', 'energy_avg', 'energy_std'] -plot_input = ['ramp_down_1', 'zlens_1', 'adjust_factor', 'ramp_down_2'] - -# For convenience, dictionary with pyplot.ylim -# tuples for all elements in plot_output -d_ylim = { - 'f': (3.e-7, 5.e-3), - 'emittance': (3.e-7, 5.e-3), - 'charge': (1.e-7, 1.e-5), - 'energy_avg': (5.e1, 5.e2), - 'energy_std': (1.e-2, 1.e-1) -} - -# For convenience, dictionary with units for all elements in plot_output -d_yunits = { - 'f': " (a.u.)", - 'emittance': " (m rad)", - 'charge': " (pC/m)", - 'energy_avg': " (MeV)", - 'energy_std': " (%)" -} - -# Print input and output parameters for the optimal run -print("Best run:") -ind_best = np.nanargmin(results_dict['f']) -for key in plot_input + plot_output: - print(key, results_dict[key][ind_best]) -print("charge_f/charge_i ", - results[ind_best]['charge']/np.nanmax(results_dict['charge'])) - -# And now plotting begins -plt.figure(figsize=(16, 10)) -plt.style.use('dark_background') -# Loop over all input parameters in plot_input -for icount, iname in enumerate(plot_input): - # Loop over all output parameters in plot_output - for count, name in enumerate(plot_output): - plt.subplot(len(plot_input), len(plot_output), - len(plot_output)*icount + count+1) - if generator_type == 'aposmm': - # Plot 1 point per run. Runs that are part of the same local - # minimum search are batched together and plotted with the - # same color. - for batch in pickle_data[1]['run_order']: - my_sims = np.isin(results_dict['sim_id'], - pickle_data[1]['run_order'][batch]) - plt.scatter(np.squeeze(results_dict[iname][my_sims]), - np.squeeze(results_dict[name][my_sims]), - s=2) - elif generator_type == 'random': - # Plot 1 point per run. Colorbar stands for sim ID - # (larger value = later run). - plt.scatter(np.squeeze(results_dict[iname]), - np.squeeze(results_dict[name]), - c=results_dict['sim_id'], s=2, cmap='viridis') - cbar = plt.colorbar() - cbar.ax.set_ylabel('start time (arb. units)') - else: - print("generator_type should be either random or aposmm") - sys.exit() - plt.yscale('log') - plt.xlim(np.min(results_dict[iname]), np.max(results_dict[iname])) - plt.ylim(d_ylim[name]) - plt.grid() - plt.title(name) - plt.xlabel(iname) - plt.ylabel(name + d_yunits[name]) - -plt.tight_layout() - -# Save figure -plt.savefig('results.pdf', bbox_inches='tight') -plt.savefig('results.png', bbox_inches='tight') diff --git a/Tools/LibEnsemble/read_sim_output.py b/Tools/LibEnsemble/read_sim_output.py deleted file mode 100644 index 5eabf18030a..00000000000 --- a/Tools/LibEnsemble/read_sim_output.py +++ /dev/null @@ -1,115 +0,0 @@ -import glob -import os -import shutil - -import numpy as np -import scipy.constants as scc -import yt - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It reads the output plotfiles of a simulation and returns beam -quantity relevant for the LibEnsemble optimizations ('f') as well as other -beam quantities for convenience. -""" - -yt.funcs.mylog.setLevel(50) - -def slice_emittance(x, xp, g, b, w): - xpgb = xp * g * b - xav = np.average(x, weights = w) - xpav = np.average(xpgb, weights = w) - xstd2 = np.average((x - xav)**2, weights = w) - xpstd2 = np.average((xpgb - xpav)**2, weights = w) - xp2 = (np.average((x - xav) * (xpgb-xpav), weights = w))**2 - em = np.sqrt(xstd2 * xpstd2 - xp2) - return em - -def _beam_properties(filepath): - """ - Reads plotfile filepath and compute and return beam parameters - - Parameters - ---------- - filepath : path to plotfile to read - """ - - # Read beam quantities from plotfile - ds = yt.load(filepath) - - if ('beam' in [i[0] for i in ds.field_list]): - ad = ds.all_data() - w = ad['beam', 'particle_weight'].v - if (w.shape[0] <= 200): - print('Insufficient particles in ',filepath) - return 0.0, 0.0, 0.0, 0.0 - else: - x = ad['beam', 'particle_position_x'].v - z = ad['beam', 'particle_position_y'].v - ux = ad['beam', 'particle_momentum_x'].v/scc.m_e/scc.c - uy = ad['beam', 'particle_momentum_y'].v/scc.m_e/scc.c - uz = ad['beam', 'particle_momentum_z'].v/scc.m_e/scc.c - - # Compute beam parameters - # Defined like that, the beam charge is > 0. - charge = np.sum(w) * scc.e - gamma = np.sqrt(1. + ux**2 + uy**2 + uz**2) - beta = np.sqrt(1.0 - 1.0 / gamma**2) - energy_MeV = scc.physical_constants["electron mass energy equivalent in MeV"][0] * (gamma - 1.) - energy_avg = np.average(energy_MeV, weights = w) - energy_std = np.sqrt(np.average((energy_MeV - energy_avg)**2, weights = w)) / energy_avg * 100 - nslices = 20 - zslices = np.linspace(np.min(z), np.max(z), nslices+1) - exlist = np.zeros(nslices) - for slicei in range(nslices): - cond = [(z > zslices[slicei]) & (z < zslices[slicei+1])][0] - wslice = w[cond] - if (wslice.shape[0] > 10): - xslice = x[cond] - xpslice = ux[cond]/uz[cond] - gslice = gamma[cond] - bslice = beta[cond] - exlist[slicei] = slice_emittance(xslice, xpslice, gslice, bslice, wslice) - emittance = np.mean(exlist) - return charge, energy_avg, energy_std, emittance - else: - print('No beam particles in ',filepath) - return 0.0, 0.0, 0.0, 0.0 - -def read_sim_output(workdir): - """ - Return optimizing quantity 'f' and other parameters for convenience. - - Parameters - ---------- - workdir : Path to directory where the simulation ran. - """ - # Get beam properties at the beginning of the run - file_list = glob.glob('diags/plotfiles/plt?????') - if (len(file_list) <2): - print(workdir,' did not have final plotfile') - return np.array([np.nan, np.nan, np.nan, np.nan, np.nan]) - file_list.sort() - datafile = file_list[0] - filepath = os.path.join(workdir, datafile) - charge_i, _, _, emittance_i = _beam_properties(filepath) - - # Get beam properties at the end of the run - datafile = file_list[-1] - filepath = os.path.join(workdir, datafile) - charge_f, energy_avg, energy_std, emittance_f = _beam_properties(filepath) - - if (charge_f > 0.0): - # delete simulation results, just to have smaller data - shutil.rmtree('diags') - - # Build a quantity to minimize (f) that encompasses - # emittance AND charge loss 1% charge loss has the - # same impact as doubling the initial emittance. - # we minimize f! - f = emittance_f + emittance_i*(1.-charge_f/charge_i)*100 - warpx_out = np.array([f, energy_std, energy_avg, charge_f, emittance_f]) - - return warpx_out - else: - return np.array([np.nan, np.nan, np.nan, np.nan, np.nan]) diff --git a/Tools/LibEnsemble/requirements.txt b/Tools/LibEnsemble/requirements.txt deleted file mode 100644 index 88217255e85..00000000000 --- a/Tools/LibEnsemble/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -libensemble -matplotlib -nlopt -numpy -pytest -scipy -yt diff --git a/Tools/LibEnsemble/run_libensemble_on_warpx.py b/Tools/LibEnsemble/run_libensemble_on_warpx.py deleted file mode 100644 index d825288dc56..00000000000 --- a/Tools/LibEnsemble/run_libensemble_on_warpx.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It is the entry point script that runs LibEnsemble. Libensemble -then launches WarpX simulations. - -Execute locally via the following command: - python run_libensemble_on_warpx.py --comms local --nworkers 3 -On summit, use the submission script: - bsub summit_submit_mproc.sh - -The number of concurrent evaluations of the objective function will be -nworkers=1 as one worker is for the persistent gen_f. -""" - -# Either 'random' or 'aposmm' -generator_type = 'aposmm' -# Either 'local' or 'summit' -machine = 'local' - -import sys - -import numpy as np - -# Import libEnsemble modules -from libensemble.libE import libE -from warpx_simf import run_warpx # Sim function from current directory - -if generator_type == 'random': - from libensemble.alloc_funcs.give_sim_work_first import ( - give_sim_work_first as alloc_f, - ) - from libensemble.gen_funcs.sampling import uniform_random_sample as gen_f -elif generator_type == 'aposmm': - import libensemble.gen_funcs - libensemble.gen_funcs.rc.aposmm_optimizers = 'nlopt' - from libensemble.alloc_funcs.persistent_aposmm_alloc import ( - persistent_aposmm_alloc as alloc_f, - ) - from libensemble.gen_funcs.persistent_aposmm import aposmm as gen_f -else: - print("you shouldn' hit that") - sys.exit() - -import all_machine_specs -from libensemble import libE_logger -from libensemble.executors.mpi_executor import MPIExecutor -from libensemble.tools import add_unique_random_streams, parse_args, save_libE_output - -# Import machine-specific run parameters -if machine == 'local': - machine_specs = all_machine_specs.local_specs -elif machine == 'summit': - machine_specs = all_machine_specs.summit_specs -else: - print("you shouldn' hit that") - sys.exit() - -libE_logger.set_level('INFO') - -nworkers, is_master, libE_specs, _ = parse_args() - -# Set to full path of warp executable -sim_app = machine_specs['sim_app'] - -# Problem dimension. This is the number of input parameters exposed, -# that LibEnsemble will vary in order to minimize a single output parameter. -n = 4 - -exctr = MPIExecutor(central_mode=True) -exctr.register_calc(full_path=sim_app, calc_type='sim') - -# State the objective function, its arguments, output, and necessary parameters -# (and their sizes). Here, the 'user' field is for the user's (in this case, -# the simulation) convenience. Feel free to use it to pass number of nodes, -# number of ranks per note, time limit per simulation etc. -sim_specs = { - # Function whose output is being minimized. The parallel WarpX run is - # launched from run_WarpX. - 'sim_f': run_warpx, - # Name of input for sim_f, that LibEnsemble is allowed to modify. - # May be a 1D array. - 'in': ['x'], - 'out': [ - # f is the single float output that LibEnsemble minimizes. - ('f', float), - # All parameters below are not used for calculation, - # just output for convenience. - # Final relative energy spread. - ('energy_std', float, (1,)), - # Final average energy, in MeV. - ('energy_avg', float, (1,)), - # Final beam charge. - ('charge', float, (1,)), - # Final beam emittance. - ('emittance', float, (1,)), - # input parameter: length of first downramp. - ('ramp_down_1', float, (1,)), - # input parameter: Length of second downramp. - ('ramp_down_2', float, (1,)), - # input parameter: position of the focusing lens. - ('zlens_1', float, (1,)), - # Relative strength of the lens (1. is from - # back-of-the-envelope calculation) - ('adjust_factor', float, (1,)), - ], - 'user': { - # name of input file - 'input_filename': 'inputs', - # Run timeouts after 3 mins - 'sim_kill_minutes': 3, - # machine-specific parameters - 'machine_specs': machine_specs - } -} - -# State the generating function, its arguments, output, -# and necessary parameters. -if generator_type == 'random': - # Here, the 'user' field is for the user's (in this case, - # the RNG) convenience. - gen_specs = { - # Generator function. Will randomly generate new sim inputs 'x'. - 'gen_f': gen_f, - # Generator input. This is a RNG, no need for inputs. - 'in': [], - 'out': [ - # parameters to input into the simulation. - ('x', float, (n,)) - ], - 'user': { - # Total max number of sims running concurrently. - 'gen_batch_size': nworkers, - # Lower bound for the n parameters. - 'lb': np.array([2.e-3, 2.e-3, 0.005, .1]), - # Upper bound for the n parameters. - 'ub': np.array([2.e-2, 2.e-2, 0.028, 3.]), - } - } - - alloc_specs = { - # Allocator function, decides what a worker should do. - # We use a LibEnsemble allocator. - 'alloc_f': alloc_f, - 'out': [ - ('allocated', bool) - ], - 'user': { - # If true wait for all sims to process before generate more - 'batch_mode': True, - # Only one active generator at a time - 'num_active_gens': 1 - } - } - -elif generator_type == 'aposmm': - # Here, the 'user' field is for the user's (in this case, - # the optimizer) convenience. - gen_specs = { - # Generator function. Will randomly generate new sim inputs 'x'. - 'gen_f': gen_f, - 'in': [], - 'out': [ - # parameters to input into the simulation. - ('x', float, (n,)), - # x scaled to a unique cube. - ('x_on_cube', float, (n,)), - # unique ID of simulation. - ('sim_id', int), - # Whether this point is a local minimum. - ('local_min', bool), - # whether the point is from a local optimization run - # or a random sample point. - ('local_pt', bool) - ], - 'user': { - # Number of sims for initial random sampling. - # Optimizer starts afterwards. - 'initial_sample_size': max(nworkers-1, 1), - # APOSMM/NLOPT optimization method - 'localopt_method': 'LN_BOBYQA', - 'num_pts_first_pass': nworkers, - # Relative tolerance of inputs - 'xtol_rel': 1e-3, - # Absolute tolerance of output 'f'. Determines when - # local optimization stops. - 'ftol_abs': 3e-8, - # Lower bound for the n input parameters. - 'lb': np.array([2.e-3, 2.e-3, 0.005, .1]), - # Upper bound for the n input parameters. - 'ub': np.array([2.e-2, 2.e-2, 0.028, 3.]), - } - } - - alloc_specs = { - # Allocator function, decides what a worker should do. - # We use a LibEnsemble allocator. - 'alloc_f': alloc_f, - 'out': [('given_back', bool)], - 'user': {}} - -else: - print("you shouldn' hit that") - sys.exit() - -# Save H to file every N simulation evaluations -libE_specs['save_every_k_sims'] = 100 -# Sim directory to be copied for each worker -libE_specs['sim_input_dir'] = 'sim' - -sim_max = machine_specs['sim_max'] # Maximum number of simulations -exit_criteria = {'sim_max': sim_max} # Exit after running sim_max simulations - -# Create a different random number stream for each worker and the manager -persis_info = add_unique_random_streams({}, nworkers + 1) - -# Run LibEnsemble, and store results in history array H -H, persis_info, flag = libE(sim_specs, gen_specs, exit_criteria, - persis_info, alloc_specs, libE_specs) - -# Save results to numpy file -if is_master: - save_libE_output(H, persis_info, __file__, nworkers) diff --git a/Tools/LibEnsemble/sim/inputs b/Tools/LibEnsemble/sim/inputs deleted file mode 100644 index 03e9968e5b0..00000000000 --- a/Tools/LibEnsemble/sim/inputs +++ /dev/null @@ -1,215 +0,0 @@ -################################# -########## MESH PATCH ########### -################################# -amr.max_level = 0 - -################################# -######### BOX PARAMETERS ######## -################################# -# warpx.zmax_plasma_to_compute_max_step = 0.33 -warpx.zmax_plasma_to_compute_max_step = -1. -# max_step = 3150 - -amr.n_cell = 32 2048 # 64 3072 - -amr.max_grid_size = 4096 -amr.blocking_factor = 16 - -geometry.dims = 2 -geometry.is_periodic = 0 0 -# physical domain when running in the lab frame -geometry.prob_lo = -0.00024190484157981564 -0.00016126989438654374 -geometry.prob_hi = 0.00024190484157981564 0.0 - -################################# -############ NUMERICS ########### -################################# -warpx.verbose = 1 -algo.current_deposition = esirkepov -algo.charge_deposition = standard -#algo.field_gathering = standard -algo.field_gathering = momentum-conserving -algo.particle_pusher = vay -algo.maxwell_solver = ckc -algo.load_balance_intervals = -1 -algo.particle_shape = 3 -warpx.use_filter = 1 -warpx.filter_npass_each_dir = 1 4 -warpx.cfl = .9999 -warpx.do_pml = 0 -warpx.do_dynamic_scheduling = 1 -# Moving window -warpx.do_moving_window = 1 -warpx.moving_window_dir = z -warpx.moving_window_v = 1.0 # in units of the speed of light - -################################# -####### BOOST PARAMETERS ######## -################################# -warpx.gamma_boost = 30.0 -warpx.boost_direction = z - -################################# -############ PLASMA ############# -################################# -particles.species_names = electrons ions electrons2 ions2 beam - -particles.use_fdtd_nci_corr = 1 -particles.rigid_injected_species = beam - -electrons.charge = -q_e -electrons.mass = m_e -electrons.injection_style = NUniformPerCell -electrons.num_particles_per_cell_each_dim = 1 1 -electrons.momentum_distribution_type = "at_rest" -electrons.xmin = -150.e-6 -electrons.xmax = 150.e-6 -electrons.ymin = -150.e-6 -electrons.ymax = 150.e-6 -electrons.zmin = 0.0 -electrons.zmax = 0.32 -electrons.profile = "predefined" -electrons.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -electrons.predefined_profile_params = 0.0 .02 .297 .003 40.e-6 1.7e23 -electrons.do_continuous_injection = 1 - -ions.charge = q_e -ions.mass = m_p -ions.injection_style = NUniformPerCell -ions.num_particles_per_cell_each_dim = 1 1 -ions.momentum_distribution_type = "at_rest" -ions.xmin = -150.e-6 -ions.xmax = 150.e-6 -ions.ymin = -150.e-6 -ions.ymax = 150.e-6 -ions.zmin = 0.0 -ions.zmax = 0.32 -ions.profile = "predefined" -ions.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -ions.predefined_profile_params = 0.0 .02 .297 .003 40.e-6 1.7e23 -ions.do_continuous_injection = 1 - -electrons2.charge = -q_e -electrons2.mass = m_e -electrons2.injection_style = NUniformPerCell -electrons2.num_particles_per_cell_each_dim = 1 1 1 -electrons2.momentum_distribution_type = "at_rest" -electrons2.xmin = -150.e-6 -electrons2.xmax = 150.e-6 -electrons2.ymin = -150.e-6 -electrons2.ymax = 150.e-6 -electrons2.zmin = 0.3485 -electrons2.zmax = 0.6685 -electrons2.profile = "predefined" -electrons2.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -electrons2.predefined_profile_params = 0.3485 .02 .297 .003 40.e-6 1.7e23 -electrons2.do_continuous_injection = 1 - -ions2.charge = q_e -ions2.mass = m_p -ions2.injection_style = NUniformPerCell -ions2.num_particles_per_cell_each_dim = 1 1 1 -ions2.momentum_distribution_type = "at_rest" -ions2.xmin = -150.e-6 -ions2.xmax = 150.e-6 -ions2.ymin = -150.e-6 -ions2.ymax = 150.e-6 -ions2.zmin = 0.3485 -ions2.zmax = 0.6685 -ions2.profile = "predefined" -ions2.predefined_profile_name = "parabolic_channel" -# predefined_profile_params = z_start ramp_up plateau ramp_down rc n0 -ions2.predefined_profile_params = 0.3485 .02 .297 .003 40.e-6 1.7e23 -ions2.do_continuous_injection = 1 - -beam.charge = -q_e -beam.mass = m_e -beam.injection_style = "gaussian_beam" -beam.x_rms = 6.e-7 -beam.y_rms = 6.e-7 -beam.z_rms = 3.e-6 -beam.x_cut = 3. -beam.y_cut = 3. -beam.z_cut = 2. -beam.x_m = 0. -beam.y_m = 0. -beam.z_m = -98.e-6 -beam.npart = 100000 -beam.q_tot = -0.9e-12 -beam.momentum_distribution_type = "gaussian" -beam.ux_m = 0. -beam.uy_m = 0. -beam.uz_m = 1956.9469069265976 -beam.ux_th = 0.4166666666666667 -beam.uy_th = 0. -beam.uz_th = 39.138943248532286 -beam.zinject_plane = 0.02 -beam.rigid_advance = true - -################################# -######### Lens Mirror ########### -################################# - -my_constants.ga = 13330 -my_constants.zlen = 0.3375 -my_constants.dlen = 0.019 -my_constants.wlen = 0.002 -my_constants.mcce = 510999. -my_constants.gab = 30.0 -my_constants.vb = 299625860.344456 -my_constants.adjust_factor = 1. - -particles.E_ext_particle_init_style = parse_E_ext_particle_function -particles.Ex_external_particle_function(x,y,z,t) = "( adjust_factor * gab * 2 * mcce * ga * x / (wlen * dlen) ) * ((gab*(z+vb*t))>=zlen) * ((gab*(z+vb*t))<=(zlen+wlen))" -particles.Ey_external_particle_function(x,y,z,t) = "0." -particles.Ez_external_particle_function(x,y,z,t) = "0." - -particles.B_ext_particle_init_style = parse_B_ext_particle_function -particles.Bx_external_particle_function(x,y,z,t) = "0." -particles.By_external_particle_function(x,y,z,t) = "( - adjust_factor * vb * gab * 2 * mcce * ga * x / (wlen * dlen * clight * clight) ) * ((gab*(z+vb*t))>=zlen) * ((gab*(z+vb*t))<=(zlen+wlen))" -particles.Bz_external_particle_function(x,y,z,t) = "0." - -warpx.num_mirrors=2 -warpx.mirror_z = 0.321 0.6695 -warpx.mirror_z_width = 8.0e-6 8.0e-6 -warpx.mirror_z_npoints = 4 4 - -################################# -############# LASER ############# -################################# -lasers.names = laser1 laser2 - -laser1.profile = Gaussian -laser1.position = 0. 0. -1.e-9 # This point is on the laser plane -laser1.direction = 0. 0. 1. # The plane normal direction -laser1.polarization = 0. 1. 0. # The main polarization vector -laser1.e_max = 6.82274e12 # Maximum amplitude of the laser field (in V/m) -laser1.profile_waist = 50.e-6 # The waist of the laser (in meters) -laser1.profile_duration = 7.33841e-14 # The duration of the laser (in seconds) -laser1.profile_t_peak = 1.46764864e-13 # The time at which the laser reaches its peak (in seconds) -laser1.profile_focal_distance = 0.00875 # Focal distance from the antenna (in meters) -laser1.wavelength = 0.8e-6 # The wavelength of the laser (in meters) - -laser2.profile = Gaussian -laser2.position = 0. 0. 0.348499999 # This point is on the laser plane -laser2.direction = 0. 0. 1. # The plane normal direction -laser2.polarization = 0. 1. 0. # The main polarization vector -laser2.e_max = 6.82274e12 # Maximum amplitude of the laser field (in V/m) -laser2.profile_waist = 50.e-6 # The waist of the laser (in meters) -laser2.profile_duration = 7.33841e-14 # The duration of the laser (in seconds) -laser2.profile_t_peak = 1.1626176366295598e-09 # The time at which the laser reaches its peak (in seconds) -laser2.profile_focal_distance = 0.00875 # Focal distance from the antenna (in meters) -laser2.wavelength = 0.8e-6 # The wavelength of the laser (in meters) -laser2.do_continuous_injection = 1 # Lasers are initialized outside of initial box - -diagnostics.diags_names = diag -diag.file_prefix = diags/plotfiles/plt -diag.diag_type = Full -diag.intervals = 10000 -diag.species = electrons electrons2 beam -diag.electrons.variables = w -diag.electrons2.variables = w -diag.beam.variables = w ux uy uz diff --git a/Tools/LibEnsemble/summit_submit_mproc.sh b/Tools/LibEnsemble/summit_submit_mproc.sh deleted file mode 100755 index d83c57b0e1e..00000000000 --- a/Tools/LibEnsemble/summit_submit_mproc.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/bash -x -#BSUB -P -#BSUB -J libe_mproc -#BSUB -W 00:30 -#BSUB -nnodes 4 -#BSUB -alloc_flags "smt1" - -# Script to run libEnsemble using multiprocessing on launch nodes. -# Assumes Conda environment is set up. - -# To be run with central job management -# - Manager and workers run on launch node. -# - Workers submit tasks to the nodes in the job available. - -# Name of calling script- -export EXE=run_libensemble_on_warpx.py - -# Communication Method -export COMMS="--comms local" - -# Number of workers. -export NWORKERS="--nworkers 24" - -# Wallclock for libE. Slightly smaller than job wallclock -#export LIBE_WALLCLOCK=15 # Optional if pass to script - -# Name of Conda environment -export CONDA_ENV_NAME=libensemble - -export LIBE_PLOTS=true # Require plot scripts in $PLOT_DIR (see at end) -export PLOT_DIR=.. - -# Need these if not already loaded - -# module load python -# module load gcc -# module load cuda - -module unload xalt - -# Activate conda environment -export PYTHONNOUSERSITE=1 -. activate $CONDA_ENV_NAME - -# hash -d python # Check pick up python in conda env -hash -r # Check no commands hashed (pip/python...) - -# Launch libE. -#python $EXE $NUM_WORKERS $LIBE_WALLCLOCK > out.txt 2>&1 -python $EXE $COMMS $NWORKERS > out.txt 2>&1 - -if [[ $LIBE_PLOTS = "true" ]]; then - python $PLOT_DIR/plot_libe_calcs_util_v_time.py - python $PLOT_DIR/plot_libe_runs_util_v_time.py - python $PLOT_DIR/plot_libe_histogram.py -fi diff --git a/Tools/LibEnsemble/warpx_simf.py b/Tools/LibEnsemble/warpx_simf.py deleted file mode 100644 index f310dd0f95e..00000000000 --- a/Tools/LibEnsemble/warpx_simf.py +++ /dev/null @@ -1,110 +0,0 @@ -import os -import time - -import numpy as np -from libensemble.executors.executor import Executor -from libensemble.message_numbers import TASK_FAILED, WORKER_DONE -from read_sim_output import read_sim_output -from write_sim_input import write_sim_input - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It defines a sim_f function that takes LibEnsemble history and -input parameters, run a WarpX simulation and returns 'f'. -""" - - -def run_warpx(H, persis_info, sim_specs, libE_info): - """ - This function runs a WarpX simulation and returns quantity 'f' as well as - other physical quantities measured in the run for convenience. Status check - is done periodically on the simulation, provided by LibEnsemble. - """ - - # Setting up variables needed for input and output - # keys = variable names - # x = variable values - # libE_output = what will be returned to libE - - calc_status = 0 # Returns to worker - - input_file = sim_specs['user']['input_filename'] - time_limit = sim_specs['user']['sim_kill_minutes'] * 60.0 - machine_specs = sim_specs['user']['machine_specs'] - - exctr = Executor.executor # Get Executor - - # Modify WarpX input file with input parameters calculated by gen_f - # and passed to this sim_f. - write_sim_input(input_file, H['x']) - - # Passed to command line in addition to the executable. - # Here, only input file - app_args = input_file - os.environ["OMP_NUM_THREADS"] = machine_specs['OMP_NUM_THREADS'] - - # Launch the executor to actually run the WarpX simulation - if machine_specs['name'] == 'summit': - task = exctr.submit(calc_type='sim', - extra_args=machine_specs['extra_args'], - app_args=app_args, - stdout='out.txt', - stderr='err.txt', - wait_on_run=True) - else: - task = exctr.submit(calc_type='sim', - num_procs=machine_specs['cores'], - app_args=app_args, - stdout='out.txt', - stderr='err.txt', - wait_on_run=True) - - # Periodically check the status of the simulation - poll_interval = 1 # secs - while(not task.finished): - time.sleep(poll_interval) - task.poll() - if task.runtime > time_limit: - task.kill() # Timeout - - # Set calc_status with optional prints. - if task.finished: - if task.state == 'FINISHED': - calc_status = WORKER_DONE - elif task.state == 'FAILED': - print("Warning: Task {} failed: Error code {}" - .format(task.name, task.errcode)) - calc_status = TASK_FAILED - elif task.state == 'USER_KILLED': - print("Warning: Task {} has been killed" - .format(task.name)) - else: - print("Warning: Task {} in unknown state {}. Error code {}" - .format(task.name, task.state, task.errcode)) - - # Safety - time.sleep(0.2) - - # Get output from a run and delete output files - warpx_out = read_sim_output(task.workdir) - - # Excluding results - NaN - from runs where beam was lost - if (warpx_out[0] != warpx_out[0]): - print(task.workdir, ' output led to NaN values (beam was lost or run did not finish)') - - # Pass the sim output values to LibEnsemble. - # When optimization is ON, 'f' is then passed to the generating function - # gen_f to generate new inputs for next runs. - # All other parameters are here just for convenience. - libE_output = np.zeros(1, dtype=sim_specs['out']) - libE_output['f'] = warpx_out[0] - libE_output['energy_std'] = warpx_out[1] - libE_output['energy_avg'] = warpx_out[2] - libE_output['charge'] = warpx_out[3] - libE_output['emittance'] = warpx_out[4] - libE_output['ramp_down_1'] = H['x'][0][0] - libE_output['ramp_down_2'] = H['x'][0][1] - libE_output['zlens_1'] = H['x'][0][2] - libE_output['adjust_factor'] = H['x'][0][3] - - return libE_output, persis_info, calc_status diff --git a/Tools/LibEnsemble/write_sim_input.py b/Tools/LibEnsemble/write_sim_input.py deleted file mode 100644 index e4a6398689e..00000000000 --- a/Tools/LibEnsemble/write_sim_input.py +++ /dev/null @@ -1,118 +0,0 @@ -import re - -import scipy.constants as scc - -""" -This file is part of the suite of scripts to use LibEnsemble on top of WarpX -simulations. It provides functions to modify some parameters in a WarpX -input file. -""" - - -def _set_value(str_text, str_line, val): - """ - This function search for str_line in str_text and overwrites the whole - matching line with str(val) - """ - str_text = re.sub('\n' + str_line + '.*', - '\n' + str_line + str(val), - str_text) - return str_text - - -def write_sim_input(input_file, x_values): - """ - This function modifies input_file to replace the values of some parameters - by values provided in x_values. - - Parameters - ---------- - input_file : WarpX input file. - x_values : Specific value of input x for this run. - """ - - # Parameters exposed to optimization - ramp_down_1 = x_values[0][0] - ramp_down_2 = x_values[0][1] - zlens_1 = x_values[0][2] - adjust_factor = x_values[0][3] - - # Fixed parameters - ramp_up_1 = 0.02 - plateau_1 = 0.297 - ramp_up_2 = ramp_up_1 - plateau_2 = plateau_1 - gap_12 = .0285 - - end_stage_1 = ramp_up_1 + plateau_1 + ramp_down_1 - beg_stage_2 = end_stage_1 + gap_12 - end_stage_2 = beg_stage_2 + ramp_up_2 + plateau_2 + ramp_down_2 - - # Updating mirrors position - mirror_z_1 = end_stage_1 + 1.0e-3 - mirror_z_2 = end_stage_2 + 1.0e-3 - - # Updating laser 2 configuration - z_antenna_2 = beg_stage_2 - 1.0e-9 #with value used for stage 1 - z_t_peak_2 = z_antenna_2 / scc.c + 2 * 7.33841e-14 # using pulse duration - - # End simulation when beam has just escaped the last stage - gamma_b = 30. - zmax_stop_run = end_stage_2 - 55.e-6 * gamma_b**2 * 2. - - with open(input_file) as file_handler: - output_text = file_handler.read() - - # Set end of stage 1 - output_text = _set_value( - output_text, 'electrons.zmax = ', str(end_stage_1)) - output_text = _set_value( - output_text, 'ions.zmax = ', str(end_stage_1)) - # Set length of final downramp of stage 1 - output_text = _set_value( - output_text, 'electrons.predefined_profile_params = ', - '0.0 .02 .297 ' + str(ramp_down_1) + ' 40.e-6 1.7e23') - output_text = _set_value( - output_text, 'ions.predefined_profile_params = ', - '0.0 .02 .297 ' + str(ramp_down_1) + ' 40.e-6 1.7e23') - # Set position of lens - output_text = _set_value( - output_text, 'my_constants.zlen = ', str(end_stage_1 + zlens_1)) - # Set beginning of stage 2 - output_text = _set_value( - output_text, 'electrons2.zmin = ', str(beg_stage_2)) - output_text = _set_value( - output_text, 'ions2.zmin = ', str(beg_stage_2)) - # Set end of stage 2 - output_text = _set_value( - output_text, 'electrons2.zmax = ', str(end_stage_2)) - output_text = _set_value( - output_text, 'ions2.zmax = ', str(end_stage_2)) - # Set length of final downramp of stage 2 - output_text = _set_value( - output_text, 'electrons2.predefined_profile_params = ', - str(beg_stage_2) + ' .02 .297 ' + str(ramp_down_2) + ' 40.e-6 1.7e23') - output_text = _set_value( - output_text, 'ions2.predefined_profile_params = ', - str(beg_stage_2) + ' .02 .297 ' + str(ramp_down_2) + ' 40.e-6 1.7e23') - # Set adjustment factor on lens strength - output_text = _set_value( - output_text, 'my_constants.adjust_factor = ', str(adjust_factor)) - # Set when to stop the run - output_text = _set_value( - output_text, 'warpx.zmax_plasma_to_compute_max_step = ', - str(zmax_stop_run)) - # Set mirrors position - output_text = _set_value( - output_text, 'warpx.mirror_z = ', str(mirror_z_1)+' '+str(mirror_z_2)) - # Set laser2 position - output_text = _set_value( - output_text, 'laser2.position = ', '0. 0. '+ str(z_antenna_2)) - # Set laser2 time until peak field is reached - output_text = _set_value( - output_text, 'laser2.profile_t_peak = ', str(z_t_peak_2)) - -# Write new input file - fout = open(input_file, 'w') - fout.write(output_text) - fout.close() From 5e95faf0659842c487c0400d4a882611806a027e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 12 Apr 2024 14:12:33 -0700 Subject: [PATCH 047/190] Always update guard cells in WarpXSumGuardCells (#4840) * Always update guard cells in WarpXSumGuardCells * Apply suggestions from code review * Update benchmarks * Update capacitive discharge test * Update benchmarks * Update Regression/WarpX-tests.ini * Update Examples/Tests/ohm_solver_EM_modes/analysis_rz.py * Update Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * Update benchmark * Update box position * Update checksum --------- Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- .../capacitive_discharge/analysis_1d.py | 66 +++++++++---------- .../capacitive_discharge/analysis_dsmc.py | 66 +++++++++---------- .../Tests/AcceleratorLattice/inputs_quad_3d | 4 +- .../Tests/ohm_solver_EM_modes/analysis_rz.py | 2 +- .../LaserAccelerationBoost.json | 42 ++++++------ .../LaserAcceleration_BTD.json | 50 +++++++------- .../LaserInjectionFromLASYFile_RZ.json | 14 ++-- .../LaserInjectionFromRZLASYFile.json | 16 ++--- .../benchmarks_json/Python_dsmc_1d.json | 38 +++++------ .../Python_ohms_law_solver_EM_modes_rz.json | 14 ++-- ...s_law_solver_magnetic_reconnection_2d.json | 20 +++--- .../Python_prev_positions.json | 24 +++---- .../hard_edged_quadrupoles.json | 24 +++---- .../benchmarks_json/magnetostatic_eb_3d.json | 32 ++++----- .../particle_thermal_boundary.json | 16 ++--- .../reduced_diags_single_precision.json | 54 +++++++-------- Source/Parallelization/WarpXSumGuardCells.H | 12 +--- Source/Parallelization/WarpXSumGuardCells.cpp | 18 +---- 18 files changed, 245 insertions(+), 267 deletions(-) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_1d.py b/Examples/Physics_applications/capacitive_discharge/analysis_1d.py index c6529dabd82..29b5272d8b1 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_1d.py @@ -5,39 +5,39 @@ import numpy as np ref_density = np.array([ - 1.27953969e+14, 2.23553999e+14, 2.55384510e+14, 2.55663110e+14, - 2.55805760e+14, 2.55812087e+14, 2.55813911e+14, 2.55754104e+14, - 2.55929601e+14, 2.56085472e+14, 2.55932867e+14, 2.55828121e+14, - 2.55901711e+14, 2.55985074e+14, 2.56182697e+14, 2.56446847e+14, - 2.56483696e+14, 2.56301187e+14, 2.56245301e+14, 2.56797584e+14, - 2.57257907e+14, 2.57023627e+14, 2.56500876e+14, 2.56106851e+14, - 2.56283546e+14, 2.56723967e+14, 2.56960855e+14, 2.56825486e+14, - 2.56674669e+14, 2.56567191e+14, 2.56310927e+14, 2.56361171e+14, - 2.56692197e+14, 2.56743606e+14, 2.56653108e+14, 2.56883854e+14, - 2.56763228e+14, 2.56343726e+14, 2.56385489e+14, 2.56570110e+14, - 2.56538112e+14, 2.56472179e+14, 2.56322922e+14, 2.56195384e+14, - 2.56474576e+14, 2.56764233e+14, 2.56533016e+14, 2.56257170e+14, - 2.56362463e+14, 2.56363962e+14, 2.56311292e+14, 2.56678788e+14, - 2.57061138e+14, 2.56785892e+14, 2.56406603e+14, 2.56334908e+14, - 2.56120051e+14, 2.56003269e+14, 2.56132187e+14, 2.56329572e+14, - 2.56535713e+14, 2.56708950e+14, 2.56661860e+14, 2.56448986e+14, - 2.56386823e+14, 2.56233660e+14, 2.56137632e+14, 2.56206263e+14, - 2.56364996e+14, 2.56483536e+14, 2.56308741e+14, 2.56447231e+14, - 2.56896301e+14, 2.56691405e+14, 2.56170780e+14, 2.56122216e+14, - 2.56427399e+14, 2.56897558e+14, 2.56928868e+14, 2.56659033e+14, - 2.56749993e+14, 2.56952497e+14, 2.56798907e+14, 2.56377081e+14, - 2.56453057e+14, 2.56796632e+14, 2.56944576e+14, 2.57248469e+14, - 2.57279426e+14, 2.56849516e+14, 2.56601834e+14, 2.56850545e+14, - 2.56953072e+14, 2.56442586e+14, 2.56329006e+14, 2.56790661e+14, - 2.57083582e+14, 2.57075550e+14, 2.56719615e+14, 2.56220486e+14, - 2.56222323e+14, 2.56547365e+14, 2.56499423e+14, 2.56434041e+14, - 2.56378587e+14, 2.56249892e+14, 2.56380492e+14, 2.56504513e+14, - 2.56337631e+14, 2.56204891e+14, 2.56325116e+14, 2.56297798e+14, - 2.56112782e+14, 2.56054218e+14, 2.56320120e+14, 2.56580938e+14, - 2.56446800e+14, 2.56267011e+14, 2.56372853e+14, 2.56617592e+14, - 2.56630745e+14, 2.56615242e+14, 2.56625259e+14, 2.56561320e+14, - 2.56640072e+14, 2.56693273e+14, 2.56613237e+14, 2.24169847e+14, - 1.27683197e+14 + 1.27989677e+14, 2.23601330e+14, 2.55400265e+14, 2.55664972e+14, + 2.55806841e+14, 2.55806052e+14, 2.55815865e+14, 2.55755151e+14, + 2.55920582e+14, 2.56078972e+14, 2.55933056e+14, 2.55840862e+14, + 2.56043990e+14, 2.56247464e+14, 2.56289153e+14, 2.56277109e+14, + 2.56242679e+14, 2.56328850e+14, 2.56373595e+14, 2.56395979e+14, + 2.56451842e+14, 2.56601614e+14, 2.56530735e+14, 2.56113125e+14, + 2.56207966e+14, 2.56474622e+14, 2.56378743e+14, 2.56230373e+14, + 2.56271563e+14, 2.56317460e+14, 2.56137709e+14, 2.56104767e+14, + 2.56244330e+14, 2.56188574e+14, 2.56070283e+14, 2.56212277e+14, + 2.56374214e+14, 2.56309276e+14, 2.56199679e+14, 2.56227549e+14, + 2.56434824e+14, 2.56578649e+14, 2.56418136e+14, 2.56781877e+14, + 2.57464056e+14, 2.57125505e+14, 2.56477914e+14, 2.56355343e+14, + 2.56599651e+14, 2.56955102e+14, 2.56988933e+14, 2.56954167e+14, + 2.56848084e+14, 2.56278229e+14, 2.55969967e+14, 2.56075186e+14, + 2.56011026e+14, 2.56054081e+14, 2.56203573e+14, 2.56131373e+14, + 2.56181410e+14, 2.56460942e+14, 2.56519893e+14, 2.56394874e+14, + 2.56273725e+14, 2.56263570e+14, 2.56463482e+14, 2.56541354e+14, + 2.56467160e+14, 2.56523576e+14, 2.56760359e+14, 2.56899283e+14, + 2.56889333e+14, 2.56984101e+14, 2.57009449e+14, 2.57017095e+14, + 2.57027915e+14, 2.57002409e+14, 2.56938087e+14, 2.56604366e+14, + 2.56392558e+14, 2.56481669e+14, 2.56337220e+14, 2.56107563e+14, + 2.56286144e+14, 2.56451638e+14, 2.56547674e+14, 2.56846908e+14, + 2.56725251e+14, 2.56321829e+14, 2.56387226e+14, 2.56750085e+14, + 2.56775478e+14, 2.56589827e+14, 2.56723109e+14, 2.56698688e+14, + 2.56674607e+14, 2.57128951e+14, 2.57144972e+14, 2.56602827e+14, + 2.56564415e+14, 2.56824494e+14, 2.56459673e+14, 2.56106618e+14, + 2.56442627e+14, 2.57106509e+14, 2.57468066e+14, 2.57155708e+14, + 2.56501993e+14, 2.56062276e+14, 2.56109738e+14, 2.56284272e+14, + 2.56320557e+14, 2.56253594e+14, 2.56155365e+14, 2.56044920e+14, + 2.55994560e+14, 2.56166612e+14, 2.56327633e+14, 2.56390898e+14, + 2.56474891e+14, 2.56669566e+14, 2.56646041e+14, 2.56264156e+14, + 2.56041610e+14, 2.56041551e+14, 2.56088641e+14, 2.23853646e+14, + 1.27580207e+14 ]) density_data = np.load( 'ion_density_case_1.npy' ) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py index a7a76be46ad..94a08186183 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -18,39 +18,39 @@ my_check = checksumAPI.evaluate_checksum(test_name, fn, do_particles=True) ref_density = np.array([ - 1.27957355e+14, 2.23554080e+14, 2.55373436e+14, 2.55659492e+14, - 2.55814670e+14, 2.55818418e+14, 2.55811882e+14, 2.55742272e+14, - 2.55912888e+14, 2.56086072e+14, 2.55944486e+14, 2.55830183e+14, - 2.55909337e+14, 2.56008609e+14, 2.56205930e+14, 2.56421940e+14, - 2.56369990e+14, 2.56151020e+14, 2.55925823e+14, 2.55924941e+14, - 2.56067211e+14, 2.56264104e+14, 2.56435035e+14, 2.56543804e+14, - 2.56715146e+14, 2.56639305e+14, 2.56509438e+14, 2.56478881e+14, - 2.56406748e+14, 2.56194832e+14, 2.56126186e+14, 2.56442221e+14, - 2.56603784e+14, 2.56592554e+14, 2.56475838e+14, 2.56304135e+14, - 2.56310993e+14, 2.56298883e+14, 2.56386742e+14, 2.56555670e+14, - 2.56588013e+14, 2.56851444e+14, 2.56928531e+14, 2.56637559e+14, - 2.56678652e+14, 2.56827322e+14, 2.56630197e+14, 2.56295404e+14, - 2.56285079e+14, 2.56558116e+14, 2.56676094e+14, 2.56577780e+14, - 2.56599749e+14, 2.56540500e+14, 2.56292984e+14, 2.56230350e+14, - 2.56363607e+14, 2.56553909e+14, 2.56501054e+14, 2.56249684e+14, - 2.56280268e+14, 2.56558208e+14, 2.56437837e+14, 2.56152650e+14, - 2.56143349e+14, 2.56067330e+14, 2.56020624e+14, 2.56039223e+14, - 2.56306096e+14, 2.56693084e+14, 2.56649778e+14, 2.56589778e+14, - 2.56594097e+14, 2.56368788e+14, 2.56290090e+14, 2.56420940e+14, - 2.56581419e+14, 2.56642649e+14, 2.56426887e+14, 2.56360122e+14, - 2.56573424e+14, 2.56679138e+14, 2.56488767e+14, 2.56217444e+14, - 2.56353118e+14, 2.56640765e+14, 2.56809490e+14, 2.56933226e+14, - 2.56633538e+14, 2.56203430e+14, 2.56202958e+14, 2.56564020e+14, - 2.56816347e+14, 2.56709830e+14, 2.56557382e+14, 2.56573904e+14, - 2.56745541e+14, 2.56784430e+14, 2.56580054e+14, 2.56210130e+14, - 2.56271415e+14, 2.56821160e+14, 2.56703292e+14, 2.56169296e+14, - 2.56166549e+14, 2.56467777e+14, 2.56573240e+14, 2.56437594e+14, - 2.56253730e+14, 2.56176123e+14, 2.56351125e+14, 2.56569916e+14, - 2.56761101e+14, 2.56891411e+14, 2.56628312e+14, 2.56180062e+14, - 2.56063564e+14, 2.56189728e+14, 2.56609454e+14, 2.57263643e+14, - 2.57097673e+14, 2.56666761e+14, 2.56622585e+14, 2.56432378e+14, - 2.56386718e+14, 2.56734491e+14, 2.57042448e+14, 2.24471147e+14, - 1.27720853e+14 + 1.27945114e+14, 2.23583642e+14, 2.55392166e+14, 2.55655614e+14, + 2.55811304e+14, 2.55813038e+14, 2.55817129e+14, 2.55752279e+14, + 2.55912208e+14, 2.56079483e+14, 2.55932916e+14, 2.55822911e+14, + 2.55910910e+14, 2.55995538e+14, 2.56185960e+14, 2.56518490e+14, + 2.56843281e+14, 2.56847244e+14, 2.56317496e+14, 2.55995033e+14, + 2.56036918e+14, 2.56185305e+14, 2.56473454e+14, 2.56612720e+14, + 2.56460349e+14, 2.56162495e+14, 2.56154687e+14, 2.56351568e+14, + 2.56518116e+14, 2.56589716e+14, 2.56491198e+14, 2.56462652e+14, + 2.56475832e+14, 2.56502009e+14, 2.56512082e+14, 2.56400671e+14, + 2.56317027e+14, 2.56348890e+14, 2.56604754e+14, 2.56846626e+14, + 2.56749617e+14, 2.56563946e+14, 2.56508159e+14, 2.56706913e+14, + 2.56921135e+14, 2.56772946e+14, 2.56509222e+14, 2.56556290e+14, + 2.56632138e+14, 2.56452482e+14, 2.56371644e+14, 2.56363151e+14, + 2.56471549e+14, 2.56541772e+14, 2.56342768e+14, 2.56345629e+14, + 2.56415143e+14, 2.56259609e+14, 2.56264787e+14, 2.56619762e+14, + 2.56992843e+14, 2.57102387e+14, 2.56719391e+14, 2.56255624e+14, + 2.56276638e+14, 2.56350832e+14, 2.56295294e+14, 2.56153556e+14, + 2.56170347e+14, 2.56246380e+14, 2.56088509e+14, 2.56121028e+14, + 2.56423889e+14, 2.56597963e+14, 2.56470310e+14, 2.56419968e+14, + 2.56625715e+14, 2.56711965e+14, 2.56651610e+14, 2.56750226e+14, + 2.56886322e+14, 2.56839170e+14, 2.56469219e+14, 2.56254127e+14, + 2.56422126e+14, 2.56381502e+14, 2.56283046e+14, 2.56358078e+14, + 2.56454503e+14, 2.56810744e+14, 2.57281306e+14, 2.57314168e+14, + 2.57216217e+14, 2.57166645e+14, 2.56969293e+14, 2.56763128e+14, + 2.56479604e+14, 2.56454515e+14, 2.56670498e+14, 2.56378555e+14, + 2.56177400e+14, 2.56510462e+14, 2.56512126e+14, 2.56391975e+14, + 2.56717395e+14, 2.57024288e+14, 2.56901669e+14, 2.56632950e+14, + 2.56470315e+14, 2.56530515e+14, 2.56839839e+14, 2.57084249e+14, + 2.56897794e+14, 2.56360159e+14, 2.56251337e+14, 2.56628550e+14, + 2.56743475e+14, 2.56616083e+14, 2.56830920e+14, 2.57164423e+14, + 2.56868182e+14, 2.56749114e+14, 2.57108990e+14, 2.56791445e+14, + 2.56203737e+14, 2.56114492e+14, 2.56630346e+14, 2.24843400e+14, + 1.28200732e+14 ]) density_data = np.load( 'ion_density_case_1.npy' ) diff --git a/Examples/Tests/AcceleratorLattice/inputs_quad_3d b/Examples/Tests/AcceleratorLattice/inputs_quad_3d index 2bc98f64118..d62b9b45a0e 100644 --- a/Examples/Tests/AcceleratorLattice/inputs_quad_3d +++ b/Examples/Tests/AcceleratorLattice/inputs_quad_3d @@ -2,8 +2,8 @@ max_step = 50 amr.n_cell = 8 8 32 amr.max_level = 0 geometry.dims = 3 -geometry.prob_lo = -0.2 -0.2 0. -geometry.prob_hi = +0.2 +0.2 1.6 +geometry.prob_lo = -0.2 -0.2 -0.025 +geometry.prob_hi = +0.2 +0.2 1.575 warpx.const_dt = 1e-9 warpx.do_electrostatic = labframe diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py index b96bc3b4629..de7c9966b20 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py @@ -153,7 +153,7 @@ def process(it): amps = np.abs(F_kw[2, 1, len(kz)//2-2:len(kz)//2+2]) print("Amplitude sample: ", amps) assert np.allclose( - amps, np.array([61.50945919, 19.74831134, 101.01820349, 10.8974811]) + amps, np.array([ 61.02377286, 19.80026021, 100.47687017, 10.83331295]) ) if sim.test: diff --git a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json index 0efcdaeca3c..dd59536ad37 100644 --- a/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json +++ b/Regression/Checksum/benchmarks_json/LaserAccelerationBoost.json @@ -1,38 +1,38 @@ { "lev=0": { - "Bx": 4818955.480792835, - "By": 1752.8025402207227, - "Bz": 14516.21278267981, - "Ex": 2366115496505.249, - "Ey": 1446112025634143.0, - "Ez": 21864189507353.19, - "jx": 1996366349839211.5, - "jy": 5.312583827165288e+16, - "jz": 2.049135262445976e+16, - "rho": 68443961.71835628 + "Bx": 4818955.480797943, + "By": 1752.8025638791275, + "Bz": 14516.212782554387, + "Ex": 2366115503598.9224, + "Ey": 1446112025635674.2, + "Ez": 21864189507357.867, + "jx": 1996366349775593.5, + "jy": 5.312583827155926e+16, + "jz": 2.0491352624508764e+16, + "rho": 68443961.71852128 }, "electrons": { - "particle_momentum_x": 2.2135945391227107e-23, - "particle_momentum_y": 2.8224559499572622e-22, - "particle_momentum_z": 5.260626010211241e-22, - "particle_position_x": 0.010800577787628053, + "particle_momentum_x": 2.2135945391319113e-23, + "particle_momentum_y": 2.8224559499558413e-22, + "particle_momentum_z": 5.260626010214114e-22, + "particle_position_x": 0.010800577787628052, "particle_position_y": 0.2111506062831815, "particle_weight": 4.121554826246186e+16 }, "ions": { - "particle_momentum_x": 6.248472277235318e-23, - "particle_momentum_y": 4.449097689427615e-22, - "particle_momentum_z": 5.768168724780326e-22, + "particle_momentum_x": 6.248472277246885e-23, + "particle_momentum_y": 4.449097689427654e-22, + "particle_momentum_z": 5.768168724998047e-22, "particle_position_x": 0.010800001678510512, "particle_position_y": 0.21114947608115425, "particle_weight": 4.121554826246186e+16 }, "beam": { - "particle_momentum_x": 3.535745635169933e-19, + "particle_momentum_x": 3.5357456351701565e-19, "particle_momentum_y": 4.363391839372122e-19, - "particle_momentum_z": 5.658606416951657e-17, - "particle_position_x": 0.008314723025211447, + "particle_momentum_z": 5.658606416951653e-17, + "particle_position_x": 0.008314723025211468, "particle_position_y": 1.1704335743854242, "particle_weight": 62415090744.60765 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json index dd224516c5c..140bff16cc2 100644 --- a/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json +++ b/Regression/Checksum/benchmarks_json/LaserAcceleration_BTD.json @@ -1,32 +1,32 @@ { "lev=0": { - "Bx": 497756265.44724107, - "By": 252197580.117894, - "Bz": 16988475.5444833, - "Ex": 7.299911104263803e+16, - "Ey": 1.460116488178734e+17, - "Ez": 2.26033100286114e+16, - "jx": 8.27644503757345e+17, - "jy": 2.1062078409959675e+18, - "jz": 2.8491727096438305e+19, - "rho": 91863764144.41415 - }, - "electrons": { - "particle_momentum_x": 5.76165226700654e-20, - "particle_momentum_y": 2.7567389504898156e-19, - "particle_momentum_z": 4.134562048099117e-19, - "particle_position_x": 0.0025269863605945427, - "particle_position_y": 0.0024538321295153346, - "particle_position_z": 0.17818421763751244, - "particle_weight": 1675789447169.5652 + "Bx": 499326907.0665159, + "By": 254256667.12630302, + "Bz": 16997157.572204895, + "Ex": 7.361789596479304e+16, + "Ey": 1.464987954436535e+17, + "Ez": 2.2588202735123868e+16, + "jx": 8.306469909158771e+17, + "jy": 2.1067947367705277e+18, + "jz": 2.86663445441587e+19, + "rho": 92447775221.08655 }, "beam": { - "particle_momentum_x": 1.1926313055043134e-17, - "particle_momentum_y": 2.7056205218547404e-17, - "particle_momentum_z": 5.1562131494813424e-14, - "particle_position_x": 0.000560821518739447, - "particle_position_y": 0.000800729816549036, - "particle_position_z": 2.9800048650570097, + "particle_momentum_x": 1.1926337276935442e-17, + "particle_momentum_y": 2.70596966890012e-17, + "particle_momentum_z": 5.1562142636293934e-14, + "particle_position_x": 0.0005608203524288058, + "particle_position_y": 0.000800752801704984, + "particle_position_z": 2.980004865036241, "particle_weight": 62415.090744607616 + }, + "electrons": { + "particle_momentum_x": 5.774426971193643e-20, + "particle_momentum_y": 2.742475325800203e-19, + "particle_momentum_z": 4.1461064946749695e-19, + "particle_position_x": 0.0025250307389502184, + "particle_position_y": 0.0024550212004677482, + "particle_position_z": 0.17819223528078157, + "particle_weight": 1675789447169.5652 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json index 242cf00f0c3..379e0991819 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromLASYFile_RZ.json @@ -1,12 +1,12 @@ { "lev=0": { - "Br": 100278645.72225758, - "Bz": 2509008.1146699786, - "Er": 3.263343388037509, - "Et": 3.0045297157982412e+16, - "Ez": 0.2422526411295923, - "jr": 49.67802097760541, - "jt": 4.3051397628943917e+17, + "Br": 100277153.12954171, + "Bz": 2508844.115818364, + "Er": 3.263693682174799, + "Et": 3.004484377567921e+16, + "Ez": 0.24219246674343975, + "jr": 49.68134418425658, + "jt": 4.305088291175302e+17, "jz": 0.0 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json b/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json index 8d58f5c9551..ef53dcabeec 100644 --- a/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json +++ b/Regression/Checksum/benchmarks_json/LaserInjectionFromRZLASYFile.json @@ -1,12 +1,12 @@ { "lev=0": { - "Br": 193609031.89343664, - "Bz": 4689634.787639907, - "Er": 197280940689075.1, - "Et": 5.9716806429474664e+16, - "Ez": 1383812184076636.0, - "jr": 18582087468.75, - "jt": 1.193738117372394e+18, + "Br": 193606866.6957581, + "Bz": 4689375.7330573285, + "Er": 197284500110989.84, + "Et": 5.971613843782555e+16, + "Ez": 1383808350045244.5, + "jr": 18582922265.875, + "jt": 1.193688996955741e+18, "jz": 0.0 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json index 0b38f78e6f9..67737738b2c 100644 --- a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json @@ -1,27 +1,27 @@ { "lev=0": { - "rho_electrons": 0.00443743609125863, - "rho_he_ions": 0.005198801518328451 + "rho_electrons": 0.004437417901725754, + "rho_he_ions": 0.005200476260168854 + }, + "he_ions": { + "particle_momentum_x": 2.771800535001148e-19, + "particle_momentum_y": 2.759439998478633e-19, + "particle_momentum_z": 3.6231785185705813e-19, + "particle_position_x": 2201.7215060103235, + "particle_weight": 17191519531250.002 }, "neutrals": { - "particle_momentum_x": 1.404700281648976e-19, - "particle_momentum_y": 1.4028127127618884e-19, - "particle_momentum_z": 1.4090901433394346e-19, - "particle_position_x": 1120.7727446759352, + "particle_momentum_x": 1.4052816330675472e-19, + "particle_momentum_y": 1.4027759931535706e-19, + "particle_momentum_z": 1.4119910335756157e-19, + "particle_position_x": 1121.3642639505965, "particle_weight": 6.4588e+19 }, - "he_ions": { - "particle_momentum_x": 2.770386771117138e-19, - "particle_momentum_y": 2.7568040242914223e-19, - "particle_momentum_z": 3.619756966185903e-19, - "particle_position_x": 2200.683185473434, - "particle_weight": 17185500000000.002 - }, "electrons": { - "particle_momentum_x": 3.5129762363657864e-20, - "particle_momentum_y": 3.5431134517510143e-20, - "particle_momentum_z": 1.2592093336142964e-19, - "particle_position_x": 2142.0662480700303, - "particle_weight": 14593699218750.002 + "particle_momentum_x": 3.535219734698525e-20, + "particle_momentum_y": 3.5373517451957064e-20, + "particle_momentum_z": 1.2562184152660903e-19, + "particle_position_x": 2141.9783924023695, + "particle_weight": 14592914062500.002 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json index ca50554c394..ec1b6272092 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_EM_modes_rz.json @@ -1,12 +1,12 @@ { "lev=0": {}, "ions": { - "particle_momentum_x": 5.043858996017125e-17, - "particle_momentum_y": 5.0444743275098145e-17, - "particle_momentum_z": 5.05192925609973e-17, - "particle_position_x": 143164.42694236583, - "particle_position_y": 143166.51848290052, - "particle_theta": 2573261.754119082, - "particle_weight": 8.12868064536689e+18 + "particle_momentum_x": 5.0438993756415296e-17, + "particle_momentum_y": 5.0444406612873916e-17, + "particle_momentum_z": 5.0519292431385393e-17, + "particle_position_x": 143164.41713467025, + "particle_position_y": 143166.51845281923, + "particle_theta": 2573261.8729711357, + "particle_weight": 8.128680645366887e+18 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json index c7b467ad5f4..316bcb5b30d 100644 --- a/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_ohms_law_solver_magnetic_reconnection_2d.json @@ -2,17 +2,17 @@ "lev=0": { "Bx": 1524.8789585554075, "By": 639.8314135126764, - "Bz": 7.476000395458839, - "Ex": 56499935.20497879, - "Ey": 75068107.08471295, - "Ez": 309535670.18599623 + "Bz": 7.476064371022787, + "Ex": 56500024.54347144, + "Ey": 75064104.7364582, + "Ez": 309548482.989921 }, "ions": { - "particle_momentum_x": 7.14295522755638e-15, - "particle_momentum_y": 7.1380780610425e-15, - "particle_momentum_z": 7.141134469227045e-15, - "particle_position_x": 11170689.203149632, - "particle_position_y": 5585328.083196239, + "particle_momentum_x": 7.142955224072594e-15, + "particle_momentum_y": 7.13807805985974e-15, + "particle_momentum_z": 7.141134511815448e-15, + "particle_position_x": 11170689.202853566, + "particle_position_y": 5585328.08326772, "particle_weight": 9.036667901693183e+18 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_prev_positions.json b/Regression/Checksum/benchmarks_json/Python_prev_positions.json index 55c886775d5..8c7c6bf8b73 100644 --- a/Regression/Checksum/benchmarks_json/Python_prev_positions.json +++ b/Regression/Checksum/benchmarks_json/Python_prev_positions.json @@ -3,21 +3,21 @@ "Bx": 0.0, "By": 0.0, "Bz": 0.0, - "Ex": 3710588.849989976, + "Ex": 3710700.956616212, "Ey": 0.0, - "Ez": 646727.8074440088, - "jx": 15259.034603501308, - "jy": 650.139263398662, - "jz": 943.0244062246846 + "Ez": 646752.4642948855, + "jx": 15258.82611558372, + "jy": 650.13507310447, + "jz": 942.8394748261528 }, "electrons": { - "particle_momentum_x": 8.78764082600202e-23, + "particle_momentum_x": 8.787556176835159e-23, "particle_momentum_y": 3.78128494181519e-24, - "particle_momentum_z": 5.3969075504746586e-24, - "particle_position_x": 0.6227948811898449, - "particle_position_y": 0.5553787034074176, - "particle_prev_x": 0.6152547961491118, - "particle_prev_z": 0.5559363018627721, + "particle_momentum_z": 5.395758515337991e-24, + "particle_position_x": 0.6227736757162666, + "particle_position_y": 0.5553800210930071, + "particle_prev_x": 0.615237399194202, + "particle_prev_z": 0.5559373934585523, "particle_weight": 8569335937.5 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json index f8186748f2c..8a1ff9de966 100644 --- a/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json +++ b/Regression/Checksum/benchmarks_json/hard_edged_quadrupoles.json @@ -3,20 +3,20 @@ "Bx": 0.0, "By": 0.0, "Bz": 0.0, - "Ex": 9.882421146615367e-06, - "Ey": 1.0440261046714249e-05, - "Ez": 1.003739697324731e-05, - "jx": 2.914866280957325e-10, - "jy": 8.46605718473121e-19, - "jz": 3.82349275686397e-08 + "Ex": 8.263130364346929e-06, + "Ey": 8.74492138450055e-06, + "Ez": 8.97527046111985e-06, + "jx": 2.9148662813789523e-10, + "jy": 8.513426949250112e-19, + "jz": 3.8234927568748794e-08 }, "electron": { - "particle_momentum_x": 2.0819392991319055e-25, - "particle_momentum_y": 6.046869894766189e-34, - "particle_momentum_z": 2.730924530729043e-23, - "particle_position_x": 0.03492328774658799, - "particle_position_y": 2.2742551618036812e-11, - "particle_position_z": 1.4915217664612082, + "particle_momentum_x": 2.0819392994331126e-25, + "particle_momentum_y": 6.080703685011498e-34, + "particle_momentum_z": 2.7309245307369126e-23, + "particle_position_x": 0.03492328774773231, + "particle_position_y": 2.3430381516062662e-11, + "particle_position_z": 1.491521766466965, "particle_weight": 1.0 } } diff --git a/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json b/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json index 801611f748b..a1ec0b4c831 100644 --- a/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json +++ b/Regression/Checksum/benchmarks_json/magnetostatic_eb_3d.json @@ -1,21 +1,21 @@ { - "beam": { - "particle_momentum_x": 1.356321653466651e-21, - "particle_momentum_y": 1.356321653466648e-21, - "particle_momentum_z": 7.150873477755126e-16, - "particle_position_x": 11163.999973966627, - "particle_position_y": 11163.999973966625, - "particle_position_z": 131662.5003103599, - "particle_weight": 20895107655113.465 - }, "lev=0": { - "Az": 11.35866332708226, - "Bx": 111.55929404955174, - "By": 111.55929404955177, - "Ex": 31268316751.31371, - "Ey": 31268316751.31371, - "jz": 1034841326.0482643, - "phi": 3144575598.841474, + "Az": 11.358663326449284, + "Bx": 111.55929407644248, + "By": 111.55929407644244, + "Ex": 31257180402.55472, + "Ey": 31257180402.55473, + "jz": 1034841325.9848926, + "phi": 3143521213.0157924, "rho": 3.449203918900721 + }, + "beam": { + "particle_momentum_x": 1.3604657334742729e-21, + "particle_momentum_y": 1.3604657334742772e-21, + "particle_momentum_z": 7.150873450281544e-16, + "particle_position_x": 11163.99997371537, + "particle_position_y": 11163.999973715368, + "particle_position_z": 131662.50031035842, + "particle_weight": 20895107655113.465 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json b/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json index 5c302d4c5b3..318aab0a1bc 100644 --- a/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json +++ b/Regression/Checksum/benchmarks_json/particle_thermal_boundary.json @@ -1,12 +1,12 @@ { "lev=0": { - "Bx": 167.49421217759414, - "By": 269.6793406439766, - "Bz": 258.3282607081799, - "Ex": 864657516595.9229, - "Ey": 24072965021.088013, - "Ez": 702786324880.7338, - "divE": 5.137699003981513e+19, - "rho": 357057125.6699786 + "Bx": 153.42850019293041, + "By": 290.08173598472, + "Bz": 187.69788476927954, + "Ex": 703014828960.0149, + "Ey": 28192823505.416325, + "Ez": 693698719737.6497, + "divE": 4.868580396562553e+19, + "rho": 324055400.4292488 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json b/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json index 2944a6ae69a..d76d1ed7d4c 100644 --- a/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json +++ b/Regression/Checksum/benchmarks_json/reduced_diags_single_precision.json @@ -1,27 +1,27 @@ { + "lev=0": { + "Bx": 0.08424138754164545, + "By": 0.08475193383267765, + "Bz": 0.08796438426563924, + "Ex": 106872541.64261818, + "Ey": 107805199.35060501, + "Ez": 107782783.64583969, + "jx": 728219.6885312796, + "jy": 736402.7559853494, + "jz": 734812.8777186275, + "rho": 0.02775080584090528, + "rho_electrons": 0.5250012277583664, + "rho_protons": 0.5250012273027096 + }, "electrons": { - "particle_momentum_x": 2.4417210113801336e-19, - "particle_momentum_y": 2.4564851493412593e-19, - "particle_momentum_z": 2.4341594564719795e-19, - "particle_position_x": 16379.247754118871, - "particle_position_y": 16382.836191849463, - "particle_position_z": 16382.833818260406, + "particle_momentum_x": 2.4417210857855684e-19, + "particle_momentum_y": 2.4564845792611e-19, + "particle_momentum_z": 2.434159547126453e-19, + "particle_position_x": 16379.247745715009, + "particle_position_y": 16382.83609924704, + "particle_position_z": 16382.833801764602, "particle_weight": 800000003014656.0 }, - "lev=0": { - "Bx": 0.084241235376183, - "By": 0.084751961809296, - "Bz": 0.08796431812220362, - "Ex": 106872513.1172905, - "Ey": 107805221.85876846, - "Ez": 107782819.9927826, - "jx": 728219.9166720062, - "jy": 736402.6373479366, - "jz": 734813.3440347463, - "rho": 0.0277508111536644, - "rho_electrons": 0.52500122784204, - "rho_protons": 0.5250012273481843 - }, "photons": { "particle_momentum_x": 1.428291590249666e-18, "particle_momentum_y": 1.4222174024686332e-18, @@ -32,12 +32,12 @@ "particle_weight": 800000003014656.0 }, "protons": { - "particle_momentum_x": 1.4104799077996007e-19, - "particle_momentum_y": 1.4120357666424943e-19, - "particle_momentum_z": 1.3903168923764969e-19, - "particle_position_x": 16383.951009266078, - "particle_position_y": 16383.991230053827, - "particle_position_z": 16384.033095981926, + "particle_momentum_x": 1.4104797829545176e-19, + "particle_momentum_y": 1.4120353509003028e-19, + "particle_momentum_z": 1.3903169103008315e-19, + "particle_position_x": 16383.951009267941, + "particle_position_y": 16383.991229955107, + "particle_position_z": 16384.03309576772, "particle_weight": 800000003014656.0 } -} +} \ No newline at end of file diff --git a/Source/Parallelization/WarpXSumGuardCells.H b/Source/Parallelization/WarpXSumGuardCells.H index 260ebb3871a..9c7c2e60160 100644 --- a/Source/Parallelization/WarpXSumGuardCells.H +++ b/Source/Parallelization/WarpXSumGuardCells.H @@ -16,11 +16,7 @@ * This is typically called for the sources of the Maxwell equations (J/rho) * after deposition from the macroparticles. * - * - When WarpX is used with a finite-difference scheme: this only - * updates the *valid* cells of `mf` - * - When WarpX is used with a spectral scheme (PSATD): this - * updates both the *valid* cells and *guard* cells. (This is because a - * spectral solver requires the value of the sources over a large stencil.) + * This updates both the *valid* cells and *guard* cells. */ void WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, @@ -33,11 +29,7 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, * This is typically called for the sources of the Maxwell equations (J/rho) * after deposition from the macroparticles + filtering. * - * - When WarpX is used with a finite-difference scheme: this only - * updates the *valid* cells of `dst` - * - When WarpX is used with a spectral scheme (PSATD): this - * updates both the *valid* cells and *guard* cells. (This is because a - * spectral solver requires the value of the sources over a large stencil.) + * This updates both the *valid* cells and *guard* cells. * * Note: `i_comp` is the component where the results will be stored in `dst`; * The component from which we copy in `src` is always 0. diff --git a/Source/Parallelization/WarpXSumGuardCells.cpp b/Source/Parallelization/WarpXSumGuardCells.cpp index 20ac73cab50..e752d49eedf 100644 --- a/Source/Parallelization/WarpXSumGuardCells.cpp +++ b/Source/Parallelization/WarpXSumGuardCells.cpp @@ -19,14 +19,7 @@ WarpXSumGuardCells(amrex::MultiFab& mf, const amrex::Periodicity& period, const amrex::IntVect& src_ngrow, const int icomp, const int ncomp) { - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { - n_updated_guards = mf.nGrowVect(); - } else { // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - } + amrex::IntVect const n_updated_guards = mf.nGrowVect(); ablastr::utils::communication::SumBoundary(mf, icomp, ncomp, src_ngrow, n_updated_guards, WarpX::do_single_precision_comms, period); } @@ -37,14 +30,7 @@ WarpXSumGuardCells(amrex::MultiFab& dst, amrex::MultiFab& src, const amrex::IntVect& src_ngrow, const int icomp, const int ncomp) { - amrex::IntVect n_updated_guards; - - // Update both valid cells and guard cells - if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { - n_updated_guards = dst.nGrowVect(); - } else { // Update only the valid cells - n_updated_guards = amrex::IntVect::TheZeroVector(); - } + amrex::IntVect const n_updated_guards = dst.nGrowVect(); dst.setVal(0., icomp, ncomp, n_updated_guards); dst.ParallelAdd(src, 0, icomp, ncomp, src_ngrow, n_updated_guards, period); From 12349a8d6cd08e5497e7ccf27f6004e4e58464ef Mon Sep 17 00:00:00 2001 From: Marco Garten Date: Mon, 15 Apr 2024 09:27:09 -0700 Subject: [PATCH 048/190] Set OpenMP threads manually on Perlmutter nodes (#4857) --- Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch | 1 + Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch | 1 + 2 files changed, 2 insertions(+) diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch b/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch index a1687b2a3b4..d13c7e3b4e5 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu.sbatch @@ -33,6 +33,7 @@ INPUTS=inputs_small export SRUN_CPUS_PER_TASK=16 # 8 cores per chiplet, 2x SMP export OMP_PLACES=threads export OMP_PROC_BIND=spread +export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} srun --cpu-bind=cores \ ${EXE} ${INPUTS} \ diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch b/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch index 873fd30355f..f2ea5fa3e7f 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu.sbatch @@ -34,6 +34,7 @@ export MPICH_OFI_NIC_POLICY=GPU # threads for OpenMP and threaded compressors per MPI rank # note: 16 avoids hyperthreading (32 virtual cores, 16 physical) export SRUN_CPUS_PER_TASK=16 +export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} # GPU-aware MPI optimizations GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" From 58e6b8def4769fb18f9c7467b89f8418b587eebb Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 15 Apr 2024 22:18:20 +0200 Subject: [PATCH 049/190] AMReX/pyAMReX/PICSAR: Weekly Update (#4858) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index f47dd01e057..06eb90b4395 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 2a3955a5f5aac1aef6e6e72687f182331e049c39 && cd - + cd ../amrex && git checkout --detach 7813bb7d6bc6cea658edac1f2bde046c7d174168 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index d1c98b315a0..a54553689ad 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 2a3955a5f5aac1aef6e6e72687f182331e049c39 +branch = 7813bb7d6bc6cea658edac1f2bde046c7d174168 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 12fed1e8bb9..f35a7676ab8 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 2a3955a5f5aac1aef6e6e72687f182331e049c39 +branch = 7813bb7d6bc6cea658edac1f2bde046c7d174168 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 3f2987ec9cd..9d21519ce71 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "2a3955a5f5aac1aef6e6e72687f182331e049c39" +set(WarpX_amrex_branch "7813bb7d6bc6cea658edac1f2bde046c7d174168" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index d4c488f9113..be0e857f6f5 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "9b0a4d82bc14756530efe4e0c9ce3463ca5df742" +set(WarpX_pyamrex_branch "a9e0188ac11865f395a01f8b3195dace67b1f8e8" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 1d95dd5f2d2..774a220dc06 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 2a3955a5f5aac1aef6e6e72687f182331e049c39 && cd - +cd amrex && git checkout --detach 7813bb7d6bc6cea658edac1f2bde046c7d174168 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 3add0096b1d7fa0030fd35b1fca25f140fde33df Mon Sep 17 00:00:00 2001 From: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> Date: Mon, 15 Apr 2024 16:54:44 -0700 Subject: [PATCH 050/190] Optimizing GPU acceleration of binary collision algorithms. (#4577) * opt gpu accel of particle collisions * minor updates in .gitignore * fix for consistent and correct particle collision error * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * minor bug fixing * [minor]: remove stale code and clean up comments * undo erroneously pushed changes in GNUmakefile * undo erroneously pushed inputs_3d * revert errors from obsolete code plus minor improvements * fix for a possible segfault * replace std::min with amrex::min for future compatibility * clang-tidy updates * Update random seed * Update benchmark files * Update .gitignore Co-authored-by: Remi Lehe * Update .gitignore Co-authored-by: Remi Lehe * Use bisect function * Simplify the code * Update benchmark and fix style * use `BinaryCollision` class for DSMC collisions * comment out explicit delete of `ScatteringProcess` constructor * fix first GPU compilation issue * fix RZ issue * avoid copying `binary_collision_functor` * fix merge error * do not keep `ScatteringProcess` instances in `DSMCFunc` * only keep array of `ScatteringProcess::Executor`s * fix clang-tidy errors and reset benchmark * fix default collision type issue * copy cross-section data to `DSMCFunc` * update checksum after fixing issue with cross-section arrays * move logic to set up cross-section interpolators to `DSMCFunc.cpp` file * use internal `struct`s in collision filter functors * clean-up and fixing clang-tidy issues * second clang-tidy fix attempt * update `DSMCFunc` with new collision pair parallelization strategy * fix DSMC CI benchmark file values * Fix clang-tidy error * update reference density array for DSMC CI test * Only call redistribute if collision type has product species * fix clang-tidy error * another clang-tidy error fix * Appropriately set `local` parameter in `Redistribute` call. * set `Redistribute` arguments appropriately * check if weight is numerically zero * Apply suggestions from code review Co-authored-by: Remi Lehe * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add docstring for `CollisionPairFilter` function arguments * Use `insert` rather than `push_back` & `rotate` Co-authored-by: Remi Lehe * apply suggestions from a few of the review comments * reduce differences between `SplitAndScatterFunc.H` and `ParticleCreationFunc.H` * use `deleteInvalidParticles` from #4849 * avoid unnecessary duplication when species1 == species2 * Apply suggestions from code review * Apply suggestions from code review * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Remi Lehe Co-authored-by: roelof-groenewald Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- .../capacitive_discharge/analysis_dsmc.py | 66 +++---- .../inputs_deuterium_tritium_3d | 2 + .../Deuterium_Deuterium_Fusion_3D.json | 40 ++--- ...rium_Deuterium_Fusion_3D_intraspecies.json | 44 ++--- .../Deuterium_Tritium_Fusion_3D.json | 114 ++++++------ .../Deuterium_Tritium_Fusion_RZ.json | 110 ++++++------ .../Proton_Boron_Fusion_2D.json | 144 +++++++-------- .../Proton_Boron_Fusion_3D.json | 164 +++++++++--------- .../benchmarks_json/Python_dsmc_1d.json | 34 ++-- .../benchmarks_json/collisionISO.json | 18 +- .../BinaryCollision/BinaryCollision.H | 136 +++++++++++++-- .../Coulomb/ElasticCollisionPerez.H | 24 ++- .../Coulomb/PairWiseCoulombCollisionFunc.H | 5 +- .../Collision/BinaryCollision/DSMC/DSMCFunc.H | 25 ++- .../DSMC/SplitAndScatterFunc.H | 6 +- .../NuclearFusion/NuclearFusionFunc.H | 25 ++- .../BinaryCollision/ParticleCreationFunc.H | 12 +- 17 files changed, 551 insertions(+), 418 deletions(-) diff --git a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py index 94a08186183..4c46b9e3f10 100755 --- a/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py +++ b/Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py @@ -18,39 +18,39 @@ my_check = checksumAPI.evaluate_checksum(test_name, fn, do_particles=True) ref_density = np.array([ - 1.27945114e+14, 2.23583642e+14, 2.55392166e+14, 2.55655614e+14, - 2.55811304e+14, 2.55813038e+14, 2.55817129e+14, 2.55752279e+14, - 2.55912208e+14, 2.56079483e+14, 2.55932916e+14, 2.55822911e+14, - 2.55910910e+14, 2.55995538e+14, 2.56185960e+14, 2.56518490e+14, - 2.56843281e+14, 2.56847244e+14, 2.56317496e+14, 2.55995033e+14, - 2.56036918e+14, 2.56185305e+14, 2.56473454e+14, 2.56612720e+14, - 2.56460349e+14, 2.56162495e+14, 2.56154687e+14, 2.56351568e+14, - 2.56518116e+14, 2.56589716e+14, 2.56491198e+14, 2.56462652e+14, - 2.56475832e+14, 2.56502009e+14, 2.56512082e+14, 2.56400671e+14, - 2.56317027e+14, 2.56348890e+14, 2.56604754e+14, 2.56846626e+14, - 2.56749617e+14, 2.56563946e+14, 2.56508159e+14, 2.56706913e+14, - 2.56921135e+14, 2.56772946e+14, 2.56509222e+14, 2.56556290e+14, - 2.56632138e+14, 2.56452482e+14, 2.56371644e+14, 2.56363151e+14, - 2.56471549e+14, 2.56541772e+14, 2.56342768e+14, 2.56345629e+14, - 2.56415143e+14, 2.56259609e+14, 2.56264787e+14, 2.56619762e+14, - 2.56992843e+14, 2.57102387e+14, 2.56719391e+14, 2.56255624e+14, - 2.56276638e+14, 2.56350832e+14, 2.56295294e+14, 2.56153556e+14, - 2.56170347e+14, 2.56246380e+14, 2.56088509e+14, 2.56121028e+14, - 2.56423889e+14, 2.56597963e+14, 2.56470310e+14, 2.56419968e+14, - 2.56625715e+14, 2.56711965e+14, 2.56651610e+14, 2.56750226e+14, - 2.56886322e+14, 2.56839170e+14, 2.56469219e+14, 2.56254127e+14, - 2.56422126e+14, 2.56381502e+14, 2.56283046e+14, 2.56358078e+14, - 2.56454503e+14, 2.56810744e+14, 2.57281306e+14, 2.57314168e+14, - 2.57216217e+14, 2.57166645e+14, 2.56969293e+14, 2.56763128e+14, - 2.56479604e+14, 2.56454515e+14, 2.56670498e+14, 2.56378555e+14, - 2.56177400e+14, 2.56510462e+14, 2.56512126e+14, 2.56391975e+14, - 2.56717395e+14, 2.57024288e+14, 2.56901669e+14, 2.56632950e+14, - 2.56470315e+14, 2.56530515e+14, 2.56839839e+14, 2.57084249e+14, - 2.56897794e+14, 2.56360159e+14, 2.56251337e+14, 2.56628550e+14, - 2.56743475e+14, 2.56616083e+14, 2.56830920e+14, 2.57164423e+14, - 2.56868182e+14, 2.56749114e+14, 2.57108990e+14, 2.56791445e+14, - 2.56203737e+14, 2.56114492e+14, 2.56630346e+14, 2.24843400e+14, - 1.28200732e+14 + 1.27943881e+14, 2.23583097e+14, 2.55396716e+14, 2.55673406e+14, + 2.55827566e+14, 2.55803446e+14, 2.55798707e+14, 2.55748961e+14, + 2.55906413e+14, 2.56063991e+14, 2.55937018e+14, 2.55841390e+14, + 2.55917724e+14, 2.55988641e+14, 2.56052050e+14, 2.56285151e+14, + 2.56647960e+14, 2.56756264e+14, 2.56430158e+14, 2.56117493e+14, + 2.56065302e+14, 2.56265220e+14, 2.56328575e+14, 2.56031495e+14, + 2.56123757e+14, 2.56431173e+14, 2.56385320e+14, 2.56391170e+14, + 2.56561177e+14, 2.56513926e+14, 2.56332201e+14, 2.56252442e+14, + 2.56238982e+14, 2.56216498e+14, 2.56461281e+14, 2.56863199e+14, + 2.56908100e+14, 2.56926112e+14, 2.57001641e+14, 2.56735963e+14, + 2.56315358e+14, 2.56137028e+14, 2.56101418e+14, 2.56276827e+14, + 2.56425668e+14, 2.56181798e+14, 2.56044925e+14, 2.56330387e+14, + 2.56623150e+14, 2.56445316e+14, 2.56292750e+14, 2.56440918e+14, + 2.56433406e+14, 2.56186982e+14, 2.56236390e+14, 2.56469557e+14, + 2.56349704e+14, 2.56487457e+14, 2.56771823e+14, 2.56614683e+14, + 2.56552210e+14, 2.56850291e+14, 2.56783396e+14, 2.56483187e+14, + 2.56510868e+14, 2.56490408e+14, 2.56656042e+14, 2.56820924e+14, + 2.56640314e+14, 2.56465063e+14, 2.56510264e+14, 2.56917331e+14, + 2.57228490e+14, 2.56960593e+14, 2.56587911e+14, 2.56672682e+14, + 2.56774414e+14, 2.56548335e+14, 2.56225540e+14, 2.56079693e+14, + 2.56062796e+14, 2.56054612e+14, 2.56028683e+14, 2.56068820e+14, + 2.56380975e+14, 2.56654914e+14, 2.56776792e+14, 2.56983661e+14, + 2.56989477e+14, 2.56646250e+14, 2.56589639e+14, 2.56946205e+14, + 2.57091201e+14, 2.56913590e+14, 2.56513535e+14, 2.56122597e+14, + 2.56176340e+14, 2.56808001e+14, 2.57239393e+14, 2.56845066e+14, + 2.56662482e+14, 2.56862583e+14, 2.56518922e+14, 2.56155531e+14, + 2.56362794e+14, 2.57203564e+14, 2.57737938e+14, 2.57252026e+14, + 2.56859277e+14, 2.56658995e+14, 2.56357364e+14, 2.56393454e+14, + 2.56714308e+14, 2.57042200e+14, 2.57551087e+14, 2.57502490e+14, + 2.56641118e+14, 2.56401115e+14, 2.56644629e+14, 2.56673096e+14, + 2.56534659e+14, 2.56357745e+14, 2.56455309e+14, 2.56586850e+14, + 2.56442415e+14, 2.56335971e+14, 2.56411429e+14, 2.24109018e+14, + 1.27678869e+14 ]) density_data = np.load( 'ion_density_case_1.npy' ) diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d index 0943eee1d22..8099d92928b 100644 --- a/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d +++ b/Examples/Tests/nuclear_fusion/inputs_deuterium_tritium_3d @@ -11,6 +11,8 @@ geometry.dims = 3 geometry.prob_lo = 0. 0. 0. geometry.prob_hi = 8. 8. 16. +warpx.random_seed = 4 + ################################# ###### Boundary Condition ####### ################################# diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json index 0d426c07ed9..76d03f521b8 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D.json @@ -1,4 +1,7 @@ { + "lev=0": { + "rho": 0.0 + }, "deuterium_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -8,15 +11,6 @@ "particle_position_z": 81919021.52308556, "particle_weight": 1024.000000000021 }, - "helium3_1": { - "particle_momentum_x": 8.492383942294212e-16, - "particle_momentum_y": 8.507905928154269e-16, - "particle_momentum_z": 8.500197251270117e-16, - "particle_position_x": 151779.74563236872, - "particle_position_y": 152858.5501479658, - "particle_position_z": 322785.7613257161, - "particle_weight": 2.7065032055210726e-28 - }, "hydrogen2_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, @@ -26,16 +20,22 @@ "particle_position_z": 81920546.19181262, "particle_weight": 1024.000000000021 }, - "lev=0": { - "rho": 0.0 - }, "neutron_1": { - "particle_momentum_x": 8.492383942294212e-16, - "particle_momentum_y": 8.507905928154269e-16, - "particle_momentum_z": 8.500197251270117e-16, - "particle_position_x": 151779.74563236872, - "particle_position_y": 152858.5501479658, - "particle_position_z": 322785.7613257161, - "particle_weight": 2.7065032055210726e-28 + "particle_momentum_x": 8.574995119045322e-16, + "particle_momentum_y": 8.563358069072292e-16, + "particle_momentum_z": 8.571142005592407e-16, + "particle_position_x": 152851.7089469153, + "particle_position_y": 151779.17287974653, + "particle_position_z": 325950.1658699999, + "particle_weight": 2.7360848012854956e-28 + }, + "helium3_1": { + "particle_momentum_x": 8.574995119045322e-16, + "particle_momentum_y": 8.563358069072292e-16, + "particle_momentum_z": 8.571142005592407e-16, + "particle_position_x": 152851.7089469153, + "particle_position_y": 151779.17287974653, + "particle_position_z": 325950.1658699999, + "particle_weight": 2.7360848012854956e-28 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json index 1c6af40e173..59818f62826 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json @@ -1,4 +1,18 @@ { + "lev=0": { + "rho": 0.0, + "rho_deuterium": 8203144355.71195, + "rho_helium3": 10.368009592276463 + }, + "neutron": { + "particle_momentum_x": 2.2543499835759282e-15, + "particle_momentum_y": 2.2526527390783875e-15, + "particle_momentum_z": 2.2619641737859965e-15, + "particle_position_x": 61.961041864660686, + "particle_position_y": 61.78141653674165, + "particle_position_z": 61.741022731492514, + "particle_weight": 505562702.7678892 + }, "deuterium": { "particle_momentum_x": 1.3370046499332103e-14, "particle_momentum_y": 1.3364310231320824e-14, @@ -6,29 +20,15 @@ "particle_position_x": 2560.1613417364665, "particle_position_y": 2560.082464065988, "particle_position_z": 2560.0018477161034, - "particle_weight": 7.999999989895393e+17 + "particle_weight": 7.999999989888742e+17 }, "helium3": { - "particle_momentum_x": 2.2800935865848323e-15, - "particle_momentum_y": 2.2864983116288316e-15, - "particle_momentum_z": 2.290487090201156e-15, - "particle_position_x": 62.07326392385665, - "particle_position_y": 62.074468969610855, - "particle_position_z": 62.07087662970013, - "particle_weight": 505230602.2536906 - }, - "lev=0": { - "rho": 0.0, - "rho_deuterium": 8203144355.71876, - "rho_helium3": 10.36119892112141 - }, - "neutron": { - "particle_momentum_x": 2.2625878931428226e-15, - "particle_momentum_y": 2.2652096393691827e-15, - "particle_momentum_z": 2.27105060559148e-15, - "particle_position_x": 62.07326392385665, - "particle_position_y": 62.074468969610855, - "particle_position_z": 62.07087662970013, - "particle_weight": 505230602.2536906 + "particle_momentum_x": 2.2749239620327265e-15, + "particle_momentum_y": 2.268697031603961e-15, + "particle_momentum_z": 2.278045756364995e-15, + "particle_position_x": 61.961041864660686, + "particle_position_y": 61.78141653674165, + "particle_position_z": 61.741022731492514, + "particle_weight": 505562702.7678892 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json index 23231eab789..1ac85583f9d 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_3D.json @@ -2,76 +2,76 @@ "lev=0": { "rho": 0.0 }, - "neutron_1": { - "particle_momentum_x": 1.7519716491839538e-15, - "particle_momentum_y": 1.7523289312260283e-15, - "particle_momentum_z": 1.7480231586369996e-15, - "particle_position_x": 154379.32401483235, - "particle_position_y": 152618.63815943015, - "particle_position_z": 325970.4138010667, - "particle_weight": 4.421535775967805e-28 + "neutron_2": { + "particle_momentum_x": 1.5428541105411113e-15, + "particle_momentum_y": 1.5521530475782452e-15, + "particle_momentum_z": 1.5581061911002605e-15, + "particle_position_x": 139420.85239440785, + "particle_position_y": 137544.4080412421, + "particle_position_z": 292987.6112145397, + "particle_weight": 6.353652226479104e+18 }, - "tritium_1": { + "tritium_2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 + "particle_momentum_z": 0.0, + "particle_position_x": 409665.266470154, + "particle_position_y": 409535.84596852644, + "particle_position_z": 819126.8984535293, + "particle_weight": 1.0239999999364636e+29 }, - "deuterium_2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 3.356220762633467e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503, - "particle_weight": 1.0240001137713503e+30 + "helium4_2": { + "particle_momentum_x": 1.5428541105411113e-15, + "particle_momentum_y": 1.5521530475782452e-15, + "particle_momentum_z": 1.7807114780425105e-15, + "particle_position_x": 139420.85239440785, + "particle_position_y": 137544.4080412421, + "particle_position_z": 292987.6112145397, + "particle_weight": 6.353652226479104e+18 }, - "helium4_1": { - "particle_momentum_x": 1.7519716491839538e-15, - "particle_momentum_y": 1.7523289312260283e-15, - "particle_momentum_z": 1.7480231586369996e-15, - "particle_position_x": 154379.32401483235, - "particle_position_y": 152618.63815943015, - "particle_position_z": 325970.4138010667, - "particle_weight": 4.421535775967805e-28 + "neutron_1": { + "particle_momentum_x": 1.7286463849660049e-15, + "particle_momentum_y": 1.7253692489143775e-15, + "particle_momentum_z": 1.7397872072929338e-15, + "particle_position_x": 151971.91531558792, + "particle_position_y": 150436.3752328273, + "particle_position_z": 320653.7895132215, + "particle_weight": 4.360021412453755e-28 }, "deuterium_1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, "particle_momentum_z": 2.8875978729147693e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, + "particle_position_x": 40958427.509923086, + "particle_position_y": 40959476.344507605, + "particle_position_z": 81921930.27522033, "particle_weight": 1024.000000000021 }, - "neutron_2": { - "particle_momentum_x": 1.538345593941914e-15, - "particle_momentum_y": 1.536969107959402e-15, - "particle_momentum_z": 1.5535605933245691e-15, - "particle_position_x": 137088.2335716813, - "particle_position_y": 136370.55273167047, - "particle_position_z": 290148.26756873244, - "particle_weight": 6.427760042646557e+18 - }, - "helium4_2": { - "particle_momentum_x": 1.538345593941914e-15, - "particle_momentum_y": 1.536969107959402e-15, - "particle_momentum_z": 1.769309377628039e-15, - "particle_position_x": 137088.2335716813, - "particle_position_y": 136370.55273167047, - "particle_position_z": 290148.26756873244, - "particle_weight": 6.427760042646557e+18 + "tritium_1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.8875978729147693e-13, + "particle_position_x": 40959200.1108159, + "particle_position_y": 40960650.40789148, + "particle_position_z": 81920772.79861207, + "particle_weight": 1024.000000000021 }, - "tritium_2": { + "deuterium_2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.0158217681, - "particle_position_y": 409670.9858143465, - "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999357226e+29 + "particle_momentum_z": 3.356220762633467e-14, + "particle_position_x": 4096177.559084946, + "particle_position_y": 4096353.028787281, + "particle_position_z": 8192362.405430989, + "particle_weight": 1.024000113771424e+30 + }, + "helium4_1": { + "particle_momentum_x": 1.7286463849660049e-15, + "particle_momentum_y": 1.7253692489143775e-15, + "particle_momentum_z": 1.7397872072929338e-15, + "particle_position_x": 151971.91531558792, + "particle_position_y": 150436.3752328273, + "particle_position_z": 320653.7895132215, + "particle_weight": 4.360021412453755e-28 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json index 3fb0db8d8a5..bccb0a12b0f 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Tritium_Fusion_RZ.json @@ -1,64 +1,28 @@ { - "deuterium_1": { - "particle_momentum_x": 1.8388106511899905e-13, - "particle_momentum_y": 1.837868790009435e-13, - "particle_momentum_z": 0.0, - "particle_position_x": 40959919.499819286, - "particle_position_y": 81919224.48541151, - "particle_theta": 32166860.23003994, - "particle_weight": 3216.984554806547 - }, - "deuterium_2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 3.336364094249911e-14, - "particle_position_x": 4095908.9083809257, - "particle_position_y": 8192069.080030457, - "particle_theta": 3216444.348910214, - "particle_weight": 3.1898417901971444e+29 - }, - "helium4_1": { - "particle_momentum_x": 1.858124399143442e-15, - "particle_momentum_y": 1.876715110797694e-15, - "particle_momentum_z": 1.7098432207359157e-15, - "particle_position_x": 152920.23233108618, - "particle_position_y": 323733.9138644398, - "particle_theta": 120064.13771707338, - "particle_weight": 1.603083276067953e-27 - }, - "helium4_2": { - "particle_momentum_x": 1.5195006688950936e-15, - "particle_momentum_y": 1.52430083815551e-15, - "particle_momentum_z": 1.7654865863613367e-15, - "particle_position_x": 136867.63803188328, - "particle_position_y": 286903.30393175944, - "particle_theta": 107912.20520382549, - "particle_weight": 2.0862696876352987e+19 - }, "lev=0": { "rho": 0.0 }, - "neutron_1": { - "particle_momentum_x": 1.7160671487712845e-15, - "particle_momentum_y": 1.7154753069055672e-15, - "particle_momentum_z": 1.7098432207359157e-15, - "particle_position_x": 152920.23233108618, - "particle_position_y": 323733.9138644398, - "particle_theta": 120064.13771707338, - "particle_weight": 1.603083276067953e-27 + "helium4_2": { + "particle_momentum_x": 1.500608194743691e-15, + "particle_momentum_y": 1.5109305701135078e-15, + "particle_momentum_z": 1.7349325969148392e-15, + "particle_position_x": 134553.8162997425, + "particle_position_y": 285988.79476631305, + "particle_theta": 106450.92963352974, + "particle_weight": 2.0493998275383095e+19 }, "neutron_2": { - "particle_momentum_x": 1.5195006688950936e-15, - "particle_momentum_y": 1.52430083815551e-15, - "particle_momentum_z": 1.5463311225724366e-15, - "particle_position_x": 136867.63803188328, - "particle_position_y": 286903.30393175944, - "particle_theta": 107912.20520382549, - "particle_weight": 2.0862696876352987e+19 + "particle_momentum_x": 1.500608194743691e-15, + "particle_momentum_y": 1.5109305701135078e-15, + "particle_momentum_z": 1.5208565520173107e-15, + "particle_position_x": 134553.8162997425, + "particle_position_y": 285988.79476631305, + "particle_theta": 106450.92963352974, + "particle_weight": 2.0493998275383095e+19 }, "tritium_1": { - "particle_momentum_x": 1.8384658063720362e-13, - "particle_momentum_y": 1.8381593257898129e-13, + "particle_momentum_x": 1.8384658063720355e-13, + "particle_momentum_y": 1.8381593257898126e-13, "particle_momentum_z": 0.0, "particle_position_x": 40961278.052658774, "particle_position_y": 81919046.8061561, @@ -72,6 +36,42 @@ "particle_position_x": 409793.9651940968, "particle_position_y": 819237.3558155322, "particle_theta": 321974.4557387621, - "particle_weight": 3.218514276139388e+29 + "particle_weight": 3.218514276143075e+29 + }, + "helium4_1": { + "particle_momentum_x": 1.8900919654269547e-15, + "particle_momentum_y": 1.88323376518843e-15, + "particle_momentum_z": 1.7117554385103313e-15, + "particle_position_x": 154816.5295407665, + "particle_position_y": 326401.3426220656, + "particle_theta": 121193.01462929095, + "particle_weight": 1.615304706407927e-27 + }, + "deuterium_1": { + "particle_momentum_x": 1.8388106511899905e-13, + "particle_momentum_y": 1.837868790009435e-13, + "particle_momentum_z": 0.0, + "particle_position_x": 40959919.499819286, + "particle_position_y": 81919224.48541151, + "particle_theta": 32166860.23003994, + "particle_weight": 3216.984554806547 + }, + "neutron_1": { + "particle_momentum_x": 1.7399029833189488e-15, + "particle_momentum_y": 1.7342017841614303e-15, + "particle_momentum_z": 1.7117554385103313e-15, + "particle_position_x": 154816.5295407665, + "particle_position_y": 326401.3426220656, + "particle_theta": 121193.01462929095, + "particle_weight": 1.615304706407927e-27 + }, + "deuterium_2": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 3.336364094249911e-14, + "particle_position_x": 4095908.9083809257, + "particle_position_y": 8192069.080030457, + "particle_theta": 3216444.348910214, + "particle_weight": 3.18984179020083e+29 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json index 7f359f76e94..c6bd572ab68 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json @@ -2,53 +2,29 @@ "lev=0": { "rho": 0.0 }, - "alpha2": { - "particle_momentum_x": 4.0815251136381854e-15, - "particle_momentum_y": 4.122335955939925e-15, - "particle_momentum_z": 4.201726051973563e-15, - "particle_position_x": 410664.0477768091, - "particle_position_y": 868193.1483606595, - "particle_weight": 2.947401325355292e+18 - }, - "alpha3": { - "particle_momentum_x": 4.925995964082186e-16, - "particle_momentum_y": 4.780894181553476e-16, - "particle_momentum_z": 4.688680302272935e-16, - "particle_position_x": 52240.870920562535, - "particle_position_y": 103873.30862578771, - "particle_weight": 1.4784234459335885e+27 - }, - "proton1": { + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983792, - "particle_position_y": 81919772.69310114, - "particle_weight": 128.00000000000261 + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512789535533e+28 }, "alpha4": { - "particle_momentum_x": 2.3422553081132076e-14, - "particle_momentum_y": 2.3416860254545166e-14, - "particle_momentum_z": 2.3532380279126124e-14, - "particle_position_x": 2457367.4582781536, + "particle_momentum_x": 2.3381598715816888e-14, + "particle_momentum_y": 2.3435337108040707e-14, + "particle_momentum_z": 2.3554073734365495e-14, + "particle_position_x": 2457367.4582781526, "particle_position_y": 4915112.044373058, "particle_weight": 384.0000000000002 }, - "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 819198.7077771134, - "particle_weight": 1.2800000000000004e+37 - }, "boron3": { "particle_momentum_x": 9.277692671587846e-15, "particle_momentum_y": 9.268409636965691e-15, "particle_momentum_z": 9.279446607709548e-15, "particle_position_x": 4096178.1664224654, "particle_position_y": 8192499.7060386725, - "particle_weight": 6.399507192184686e+30 + "particle_weight": 6.399508749004609e+30 }, "boron2": { "particle_momentum_x": 0.0, @@ -56,23 +32,47 @@ "particle_momentum_z": 0.0, "particle_position_x": 409798.015821768, "particle_position_y": 819270.9858143466, - "particle_weight": 1.2799999999017534e+28 + "particle_weight": 1.279999999903553e+28 }, - "boron1": { + "proton4": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 81921136.14476715, - "particle_weight": 128.00000000000261 + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 819198.7077771134, + "particle_weight": 1.2800000000000004e+37 + }, + "alpha2": { + "particle_momentum_x": 4.057984510682467e-15, + "particle_momentum_y": 4.104188139725855e-15, + "particle_momentum_z": 4.17858000090827e-15, + "particle_position_x": 408793.7905852193, + "particle_position_y": 861780.8020495367, + "particle_weight": 2.893405947366684e+18 }, "proton3": { - "particle_momentum_x": 1.684673285246867e-15, - "particle_momentum_y": 1.6827557106531144e-15, - "particle_momentum_z": 1.6802642612723895e-15, - "particle_position_x": 2457259.537951346, - "particle_position_y": 4914248.771185745, - "particle_weight": 1.2795071921846893e+30 + "particle_momentum_x": 1.6847528610914633e-15, + "particle_momentum_y": 1.6827441855152735e-15, + "particle_momentum_z": 1.6802807054711977e-15, + "particle_position_x": 2457284.001605452, + "particle_position_y": 4914327.937654163, + "particle_weight": 1.2795087490046132e+30 + }, + "alpha3": { + "particle_momentum_x": 4.969030674549252e-16, + "particle_momentum_y": 4.889174794354074e-16, + "particle_momentum_z": 4.826711561221412e-16, + "particle_position_x": 52379.50835085382, + "particle_position_y": 104954.74986555413, + "particle_weight": 1.4737529861620076e+27 + }, + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.3312927569, + "particle_position_y": 819118.5558814355, + "particle_weight": 127.99999999999999 }, "proton5": { "particle_momentum_x": 0.0, @@ -82,36 +82,36 @@ "particle_position_y": 819101.3225783394, "particle_weight": 1.2800000000000004e+37 }, - "alpha1": { - "particle_momentum_x": 4.714228774936194e-15, - "particle_momentum_y": 4.676079523757908e-15, - "particle_momentum_z": 4.676143086393951e-15, - "particle_position_x": 463834.9497057527, - "particle_position_y": 977592.6452214213, - "particle_weight": 2.9280275175431764e-28 - }, - "alpha5": { - "particle_momentum_x": 2.3300446729308862e-14, - "particle_momentum_y": 2.3449941485701153e-14, - "particle_momentum_z": 2.3618372938672865e-14, - "particle_position_x": 2457556.8571638414, - "particle_position_y": 4914659.635379325, - "particle_weight": 3.839999999999998e-19 - }, - "proton2": { + "boron1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3723248133690294e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512789517532e+28 + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 81921136.14476715, + "particle_weight": 128.00000000000261 }, - "boron5": { + "proton1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.3312927569, - "particle_position_y": 819118.5558814355, - "particle_weight": 127.99999999999999 + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983792, + "particle_position_y": 81919772.69310114, + "particle_weight": 128.00000000000261 + }, + "alpha1": { + "particle_momentum_x": 4.691909092431811e-15, + "particle_momentum_y": 4.6836958163275755e-15, + "particle_momentum_z": 4.657203546376977e-15, + "particle_position_x": 463465.27131109464, + "particle_position_y": 978207.6061359186, + "particle_weight": 2.964832324596836e-28 + }, + "alpha5": { + "particle_momentum_x": 2.3343469003376247e-14, + "particle_momentum_y": 2.3450435277836452e-14, + "particle_momentum_z": 2.357999284036709e-14, + "particle_position_x": 2457556.8571638423, + "particle_position_y": 4914659.635379323, + "particle_weight": 3.839999999999998e-19 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json index c3517c32b5b..9a078d754c3 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json @@ -2,68 +2,32 @@ "lev=0": { "rho": 0.0 }, - "proton3": { - "particle_momentum_x": 1.684809640261926e-15, - "particle_momentum_y": 1.6828399494797829e-15, - "particle_momentum_z": 1.6804535487104095e-15, - "particle_position_x": 2457338.899376694, - "particle_position_y": 2457069.647393952, - "particle_position_z": 4914642.288898885, - "particle_weight": 1.0236580897818594e+31 - }, - "alpha5": { - "particle_momentum_x": 2.3336421176154494e-14, - "particle_momentum_y": 2.345583787205298e-14, - "particle_momentum_z": 2.3574859187827772e-14, - "particle_position_x": 2457556.857163842, - "particle_position_y": 2457059.6353793233, - "particle_position_z": 4915847.043341331, - "particle_weight": 3.0719999999999984e-18 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.119658089781855e+31 - }, - "boron1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 + "alpha2": { + "particle_momentum_x": 4.1179548991012315e-15, + "particle_momentum_y": 4.110026665992801e-15, + "particle_momentum_z": 4.169802553223462e-15, + "particle_position_x": 408575.75269073684, + "particle_position_y": 413407.5155277014, + "particle_position_z": 863983.4313441743, + "particle_weight": 1.901545867395127e+19 }, "alpha4": { - "particle_momentum_x": 2.3374367002622734e-14, - "particle_momentum_y": 2.3439200330576348e-14, - "particle_momentum_z": 2.3559200621107925e-14, - "particle_position_x": 2457367.4582781536, + "particle_momentum_x": 2.3389474647479255e-14, + "particle_momentum_y": 2.3425362750821152e-14, + "particle_momentum_z": 2.3515097401900785e-14, + "particle_position_x": 2457367.458278154, "particle_position_y": 2457512.044373057, - "particle_position_z": 4914475.776513074, + "particle_position_z": 4914475.776513075, "particle_weight": 3072.000000000002 }, - "alpha3": { - "particle_momentum_x": 4.857656981301579e-16, - "particle_momentum_y": 4.811177053359803e-16, - "particle_momentum_z": 4.766749863500389e-16, - "particle_position_x": 51282.9136300544, - "particle_position_y": 51045.39533309579, - "particle_position_z": 101578.82008857065, - "particle_weight": 1.0257306544231767e+28 - }, - "alpha2": { - "particle_momentum_x": 4.071664828713061e-15, - "particle_momentum_y": 4.119627260272026e-15, - "particle_momentum_z": 4.181812341443065e-15, - "particle_position_x": 405848.1200755021, - "particle_position_y": 408011.53191288817, - "particle_position_z": 866330.5923068221, - "particle_weight": 1.9180757241165136e+19 + "boron5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.33129275695, + "particle_position_y": 409518.5558814353, + "particle_position_z": 819306.5006950963, + "particle_weight": 1023.9999999999999 }, "proton2": { "particle_momentum_x": 0.0, @@ -72,7 +36,16 @@ "particle_position_x": 4095630.698135355, "particle_position_y": 4096073.5517983637, "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240716198e+29 + "particle_weight": 1.0227810240721708e+29 + }, + "proton3": { + "particle_momentum_x": 1.6847690263373399e-15, + "particle_momentum_y": 1.682851264092124e-15, + "particle_momentum_z": 1.6803794796027466e-15, + "particle_position_x": 2457363.4111393224, + "particle_position_y": 2457049.324917686, + "particle_position_z": 4914549.051577593, + "particle_weight": 1.0236635064099502e+31 }, "proton4": { "particle_momentum_x": 0.0, @@ -83,14 +56,23 @@ "particle_position_z": 818958.0399127571, "particle_weight": 1.0240000000000003e+38 }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 + "alpha5": { + "particle_momentum_x": 2.3387550695963677e-14, + "particle_momentum_y": 2.33445389960837e-14, + "particle_momentum_z": 2.3635948557453833e-14, + "particle_position_x": 2457556.8571638414, + "particle_position_y": 2457059.6353793237, + "particle_position_z": 4915847.043341329, + "particle_weight": 3.0719999999999984e-18 + }, + "alpha3": { + "particle_momentum_x": 4.72203056468584e-16, + "particle_momentum_y": 4.622060921861793e-16, + "particle_momentum_z": 4.5443204949049515e-16, + "particle_position_x": 50603.134609459696, + "particle_position_y": 48681.319634796026, + "particle_position_z": 100449.24933492322, + "particle_weight": 1.0094807701511393e+28 }, "proton5": { "particle_momentum_x": 0.0, @@ -101,23 +83,14 @@ "particle_position_z": 819309.1804186807, "particle_weight": 1.0240000000000003e+38 }, - "boron5": { + "proton1": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.33129275695, - "particle_position_y": 409518.5558814353, - "particle_position_z": 819306.5006950963, - "particle_weight": 1023.9999999999999 - }, - "alpha1": { - "particle_momentum_x": 4.659632773478301e-15, - "particle_momentum_y": 4.617309192789389e-15, - "particle_momentum_z": 4.6173901462667306e-15, - "particle_position_x": 453219.3436004627, - "particle_position_y": 457393.42114332493, - "particle_position_z": 970481.0580153911, - "particle_weight": 1.8966442168729786e-27 + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 }, "boron2": { "particle_momentum_x": 0.0, @@ -126,6 +99,33 @@ "particle_position_x": 409798.0158217681, "particle_position_y": 409670.9858143465, "particle_position_z": 819255.8152412223, - "particle_weight": 1.023999999936064e+29 + "particle_weight": 1.0239999999366152e+29 + }, + "alpha1": { + "particle_momentum_x": 4.665933695243743e-15, + "particle_momentum_y": 4.603805875733438e-15, + "particle_momentum_z": 4.706765986105302e-15, + "particle_position_x": 461871.79172011977, + "particle_position_y": 461162.2166206925, + "particle_position_z": 969262.7809050508, + "particle_weight": 1.9171578359058453e-27 + }, + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.119663506409945e+31 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json index 67737738b2c..53c3708056f 100644 --- a/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json +++ b/Regression/Checksum/benchmarks_json/Python_dsmc_1d.json @@ -1,27 +1,27 @@ { "lev=0": { - "rho_electrons": 0.004437417901725754, - "rho_he_ions": 0.005200476260168854 + "rho_electrons": 0.004434311371276535, + "rho_he_ions": 0.005200489146365628 }, "he_ions": { - "particle_momentum_x": 2.771800535001148e-19, - "particle_momentum_y": 2.759439998478633e-19, - "particle_momentum_z": 3.6231785185705813e-19, - "particle_position_x": 2201.7215060103235, - "particle_weight": 17191519531250.002 + "particle_momentum_x": 2.7688639005236794e-19, + "particle_momentum_y": 2.760648864116014e-19, + "particle_momentum_z": 3.6226628630061563e-19, + "particle_position_x": 2201.614485497906, + "particle_weight": 17190996093750.002 }, "neutrals": { - "particle_momentum_x": 1.4052816330675472e-19, - "particle_momentum_y": 1.4027759931535706e-19, - "particle_momentum_z": 1.4119910335756157e-19, - "particle_position_x": 1121.3642639505965, + "particle_momentum_x": 1.4040775167811838e-19, + "particle_momentum_y": 1.4009514702703257e-19, + "particle_momentum_z": 1.4093144247152345e-19, + "particle_position_x": 1119.524782452131, "particle_weight": 6.4588e+19 }, "electrons": { - "particle_momentum_x": 3.535219734698525e-20, - "particle_momentum_y": 3.5373517451957064e-20, - "particle_momentum_z": 1.2562184152660903e-19, - "particle_position_x": 2141.9783924023695, - "particle_weight": 14592914062500.002 + "particle_momentum_x": 3.5193566020597954e-20, + "particle_momentum_y": 3.5368803263788353e-20, + "particle_momentum_z": 1.2572273121326582e-19, + "particle_position_x": 2139.3709122461873, + "particle_weight": 14579566406250.002 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json index 742cb775230..96a2f5cbdac 100644 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ b/Regression/Checksum/benchmarks_json/collisionISO.json @@ -1,13 +1,4 @@ { - "electron": { - "particle_momentum_x": 3.5796522874263304e-19, - "particle_momentum_y": 3.580087349086981e-19, - "particle_momentum_z": 3.5805012694309815e-19, - "particle_position_x": 1.024116253197213, - "particle_position_y": 1.0238635904620974, - "particle_position_z": 1.0240133505121767, - "particle_weight": 714240000000.0 - }, "lev=0": { "Bx": 0.0, "By": 0.0, @@ -18,5 +9,14 @@ "jx": 0.0, "jy": 0.0, "jz": 0.0 + }, + "electron": { + "particle_momentum_x": 3.5856774369040155e-19, + "particle_momentum_y": 3.5832349891760786e-19, + "particle_momentum_z": 3.5757013442445486e-19, + "particle_position_x": 1.0241562532152744, + "particle_position_y": 1.0238555904782611, + "particle_position_z": 1.0240053504978093, + "particle_weight": 714240000000.0 } } \ No newline at end of file diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 807dc827b6b..bd6ec84dd69 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -174,7 +174,7 @@ public: #endif if (m_have_product_species){ species1.defineAllParticleTiles(); - species2.defineAllParticleTiles(); + if (!m_isSameSpecies) { species2.defineAllParticleTiles(); } } // Enable tiling @@ -214,7 +214,7 @@ public: // the colliding species (`species1` and `species2`) may be removed // (i.e., marked as invalid) in the process of creating new product particles. species1.deleteInvalidParticles(); - species2.deleteInvalidParticles(); + if (!m_isSameSpecies) { species2.deleteInvalidParticles(); } } } } @@ -325,6 +325,25 @@ public: p_n_pairs_in_each_cell, pair_offsets.data()); index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); + amrex::Gpu::DeviceVector n_ind_pairs_in_each_cell(n_cells+1); + index_type* AMREX_RESTRICT p_n_ind_pairs_in_each_cell = n_ind_pairs_in_each_cell.dataPtr(); + + amrex::ParallelFor( n_cells+1, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + const auto n_part_in_cell = (i_cell < n_cells)? cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]: 0; + // number of independent collisions in each cell + p_n_ind_pairs_in_each_cell[i_cell] = n_part_in_cell/2; + } + ); + + // start indices of independent collisions. + amrex::Gpu::DeviceVector coll_offsets(n_cells+1); + // number of total independent collision pairs + const auto n_independent_pairs = (int) amrex::Scan::ExclusiveSum(n_cells+1, + p_n_ind_pairs_in_each_cell, coll_offsets.data(), amrex::Scan::RetSum{true}); + index_type* AMREX_RESTRICT p_coll_offsets = coll_offsets.dataPtr(); + // mask: equal to 1 if particle creation occurs for a given pair, 0 otherwise amrex::Gpu::DeviceVector mask(n_total_pairs); index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); @@ -343,7 +362,6 @@ public: End of calculations only required when creating product particles */ - // Loop over cells amrex::ParallelForRNG( n_cells, [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept @@ -354,16 +372,44 @@ public: index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; - // Same but for the pairs - index_type const cell_start_pair = have_product_species? - p_pair_offsets[i_cell] : 0; - // Do not collide if there is only one particle in the cell if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } // shuffle ShuffleFisherYates( indices_1, cell_start_1, cell_half_1, engine ); + } + ); + + // Loop over independent particle pairs + // To speed up binary collisions on GPU, we try to expose as much parallelism + // as possible (while avoiding race conditions): Instead of looping with one GPU + // thread per cell, we loop with one GPU thread per "independent pairs" (i.e. pairs + // that do not touch the same macroparticles, so that there is no race condition), + // where the number of independent pairs is determined by the lower number of + // macroparticles of either species, within each cell. + amrex::ParallelForRNG( n_independent_pairs, + [=] AMREX_GPU_DEVICE (int i_coll, amrex::RandomEngine const& engine) noexcept + { + // to avoid type mismatch errors + auto ui_coll = (index_type)i_coll; + + // Use a bisection algorithm to find the index of the cell in which this pair is located + const int i_cell = amrex::bisect( p_coll_offsets, 0, n_cells, ui_coll ); + + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; + + // collision number of the cell + const index_type coll_idx = ui_coll - p_coll_offsets[i_cell]; + + // Same but for the pairs + index_type const cell_start_pair = have_product_species? + p_pair_offsets[i_cell] : 0; + #if defined WARPX_DIM_RZ const int ri = (i_cell - i_cell%nz) / nz; auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; @@ -376,9 +422,9 @@ public: cell_half_1, cell_stop_1, indices_1, indices_1, soa_1, soa_1, get_position_1, get_position_1, - q1, q1, m1, m1, dt, dV, + q1, q1, m1, m1, dt, dV, coll_idx, cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight, engine ); + p_pair_reaction_weight, engine); } ); @@ -444,7 +490,6 @@ public: auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); #endif - /* The following calculations are only required when creating product particles */ @@ -477,6 +522,32 @@ public: p_n_pairs_in_each_cell, pair_offsets.data()); index_type* AMREX_RESTRICT p_pair_offsets = pair_offsets.dataPtr(); + amrex::Gpu::DeviceVector n_ind_pairs_in_each_cell(n_cells+1); + index_type* AMREX_RESTRICT p_n_ind_pairs_in_each_cell = n_ind_pairs_in_each_cell.dataPtr(); + + amrex::ParallelFor( n_cells+1, + [=] AMREX_GPU_DEVICE (int i_cell) noexcept + { + if (i_cell < n_cells) + { + const auto n_part_in_cell_1 = cell_offsets_1[i_cell+1] - cell_offsets_1[i_cell]; + const auto n_part_in_cell_2 = cell_offsets_2[i_cell+1] - cell_offsets_2[i_cell]; + p_n_ind_pairs_in_each_cell[i_cell] = amrex::min(n_part_in_cell_1, n_part_in_cell_2); + } + else + { + p_n_ind_pairs_in_each_cell[i_cell] = 0; + } + } + ); + + // start indices of independent collisions. + amrex::Gpu::DeviceVector coll_offsets(n_cells+1); + // number of total independent collision pairs + const auto n_independent_pairs = (int) amrex::Scan::ExclusiveSum(n_cells+1, + p_n_ind_pairs_in_each_cell, coll_offsets.data(), amrex::Scan::RetSum{true}); + index_type* AMREX_RESTRICT p_coll_offsets = coll_offsets.dataPtr(); + // mask: equal to 1 if particle creation occurs for a given pair, 0 otherwise amrex::Gpu::DeviceVector mask(n_total_pairs); index_type* AMREX_RESTRICT p_mask = mask.dataPtr(); @@ -507,9 +578,6 @@ public: // Same for species 2 index_type const cell_start_2 = cell_offsets_2[i_cell]; index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; - // Same but for the pairs - index_type const cell_start_pair = have_product_species? - p_pair_offsets[i_cell] : 0; // ux from species1 can be accessed like this: // ux_1[ indices_1[i] ], where i is between @@ -522,6 +590,44 @@ public: // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); + } + ); + + // Loop over independent particle pairs + // To speed up binary collisions on GPU, we try to expose as much parallelism + // as possible (while avoiding race conditions): Instead of looping with one GPU + // thread per cell, we loop with one GPU thread per "independent pairs" (i.e. pairs + // that do not touch the same macroparticles, so that there is no race condition), + // where the number of independent pairs is determined by the lower number of + // macroparticles of either species, within each cell. + amrex::ParallelForRNG( n_independent_pairs, + [=] AMREX_GPU_DEVICE (int i_coll, amrex::RandomEngine const& engine) noexcept + { + // to avoid type mismatch errors + auto ui_coll = (index_type)i_coll; + + // Use a bisection algorithm to find the index of the cell in which this pair is located + const int i_cell = amrex::bisect( p_coll_offsets, 0, n_cells, ui_coll ); + + // The particles from species1 that are in the cell `i_cell` are + // given by the `indices_1[cell_start_1:cell_stop_1]` + index_type const cell_start_1 = cell_offsets_1[i_cell]; + index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + // Same for species 2 + index_type const cell_start_2 = cell_offsets_2[i_cell]; + index_type const cell_stop_2 = cell_offsets_2[i_cell+1]; + + // collision number of the cell + const index_type coll_idx = ui_coll - p_coll_offsets[i_cell]; + + // Same but for the pairs + index_type const cell_start_pair = have_product_species? + p_pair_offsets[i_cell]: 0; + + // ux from species1 can be accessed like this: + // ux_1[ indices_1[i] ], where i is between + // cell_start_1 (inclusive) and cell_start_2 (exclusive) + #if defined WARPX_DIM_RZ const int ri = (i_cell - i_cell%nz) / nz; auto dV = MathConst::pi*(2.0_prt*ri+1.0_prt)*dr*dr*dz; @@ -533,9 +639,9 @@ public: cell_start_1, cell_stop_1, cell_start_2, cell_stop_2, indices_1, indices_2, soa_1, soa_2, get_position_1, get_position_2, - q1, q2, m1, m2, dt, dV, + q1, q2, m1, m2, dt, dV, coll_idx, cell_start_pair, p_mask, p_pair_indices_1, p_pair_indices_2, - p_pair_reaction_weight, engine ); + p_pair_reaction_weight, engine); } ); diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index d7403eeacf9..e407ae1603b 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -37,6 +37,7 @@ * @param[in] dV is the volume of the corresponding cell. * @param[in] engine the random number generator state & factory * @param[in] isSameSpecies whether this is an intra-species collision process + * @param[in] coll_idx is the collision index offset. */ template @@ -52,10 +53,12 @@ void ElasticCollisionPerez ( T_PR const T1, T_PR const T2, T_R const dt, T_PR const L, T_R const dV, amrex::RandomEngine const& engine, - bool const isSameSpecies) + bool const isSameSpecies, T_index coll_idx) { const T_index NI1 = I1e - I1s; const T_index NI2 = I2e - I2s; + const T_index max_N = amrex::max(NI1,NI2); + const T_index min_N = amrex::max(NI1,NI2); T_PR * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; T_PR * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; @@ -95,7 +98,7 @@ void ElasticCollisionPerez ( n1 = n1 / dV; n2 = n2 / dV; { T_index i1 = I1s; T_index i2 = I2s; - for (T_index k = 0; k < amrex::max(NI1,NI2); ++k) + for (T_index k = 0; k < max_N; ++k) { n12 += amrex::min( w1[ I1[i1] ], w2[ I2[i2] ] ); ++i1; if ( i1 == I1e ) { i1 = I1s; } @@ -126,10 +129,13 @@ void ElasticCollisionPerez ( // call UpdateMomentumPerezElastic() { - T_index i1 = I1s; T_index i2 = I2s; - for (T_index k = 0; k < amrex::max(NI1,NI2); ++k) - { + T_index i1 = I1s + coll_idx; + T_index i2 = I2s + coll_idx; + // we will start from collision number = coll_idx and then add + // stride (smaller set size) until we do all collisions (larger set size) + for (T_index k = coll_idx; k < max_N; k += min_N) + { #if (defined WARPX_DIM_RZ) /* In RZ geometry, macroparticles can collide with other macroparticles * in the same *cylindrical* cell. For this reason, collisions between macroparticles @@ -159,8 +165,12 @@ void ElasticCollisionPerez ( u1y[I1[i1]] = u1xbuf_new*std::sin(-theta) + u1y[I1[i1]]*std::cos(-theta); #endif - ++i1; if ( i1 == I1e ) { i1 = I1s; } - ++i2; if ( i2 == I2e ) { i2 = I2s; } + if (max_N == NI1) { + i1 += min_N; + } + if (max_N == NI2) { + i2 += min_N; + } } } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index b31fa3922b9..79fe1d42b7d 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -76,6 +76,7 @@ public: * @param[in] m1,m2 are masses. * @param[in] dt is the time step length between two collision calls. * @param[in] dV is the volume of the corresponding cell. + * @param[in] coll_idx is the collision index offset. * @param[in] engine the random engine. */ AMREX_GPU_HOST_DEVICE AMREX_INLINE @@ -88,7 +89,7 @@ public: GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const q1, amrex::ParticleReal const q2, amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, index_type const /*cell_start_pair*/, index_type* /*p_mask*/, index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, amrex::ParticleReal* /*p_pair_reaction_weight*/, @@ -100,7 +101,7 @@ public: I1s, I1e, I2s, I2e, I1, I2, soa_1, soa_2, q1, q2, m1, m2, -1.0_prt, -1.0_prt, - dt, m_CoulombLog, dV, engine, m_isSameSpecies); + dt, m_CoulombLog, dV, engine, m_isSameSpecies, coll_idx); } amrex::ParticleReal m_CoulombLog; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H index 1039ee96969..3c3849580d1 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -75,6 +75,7 @@ public: * @param[in] m1,m2 are masses. * @param[in] dt is the time step length between two collision calls. * @param[in] dV is the volume of the corresponding cell. + * @param[in] coll_idx is the collision index offset. * @param[in] cell_start_pair is the start index of the pairs in that cell. * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given * pair. It is only needed here to store information that will be used later on when actually @@ -97,7 +98,7 @@ public: GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, @@ -117,10 +118,9 @@ public: const index_type NI1 = I1e - I1s; const index_type NI2 = I2e - I2s; const index_type max_N = amrex::max(NI1,NI2); + const index_type min_N = amrex::min(NI1,NI2); - index_type i1 = I1s; - index_type i2 = I2s; - index_type pair_index = cell_start_pair; + index_type pair_index = cell_start_pair + coll_idx; // Because the number of particles of each species is not always equal (NI1 != NI2 // in general), some macroparticles will be paired with multiple macroparticles of the @@ -135,8 +135,11 @@ public: amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; amrex::ParticleReal * const AMREX_RESTRICT theta2 = soa_2.m_rdata[PIdx::theta]; #endif - - for (index_type k = 0; k < max_N; ++k) + index_type i1 = I1s + coll_idx; + index_type i2 = I2s + coll_idx; + // we will start from collision number = coll_idx and then add + // stride (smaller set size) until we do all collisions (larger set size) + for (index_type k = coll_idx; k < max_N; k += min_N) { // c1k : how many times the current particle of species 1 is paired with a particle // of species 2. Same for c2k. @@ -175,9 +178,13 @@ public: p_pair_indices_1[pair_index] = I1[i1]; p_pair_indices_2[pair_index] = I2[i2]; - ++i1; if ( i1 == I1e ) { i1 = I1s; } - ++i2; if ( i2 == I2e ) { i2 = I2s; } - ++pair_index; + if (max_N == NI1) { + i1 += min_N; + } + if (max_N == NI2) { + i2 += min_N; + } + pair_index += min_N; } } diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index 074226d8227..a44ab90c4cc 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -162,7 +162,7 @@ public: // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX // to replace the following lambda. - auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + auto const atomicSetIdInvalid = [] AMREX_GPU_DEVICE (uint64_t & idcpu) { #if defined(AMREX_USE_OMP) #pragma omp atomic write @@ -179,11 +179,11 @@ public: // setting its id to invalid if (w1[p_pair_indices_1[i]] <= std::numeric_limits::min()) { - atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + atomicSetIdInvalid(idcpu1[p_pair_indices_1[i]]); } if (w2[p_pair_indices_2[i]] <= std::numeric_limits::min()) { - atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); + atomicSetIdInvalid(idcpu2[p_pair_indices_2[i]]); } // Set the child particle properties appropriately diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index b3826114b09..a7dbc67b862 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -110,6 +110,7 @@ public: * @param[in] m1,m2 are masses. * @param[in] dt is the time step length between two collision calls. * @param[in] dV is the volume of the corresponding cell. + * @param[in] coll_idx is the collision index offset. * @param[in] cell_start_pair is the start index of the pairs in that cell. * @param[out] p_mask is a mask that will be set to true if a fusion event occurs for a given * pair. It is only needed here to store information that will be used later on when actually @@ -132,7 +133,7 @@ public: GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, index_type const cell_start_pair, index_type* AMREX_RESTRICT p_mask, index_type* AMREX_RESTRICT p_pair_indices_1, index_type* AMREX_RESTRICT p_pair_indices_2, amrex::ParticleReal* AMREX_RESTRICT p_pair_reaction_weight, @@ -152,10 +153,9 @@ public: const index_type NI1 = I1e - I1s; const index_type NI2 = I2e - I2s; const index_type max_N = amrex::max(NI1,NI2); + const index_type min_N = amrex::min(NI1,NI2); - index_type i1 = I1s; - index_type i2 = I2s; - index_type pair_index = cell_start_pair; + index_type pair_index = cell_start_pair + coll_idx; // Because the number of particles of each species is not always equal (NI1 != NI2 // in general), some macroparticles will be paired with multiple macroparticles of the @@ -174,8 +174,11 @@ public: amrex::ParticleReal * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; amrex::ParticleReal * const AMREX_RESTRICT theta2 = soa_2.m_rdata[PIdx::theta]; #endif - - for (index_type k = 0; k < max_N; ++k) + index_type i1 = I1s + coll_idx; + index_type i2 = I2s + coll_idx; + // we will start from collision number = coll_idx and then add + // stride (smaller set size) until we do all collisions (larger set size) + for (index_type k = coll_idx; k < max_N; k += min_N) { // c1k : how many times the current particle of species 1 is paired with a particle // of species 2. Same for c2k. @@ -215,9 +218,13 @@ public: p_pair_indices_1[pair_index] = I1[i1]; p_pair_indices_2[pair_index] = I2[i2]; - ++i1; if ( i1 == I1e ) { i1 = I1s; } - ++i2; if ( i2 == I2e ) { i2 = I2s; } - ++pair_index; + if (max_N == NI1) { + i1 += min_N; + } + if (max_N == NI2) { + i2 += min_N; + } + pair_index += min_N; } } diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index 4a77006dd92..580ddf57290 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -204,7 +204,7 @@ public: // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX // to replace the following lambda. - auto const atomicSetIdMinus = [] AMREX_GPU_DEVICE (uint64_t & idcpu) + auto const atomicSetIdInvalid = [] AMREX_GPU_DEVICE (uint64_t & idcpu) { #if defined(AMREX_USE_OMP) #pragma omp atomic write @@ -218,15 +218,15 @@ public: }; // If the colliding particle weight decreases to zero, remove particle by - // setting its id to -1 - if (w1[p_pair_indices_1[i]] <= amrex::ParticleReal(0.)) + // setting its id to invalid + if (w1[p_pair_indices_1[i]] <= std::numeric_limits::min()) { - atomicSetIdMinus(idcpu1[p_pair_indices_1[i]]); + atomicSetIdInvalid(idcpu1[p_pair_indices_1[i]]); } - if (w2[p_pair_indices_2[i]] <= amrex::ParticleReal(0.)) + if (w2[p_pair_indices_2[i]] <= std::numeric_limits::min()) { - atomicSetIdMinus(idcpu2[p_pair_indices_2[i]]); + atomicSetIdInvalid(idcpu2[p_pair_indices_2[i]]); } // Initialize the product particles' momentum, using a function depending on the From 6be423ee57f9da98d03531a1500e919543902a5a Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 15 Apr 2024 18:22:05 -0700 Subject: [PATCH 051/190] Add thermal boundary specification to picmi (#4850) * add thermal boundary specification to picmi.py * include thermal boundary handling for Cartesian grids --- Python/pywarpx/picmi.py | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 5902b958bc9..4bf9d4b4163 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -602,6 +602,11 @@ class CylindricalGrid(picmistandard.PICMI_CylindricalGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -629,6 +634,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -653,6 +661,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles pywarpx.boundary.reflect_all_velocities = self.reflect_all_velocities + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: @@ -706,6 +718,11 @@ class Cartesian1DGrid(picmistandard.PICMI_Cartesian1DGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -730,6 +747,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -746,6 +766,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: @@ -807,6 +831,11 @@ class Cartesian2DGrid(picmistandard.PICMI_Cartesian2DGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -833,6 +862,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -851,6 +883,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: @@ -928,6 +964,11 @@ class Cartesian3DGrid(picmistandard.PICMI_Cartesian3DGrid): warpx_end_moving_window_step: int, default=-1 The timestep at which the moving window ends. If -1, the moving window will continue until the end of the simulation. + + warpx_boundary_u_th: dict, default=None + If a thermal boundary is used for particles, this dictionary should + specify the thermal speed for each species in the form {``: u_th}. + Note: u_th = sqrt(T*q_e/mass)/clight with T in eV. """ def init(self, kw): self.max_grid_size = kw.pop('warpx_max_grid_size', 32) @@ -956,6 +997,9 @@ def init(self, kw): pywarpx.geometry.prob_lo = self.lower_bound # physical domain pywarpx.geometry.prob_hi = self.upper_bound + # if a thermal boundary is used for particles, get the thermal speeds + self.thermal_boundary_u_th = kw.pop('warpx_boundary_u_th', None) + def grid_initialize_inputs(self): pywarpx.amr.n_cell = self.number_of_cells @@ -976,6 +1020,10 @@ def grid_initialize_inputs(self): pywarpx.boundary.particle_lo = self.lower_boundary_conditions_particles pywarpx.boundary.particle_hi = self.upper_boundary_conditions_particles + if self.thermal_boundary_u_th is not None: + for name, val in self.thermal_boundary_u_th.items(): + pywarpx.boundary.__setattr__(f'{name}.u_th', val) + if self.moving_window_velocity is not None and np.any(np.not_equal(self.moving_window_velocity, 0.)): pywarpx.warpx.do_moving_window = 1 if self.moving_window_velocity[0] != 0.: From 83d801726c457d3c5ab72bbb5d11b8da5131f3f3 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 16 Apr 2024 12:46:08 -0700 Subject: [PATCH 052/190] Add M. Haseeb in the list of developers (#4862) * Add M. Haseeb in the list of developers * Update .zenodo.json Co-authored-by: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> --------- Co-authored-by: Muhammad Haseeb <14217455+mhaseeb123@users.noreply.github.com> --- .zenodo.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.zenodo.json b/.zenodo.json index 5431e40c8bb..a2c9b443f99 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -98,6 +98,11 @@ "name": "Gu, Junmin", "orcid": "0000-0002-1521-8534" }, + { + "affiliation": "Lawrence Berkeley National Laboratory", + "name": "Haseeb, Muhammad", + "orcid": "0000-0002-0697-6894" + }, { "affiliation": "Lawrence Berkeley National Laboratory", "name": "Jambunathan, Revathi", From 3b18ed71d00efcd29e65b6b8f6722ff106a9e5f7 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 18 Apr 2024 07:44:14 +0200 Subject: [PATCH 053/190] Update pyAMReX (#4870) Pull in the latest `development`. --- cmake/dependencies/pyAMReX.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index be0e857f6f5..038b98c191b 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "a9e0188ac11865f395a01f8b3195dace67b1f8e8" +set(WarpX_pyamrex_branch "28897d26f00707ee33d2423b8c24a020cd0767e0" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") From a082660a73731658398bb8e027148d82342e46b2 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 18 Apr 2024 18:24:07 -0700 Subject: [PATCH 054/190] Add function to set value of `do_not_push` (#4868) * Add function to set value of `do_no_push` * clang-tidy fix --- Source/Particles/WarpXParticleContainer.H | 4 +++- Source/Python/Particles/WarpXParticleContainer.cpp | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 61a87a12830..d4c325fbfb8 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -426,6 +426,8 @@ public: virtual QuantumSynchrotronEngine* get_quantum_sync_engine_ptr () const {return nullptr;} #endif + void setDoNotPush (bool flag) { do_not_push = flag; } + protected: int species_id; @@ -442,7 +444,7 @@ protected: //! instead of gathering fields from the finest patch level, gather from the coarsest bool m_gather_from_main_grid = false; - int do_not_push = 0; + bool do_not_push = false; int do_not_gather = 0; // Whether to allow particles outside of the simulation domain to be diff --git a/Source/Python/Particles/WarpXParticleContainer.cpp b/Source/Python/Particles/WarpXParticleContainer.cpp index 3feae5d8c90..aa2cd7a2091 100644 --- a/Source/Python/Particles/WarpXParticleContainer.cpp +++ b/Source/Python/Particles/WarpXParticleContainer.cpp @@ -137,5 +137,9 @@ void init_WarpXParticleContainer (py::module& m) }, py::arg("lev"), py::arg("local") ) + .def("set_do_not_push", + [](WarpXParticleContainer& pc, bool flag) { pc.setDoNotPush(flag); }, + py::arg("flag") + ) ; } From fc94b2341a3a567af471585f58478a58ebd1dbc0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 22 Apr 2024 16:09:04 -0700 Subject: [PATCH 055/190] Downgrade Reg Tests (#4882) Downgrade the regression testing scripts to work around https://github.com/AMReX-Codes/regression_testing/issues/136 --- run_test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/run_test.sh b/run_test.sh index 774a220dc06..2848cce415d 100755 --- a/run_test.sh +++ b/run_test.sh @@ -80,6 +80,8 @@ cd - # Clone the AMReX regression test utility git clone https://github.com/AMReX-Codes/regression_testing.git +# FIXME: https://github.com/AMReX-Codes/regression_testing/issues/136 +cd regression_testing && git checkout db67b6bd80cff208dba59ec2938ffd02e8ad9cef && cd - # Prepare regression tests mkdir -p rt-WarpX/WarpX-benchmarks From 60e9b8e0ca7ca1bdc2c7e845f46a5eeb8ba55afe Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 23 Apr 2024 01:17:47 +0200 Subject: [PATCH 056/190] remove unnecessary include of WarpX.H (#4877) --- .../Particles/Collision/BinaryCollision/ParticleCreationFunc.H | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index 580ddf57290..83312017e44 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -15,7 +15,6 @@ #include "Particles/ParticleCreation/SmartCopy.H" #include "Particles/MultiParticleContainer.H" #include "Particles/WarpXParticleContainer.H" -#include "WarpX.H" #include #include From 1d3c837bdab502fa17d8b379773d3bc0c42fcfe0 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 23 Apr 2024 01:18:13 +0200 Subject: [PATCH 057/190] Ensure that only necessary headers are included in FieldAccessorFunctors.H (#4878) --- .../FiniteDifferenceAlgorithms/FieldAccessorFunctors.H | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H index d4fdf207e52..05b1db1fe94 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/FieldAccessorFunctors.H @@ -8,11 +8,9 @@ #ifndef WARPX_FIELD_ACCESSOR_FUNCTORS_H #define WARPX_FIELD_ACCESSOR_FUNCTORS_H -#include "WarpX.H" -#include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" - #include - +#include +#include /** * \brief Functor that returns the division of the source m_field Array4 value From b900941c295456141bc00a2aef543cb3b04488fb Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 23 Apr 2024 01:19:11 +0200 Subject: [PATCH 058/190] remove unnecessary include from FlushFormat.H (#4879) --- Source/Diagnostics/FlushFormats/FlushFormat.H | 1 - 1 file changed, 1 deletion(-) diff --git a/Source/Diagnostics/FlushFormats/FlushFormat.H b/Source/Diagnostics/FlushFormats/FlushFormat.H index 65741e4ff20..be1322bd61b 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormat.H +++ b/Source/Diagnostics/FlushFormats/FlushFormat.H @@ -5,7 +5,6 @@ #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "Particles/MultiParticleContainer.H" -#include "WarpX.H" class FlushFormat { From be379d21bcd963e789e608bbf035513ffc331a6a Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 23 Apr 2024 18:05:37 +0200 Subject: [PATCH 059/190] Further downgrade regression_testing (#4884) --- run_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_test.sh b/run_test.sh index 2848cce415d..23602682ff5 100755 --- a/run_test.sh +++ b/run_test.sh @@ -81,7 +81,7 @@ cd - # Clone the AMReX regression test utility git clone https://github.com/AMReX-Codes/regression_testing.git # FIXME: https://github.com/AMReX-Codes/regression_testing/issues/136 -cd regression_testing && git checkout db67b6bd80cff208dba59ec2938ffd02e8ad9cef && cd - +cd regression_testing && git checkout 93ddfb11456f47d6555c39388ba1a4ead61fbf4e && cd - # Prepare regression tests mkdir -p rt-WarpX/WarpX-benchmarks From cceae549a76a92482314d6685396077f1cff8a6d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 23 Apr 2024 10:06:01 -0700 Subject: [PATCH 060/190] DepositCharge: Component 1 Allocated? (#4859) * DepositCharge: Component 1 Allocated? * Move assert Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --------- Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- Source/Particles/PhysicalParticleContainer.cpp | 2 ++ Source/Particles/WarpXParticleContainer.cpp | 12 ++++++++++++ 2 files changed, 14 insertions(+) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 4b2d73c7d6a..e979afa9f77 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2228,6 +2228,8 @@ PhysicalParticleContainer::Evolve (int lev, // Deposit charge after particle push, in component 1 of MultiFab rho. // (Skipped for electrostatic solver, as this may lead to out-of-bounds) if (WarpX::electrostatic_solver_id == ElectrostaticSolverAlgo::None) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(rho->nComp() >= 2, + "Cannot deposit charge in rho component 1: only component 0 is allocated!"); const int* const AMREX_RESTRICT ion_lev = (do_field_ionization)? pti.GetiAttribs(particle_icomps["ionizationLevel"]).dataPtr():nullptr; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 142e973d1b2..7199d420173 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -883,6 +883,12 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, const long offset, const long np_to_deposit, const int thread_num, const int lev, const int depos_lev) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + rho->nComp() >= icomp - 1, + "Cannot deposit charge in rho component icomp=" + std::to_string(icomp) + + ": not enough components allocated (" + std::to_string(rho->nComp()) + "!" + ); + if (WarpX::do_shared_mem_charge_deposition) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE((depos_lev==(lev-1)) || @@ -1208,6 +1214,12 @@ WarpXParticleContainer::DepositCharge (std::unique_ptr& rho, const bool apply_boundary_and_scale_volume, const int icomp) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + rho->nComp() >= icomp - 1, + "Cannot deposit charge in rho component icomp=" + std::to_string(icomp) + + ": not enough components allocated (" + std::to_string(rho->nComp()) + "!" + ); + // Reset the rho array if reset is True int const nc = WarpX::ncomps; if (reset) { rho->setVal(0., icomp*nc, nc, rho->nGrowVect()); } From 8344f2ccaceadcdb36866a2143ae007a8d2bd297 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 23 Apr 2024 13:25:02 -0700 Subject: [PATCH 061/190] AMReX: Weekly Update (#4881) --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- run_test.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 06eb90b4395..abd7e54c013 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 7813bb7d6bc6cea658edac1f2bde046c7d174168 && cd - + cd ../amrex && git checkout --detach b752027c1aebdfb4be339b1e30932b4108286a7a && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index a54553689ad..048efbc06be 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 7813bb7d6bc6cea658edac1f2bde046c7d174168 +branch = b752027c1aebdfb4be339b1e30932b4108286a7a [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index f35a7676ab8..1a157d7e022 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 7813bb7d6bc6cea658edac1f2bde046c7d174168 +branch = b752027c1aebdfb4be339b1e30932b4108286a7a [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 9d21519ce71..022939dcbee 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "7813bb7d6bc6cea658edac1f2bde046c7d174168" +set(WarpX_amrex_branch "b752027c1aebdfb4be339b1e30932b4108286a7a" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/run_test.sh b/run_test.sh index 23602682ff5..9bfc3ba5fbf 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 7813bb7d6bc6cea658edac1f2bde046c7d174168 && cd - +cd amrex && git checkout --detach b752027c1aebdfb4be339b1e30932b4108286a7a && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 899345711fa27eaf0c7623bfcaeb0b2d797c4922 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Tue, 23 Apr 2024 14:38:56 -0700 Subject: [PATCH 062/190] Reorder particle boundary handling and call to `Redistribute` (#4852) * Reorder particle boundary handling and call to `Redistribute` * split boundary buffer gathering into separate functions for domain and embedded boundaries * rename `scrapeParticles` -> `scrapeParticlesAtEB` --- Source/EmbeddedBoundary/ParticleScraper.H | 12 ++++++------ Source/Evolve/WarpXEvolve.cpp | 15 ++++++++------- Source/Particles/MultiParticleContainer.H | 4 +++- Source/Particles/MultiParticleContainer.cpp | 12 ++++++++++-- Source/Particles/ParticleBoundaryBuffer.H | 6 ++++-- Source/Particles/ParticleBoundaryBuffer.cpp | 14 +++++++++++--- Source/Particles/PhysicalParticleContainer.cpp | 4 ++-- Source/Particles/WarpXParticleContainer.cpp | 2 +- 8 files changed, 45 insertions(+), 24 deletions(-) diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index d058de7e48f..a382a454ec6 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -65,9 +65,9 @@ */ template ::value, int> foo = 0> void -scrapeParticles (PC& pc, const amrex::Vector& distance_to_eb, int lev, F&& f) +scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, int lev, F&& f) { - scrapeParticles(pc, distance_to_eb, lev, lev, std::forward(f)); + scrapeParticlesAtEB(pc, distance_to_eb, lev, lev, std::forward(f)); } /** @@ -108,9 +108,9 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t */ template ::value, int> foo = 0> void -scrapeParticles (PC& pc, const amrex::Vector& distance_to_eb, F&& f) +scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, F&& f) { - scrapeParticles(pc, distance_to_eb, 0, pc.finestLevel(), std::forward(f)); + scrapeParticlesAtEB(pc, distance_to_eb, 0, pc.finestLevel(), std::forward(f)); } /** @@ -153,10 +153,10 @@ scrapeParticles (PC& pc, const amrex::Vector& distance_t */ template ::value, int> foo = 0> void -scrapeParticles (PC& pc, const amrex::Vector& distance_to_eb, +scrapeParticlesAtEB (PC& pc, const amrex::Vector& distance_to_eb, int lev_min, int lev_max, F&& f) { - BL_PROFILE("scrapeParticles"); + BL_PROFILE("scrapeParticlesAtEB"); for (int lev = lev_min; lev <= lev_max; ++lev) { diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index a8aa89355ed..51fcb5c262c 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -489,13 +489,7 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num mypc->ContinuousFluxInjection(cur_time, dt[0]); mypc->ApplyBoundaryConditions(); - - // interact the particles with EB walls (if present) -#ifdef AMREX_USE_EB - mypc->ScrapeParticles(amrex::GetVecOfConstPtrs(m_distance_to_eb)); -#endif - - m_particle_boundary_buffer->gatherParticles(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); + m_particle_boundary_buffer->gatherParticlesFromDomainBoundaries(*mypc); // Non-Maxwell solver: particles can move by an arbitrary number of cells if( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || @@ -523,6 +517,13 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num } } + // interact the particles with EB walls (if present) +#ifdef AMREX_USE_EB + mypc->ScrapeParticlesAtEB(amrex::GetVecOfConstPtrs(m_distance_to_eb)); + m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries(*mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb)); + mypc->deleteInvalidParticles(); +#endif + if (sort_intervals.contains(step+1)) { if (verbose) { amrex::Print() << Utils::TextMsg::Info("re-sorting particles"); diff --git a/Source/Particles/MultiParticleContainer.H b/Source/Particles/MultiParticleContainer.H index 0421febc475..f007d5088e3 100644 --- a/Source/Particles/MultiParticleContainer.H +++ b/Source/Particles/MultiParticleContainer.H @@ -213,6 +213,8 @@ public: void defineAllParticleTiles (); + void deleteInvalidParticles (); + void RedistributeLocal (int num_ghost); /** Apply BC. For now, just discard particles outside the domain, regardless @@ -294,7 +296,7 @@ public: PhysicalParticleContainer& GetPCtmp () { return *pc_tmp; } - void ScrapeParticles (const amrex::Vector& distance_to_eb); + void ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb); std::string m_B_ext_particle_s = "none"; std::string m_E_ext_particle_s = "none"; diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 97c51c6350b..dda32467925 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -632,6 +632,14 @@ MultiParticleContainer::defineAllParticleTiles () } } +void +MultiParticleContainer::deleteInvalidParticles () +{ + for (auto& pc : allcontainers) { + pc->deleteInvalidParticles(); + } +} + void MultiParticleContainer::RedistributeLocal (const int num_ghost) { @@ -945,11 +953,11 @@ void MultiParticleContainer::CheckIonizationProductSpecies() } } -void MultiParticleContainer::ScrapeParticles (const amrex::Vector& distance_to_eb) +void MultiParticleContainer::ScrapeParticlesAtEB (const amrex::Vector& distance_to_eb) { #ifdef AMREX_USE_EB for (auto& pc : allcontainers) { - scrapeParticles(*pc, distance_to_eb, ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB(*pc, distance_to_eb, ParticleBoundaryProcess::Absorb()); } #else amrex::ignore_unused(distance_to_eb); diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index 20bc63f7a19..895a3810852 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -39,8 +39,10 @@ public: const std::vector& getSpeciesNames() const; - void gatherParticles (MultiParticleContainer& mypc, - const amrex::Vector& distance_to_eb); + void gatherParticlesFromDomainBoundaries (MultiParticleContainer& mypc); + void gatherParticlesFromEmbeddedBoundaries ( + MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb + ); void redistribute (); void clearParticles (); diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 25b77ddc496..c8c683f0abf 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -354,8 +354,7 @@ void ParticleBoundaryBuffer::clearParticles (int const i) { } } -void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, - const amrex::Vector& distance_to_eb) +void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleContainer& mypc) { WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles"); @@ -439,10 +438,19 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, } } } +} +void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries ( + MultiParticleContainer& mypc, const amrex::Vector& distance_to_eb) +{ #ifdef AMREX_USE_EB WARPX_PROFILE("ParticleBoundaryBuffer::gatherParticles::EB"); + using PIter = amrex::ParConstIterSoA; + const auto& warpx_instance = WarpX::GetInstance(); + const amrex::Geometry& geom = warpx_instance.Geom(0); + auto plo = geom.ProbLoArray(); + auto& buffer = m_particle_containers[m_particle_containers.size()-1]; for (int i = 0; i < numSpecies(); ++i) { @@ -526,7 +534,7 @@ void ParticleBoundaryBuffer::gatherParticles (MultiParticleContainer& mypc, } } #else - amrex::ignore_unused(distance_to_eb); + amrex::ignore_unused(mypc, distance_to_eb); #endif } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e979afa9f77..f8afb1d39af 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1476,7 +1476,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); #endif // The function that calls this is responsible for redistributing particles. @@ -1973,7 +1973,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticles(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB(tmp_pc, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); #endif // Redistribute the new particles that were added to the temporary container. diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 7199d420173..8692dff3302 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -301,7 +301,7 @@ WarpXParticleContainer::AddNParticles (int /*lev*/, long n, // Remove particles that are inside the embedded boundaries #ifdef AMREX_USE_EB auto & distance_to_eb = WarpX::GetInstance().GetDistanceToEB(); - scrapeParticles( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); + scrapeParticlesAtEB( *this, amrex::GetVecOfConstPtrs(distance_to_eb), ParticleBoundaryProcess::Absorb()); deleteInvalidParticles(); #endif } From ae5085533c9075fe0e650b12b10194a9f0c2c9e6 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Tue, 23 Apr 2024 17:00:23 -0700 Subject: [PATCH 063/190] remove offending calls to pip install in macOS build (#4887) --- .github/workflows/macos.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index f34f9f3534d..21d5ae04faf 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -36,9 +36,6 @@ jobs: brew tap openpmd/openpmd brew install openpmd-api - python3 -m pip install --upgrade pip - python3 -m pip install --upgrade virtualenv - python3 -m venv py-venv source py-venv/bin/activate python3 -m pip install --upgrade pip From cbe43551bb1f212eb42aff12db94643801e28678 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 24 Apr 2024 02:00:58 +0200 Subject: [PATCH 064/190] make sure that include guards start with WARPX_ (#4876) --- Source/BoundaryConditions/PML_current.H | 6 +++--- Source/Diagnostics/BTD_Plotfile_Header_Impl.H | 6 +++--- Source/EmbeddedBoundary/DistanceToEB.H | 6 +++--- Source/EmbeddedBoundary/ParticleBoundaryProcess.H | 6 +++--- Source/EmbeddedBoundary/ParticleScraper.H | 6 +++--- Source/FieldSolver/ElectrostaticSolver.H | 6 +++--- .../FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H | 6 +++--- Source/FieldSolver/WarpXPushFieldsEM_K.H | 6 +++--- Source/FieldSolver/WarpX_QED_K.H | 6 +++--- Source/Fluids/MultiFluidContainer_fwd.H | 1 + Source/Fluids/WarpXFluidContainer_fwd.H | 1 + Source/Initialization/ExternalField.H | 6 +++--- Source/Initialization/ExternalField_fwd.H | 6 +++--- Source/Initialization/GetTemperature.H | 6 +++--- Source/Initialization/GetVelocity.H | 6 +++--- Source/Initialization/InjectorDensity.H | 6 +++--- Source/Initialization/InjectorFlux.H | 6 +++--- Source/Initialization/InjectorMomentum.H | 6 +++--- Source/Initialization/InjectorPosition.H | 6 +++--- Source/Initialization/PlasmaInjector.H | 6 +++--- Source/Initialization/SampleGaussianFluxDistribution.H | 6 +++--- Source/Initialization/TemperatureProperties.H | 6 +++--- Source/Initialization/VelocityProperties.H | 6 +++--- Source/Initialization/WarpXAMReXInit.H | 2 +- Source/Parallelization/GuardCellManager.H | 6 +++--- Source/Particles/Algorithms/KineticEnergy.H | 6 +++--- .../Collision/BinaryCollision/BinaryCollisionUtils.H | 6 +++--- .../BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H | 6 +++--- .../Collision/BinaryCollision/DSMC/CollisionFilterFunc.H | 6 +++--- Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H | 6 +++--- .../Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H | 6 +++--- .../NuclearFusion/BoschHaleFusionCrossSection.H | 6 +++--- .../BinaryCollision/NuclearFusion/NuclearFusionFunc.H | 6 +++--- .../NuclearFusion/ProtonBoronFusionCrossSection.H | 6 +++--- .../NuclearFusion/ProtonBoronFusionInitializeMomentum.H | 6 +++--- .../NuclearFusion/SingleNuclearFusionEvent.H | 6 +++--- .../NuclearFusion/TwoProductFusionInitializeMomentum.H | 6 +++--- .../BinaryCollision/NuclearFusion/TwoProductFusionUtil.H | 6 +++--- .../Collision/BinaryCollision/ParticleCreationFunc.H | 6 +++--- Source/Particles/Deposition/ChargeDeposition.H | 6 +++--- Source/Particles/Deposition/CurrentDeposition.H | 6 +++--- Source/Particles/Deposition/SharedDepositionUtils.H | 6 +++--- Source/Particles/ElementaryProcess/Ionization.H | 6 +++--- Source/Particles/ElementaryProcess/QEDPairGeneration.H | 6 +++--- Source/Particles/ElementaryProcess/QEDPhotonEmission.H | 6 +++--- Source/Particles/ElementaryProcess/QEDSchwingerProcess.H | 6 +++--- Source/Particles/Filter/FilterFunctors.H | 6 +++--- Source/Particles/Gather/FieldGather.H | 6 +++--- Source/Particles/Gather/ScaleFields.H | 2 +- Source/Particles/NamedComponentParticleContainer.H | 6 +++--- Source/Particles/ParticleBoundaries.H | 6 +++--- Source/Particles/ParticleBoundaries_K.H | 6 +++--- Source/Particles/ParticleBoundaryBuffer.H | 6 +++--- Source/Particles/ParticleCreation/DefaultInitialization.H | 6 +++--- Source/Particles/ParticleCreation/FilterCopyTransform.H | 6 +++--- .../ParticleCreation/FilterCreateTransformFromFAB.H | 6 +++--- Source/Particles/ParticleCreation/SmartCopy.H | 6 +++--- Source/Particles/ParticleCreation/SmartCreate.H | 6 +++--- Source/Particles/ParticleCreation/SmartUtils.H | 6 +++--- Source/Particles/ParticleIO.H | 6 +++--- Source/Particles/PinnedMemoryParticleContainer.H | 6 +++--- Source/Particles/ShapeFactors.H | 6 +++--- Source/Utils/WarpXAlgorithmSelection.H | 6 +++--- 63 files changed, 181 insertions(+), 179 deletions(-) diff --git a/Source/BoundaryConditions/PML_current.H b/Source/BoundaryConditions/PML_current.H index 88e94de1e74..9361ce13992 100644 --- a/Source/BoundaryConditions/PML_current.H +++ b/Source/BoundaryConditions/PML_current.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PML_CURRENT_H_ -#define PML_CURRENT_H_ +#ifndef WARPX_PML_CURRENT_H_ +#define WARPX_PML_CURRENT_H_ #include "BoundaryConditions/PMLComponent.H" @@ -143,4 +143,4 @@ void damp_jz_pml (int j, int k, int l, #endif } -#endif +#endif //WARPX_PML_CURRENT_H_ diff --git a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H index 9d3250fa52f..7a1f8275bca 100644 --- a/Source/Diagnostics/BTD_Plotfile_Header_Impl.H +++ b/Source/Diagnostics/BTD_Plotfile_Header_Impl.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef BTD_PLOTFILE_HEADER_IMPL_H -#define BTD_PLOTFILE_HEADER_IMPL_H +#ifndef WARPX_BTD_PLOTFILE_HEADER_IMPL_H +#define WARPX_BTD_PLOTFILE_HEADER_IMPL_H #include #include @@ -342,4 +342,4 @@ public: std::string m_Header_path; }; -#endif +#endif //WARPX_BTD_PLOTFILE_HEADER_IMPL_H diff --git a/Source/EmbeddedBoundary/DistanceToEB.H b/Source/EmbeddedBoundary/DistanceToEB.H index b46def938db..7ee47c1172c 100644 --- a/Source/EmbeddedBoundary/DistanceToEB.H +++ b/Source/EmbeddedBoundary/DistanceToEB.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef DISTANCETOEB_H_ -#define DISTANCETOEB_H_ +#ifndef WARPX_DISTANCETOEB_H_ +#define WARPX_DISTANCETOEB_H_ #include "Utils/TextMsg.H" @@ -129,4 +129,4 @@ amrex::RealVect interp_normal (int i, int j, int k, const amrex::Real W[AMREX_SP } #endif // AMREX_USE_EB -#endif // DISTANCETOEB_H_ +#endif // WARPX_DISTANCETOEB_H_ diff --git a/Source/EmbeddedBoundary/ParticleBoundaryProcess.H b/Source/EmbeddedBoundary/ParticleBoundaryProcess.H index 3099f5df43d..345cafe7b4f 100644 --- a/Source/EmbeddedBoundary/ParticleBoundaryProcess.H +++ b/Source/EmbeddedBoundary/ParticleBoundaryProcess.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARYPROCESS_H_ -#define PARTICLEBOUNDARYPROCESS_H_ +#ifndef WARPX_PARTICLEBOUNDARYPROCESS_H_ +#define WARPX_PARTICLEBOUNDARYPROCESS_H_ #include #include @@ -36,4 +36,4 @@ struct Absorb { }; } -#endif +#endif //WARPX_PARTICLEBOUNDARYPROCESS_H_ diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index a382a454ec6..ea4d0bc5c0e 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLESCRAPER_H_ -#define PARTICLESCRAPER_H_ +#ifndef WARPX_PARTICLESCRAPER_H_ +#define WARPX_PARTICLESCRAPER_H_ #include "EmbeddedBoundary/DistanceToEB.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -216,4 +216,4 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan } } -#endif +#endif //WARPX_PARTICLESCRAPER_H_ diff --git a/Source/FieldSolver/ElectrostaticSolver.H b/Source/FieldSolver/ElectrostaticSolver.H index 0a2abe2d6b2..c5c2ce8b6eb 100644 --- a/Source/FieldSolver/ElectrostaticSolver.H +++ b/Source/FieldSolver/ElectrostaticSolver.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef ELECTROSTATICSOLVER_H_ -#define ELECTROSTATICSOLVER_H_ +#ifndef WARPX_ELECTROSTATICSOLVER_H_ +#define WARPX_ELECTROSTATICSOLVER_H_ #include #include @@ -122,4 +122,4 @@ namespace ElectrostaticSolver { }; } // namespace ElectrostaticSolver -#endif // ELECTROSTATICSOLVER_H_ +#endif // WARPX_ELECTROSTATICSOLVER_H_ diff --git a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H index f84a8f963fc..a8bbc954e29 100644 --- a/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H +++ b/Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef MAGNETOSTATICSOLVER_H_ -#define MAGNETOSTATICSOLVER_H_ +#ifndef WARPX_MAGNETOSTATICSOLVER_H_ +#define WARPX_MAGNETOSTATICSOLVER_H_ #include #include @@ -54,4 +54,4 @@ namespace MagnetostaticSolver { }; } // namespace MagnetostaticSolver -#endif //MAGNETOSTATICSOLVER_H_ +#endif //WARPX_MAGNETOSTATICSOLVER_H_ diff --git a/Source/FieldSolver/WarpXPushFieldsEM_K.H b/Source/FieldSolver/WarpXPushFieldsEM_K.H index d6e7e353a54..4ff7b32b517 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM_K.H +++ b/Source/FieldSolver/WarpXPushFieldsEM_K.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef WarpXPushFieldsEM_K_h -#define WarpXPushFieldsEM_K_h +#ifndef WARPX_PushFieldsEM_K_h +#define WARPX_PushFieldsEM_K_h #include "Utils/WarpXConst.H" @@ -117,4 +117,4 @@ void damp_field_in_guards( } } -#endif +#endif //WARPX_PushFieldsEM_K_h diff --git a/Source/FieldSolver/WarpX_QED_K.H b/Source/FieldSolver/WarpX_QED_K.H index 6c86bba67b6..eccb8d6e72d 100644 --- a/Source/FieldSolver/WarpX_QED_K.H +++ b/Source/FieldSolver/WarpX_QED_K.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef WarpX_QED_K_h -#define WarpX_QED_K_h +#ifndef WARPX_QED_K_h +#define WARPX_QED_K_h #include "Utils/WarpXConst.H" @@ -359,4 +359,4 @@ const amrex::Real dyi = 1._rt/dy; } -#endif +#endif //WARPX_QED_K_h diff --git a/Source/Fluids/MultiFluidContainer_fwd.H b/Source/Fluids/MultiFluidContainer_fwd.H index a1a55c3b450..01edf7f728c 100644 --- a/Source/Fluids/MultiFluidContainer_fwd.H +++ b/Source/Fluids/MultiFluidContainer_fwd.H @@ -6,6 +6,7 @@ */ #ifndef WARPX_MultiFluidContainer_fwd_H_ +#define WARPX_MultiFluidContainer_fwd_H_ class MultiFluidContainer; diff --git a/Source/Fluids/WarpXFluidContainer_fwd.H b/Source/Fluids/WarpXFluidContainer_fwd.H index 77a6bc0c991..86fbff98fd0 100644 --- a/Source/Fluids/WarpXFluidContainer_fwd.H +++ b/Source/Fluids/WarpXFluidContainer_fwd.H @@ -6,6 +6,7 @@ */ #ifndef WARPX_WarpXFluidContainer_fwd_H_ +#define WARPX_WarpXFluidContainer_fwd_H_ class WarpXFluidContainer; diff --git a/Source/Initialization/ExternalField.H b/Source/Initialization/ExternalField.H index 7b64339a042..b32e361bff4 100644 --- a/Source/Initialization/ExternalField.H +++ b/Source/Initialization/ExternalField.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef EXTERNAL_FIELD_H_ -#define EXTERNAL_FIELD_H_ +#ifndef WARPX_EXTERNAL_FIELD_H_ +#define WARPX_EXTERNAL_FIELD_H_ #include "ExternalField_fwd.H" @@ -66,4 +66,4 @@ struct ExternalFieldParams std::string external_fields_path; }; -#endif //EXTERNAL_FIELD_H_ +#endif //WARPX_EXTERNAL_FIELD_H_ diff --git a/Source/Initialization/ExternalField_fwd.H b/Source/Initialization/ExternalField_fwd.H index f7a8547ecce..a2fd6471222 100644 --- a/Source/Initialization/ExternalField_fwd.H +++ b/Source/Initialization/ExternalField_fwd.H @@ -5,9 +5,9 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef EXTERNAL_FIELD_FWD_H_ -#define EXTERNAL_FIELD_FWD_H_ +#ifndef WARPX_EXTERNAL_FIELD_FWD_H_ +#define WARPX_EXTERNAL_FIELD_FWD_H_ struct ExternalFieldParams; -#endif //EXTERNAL_FIELD_FWD_H_ +#endif //WARPX_EXTERNAL_FIELD_FWD_H_ diff --git a/Source/Initialization/GetTemperature.H b/Source/Initialization/GetTemperature.H index 4437928697c..20760863fae 100644 --- a/Source/Initialization/GetTemperature.H +++ b/Source/Initialization/GetTemperature.H @@ -6,8 +6,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef GET_TEMPERATURE_H_ -#define GET_TEMPERATURE_H_ +#ifndef WARPX_GET_TEMPERATURE_H_ +#define WARPX_GET_TEMPERATURE_H_ #include "TemperatureProperties.H" @@ -69,4 +69,4 @@ struct GetTemperature } } }; -#endif +#endif //WARPX_GET_TEMPERATURE_H_ diff --git a/Source/Initialization/GetVelocity.H b/Source/Initialization/GetVelocity.H index 3123708bf06..c8535b8af8e 100644 --- a/Source/Initialization/GetVelocity.H +++ b/Source/Initialization/GetVelocity.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef GET_VELOCITY_H_ -#define GET_VELOCITY_H_ +#ifndef WARPX_GET_VELOCITY_H_ +#define WARPX_GET_VELOCITY_H_ #include "VelocityProperties.H" @@ -86,4 +86,4 @@ struct GetVelocity return m_dir; } }; -#endif +#endif //WARPX_GET_VELOCITY_H_ diff --git a/Source/Initialization/InjectorDensity.H b/Source/Initialization/InjectorDensity.H index 3848a42f4ee..465627f874a 100644 --- a/Source/Initialization/InjectorDensity.H +++ b/Source/Initialization/InjectorDensity.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_DENSITY_H_ -#define INJECTOR_DENSITY_H_ +#ifndef WARPX_INJECTOR_DENSITY_H_ +#define WARPX_INJECTOR_DENSITY_H_ #include "Utils/WarpXConst.H" @@ -218,4 +218,4 @@ struct InjectorDensityDeleter { } }; -#endif +#endif //WARPX_INJECTOR_DENSITY_H_ diff --git a/Source/Initialization/InjectorFlux.H b/Source/Initialization/InjectorFlux.H index e889c8b8966..6d608822c31 100644 --- a/Source/Initialization/InjectorFlux.H +++ b/Source/Initialization/InjectorFlux.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_FLUX_H_ -#define INJECTOR_FLUX_H_ +#ifndef WARPX_INJECTOR_FLUX_H_ +#define WARPX_INJECTOR_FLUX_H_ #include "Utils/WarpXConst.H" @@ -149,4 +149,4 @@ struct InjectorFluxDeleter { } }; -#endif +#endif //WARPX_INJECTOR_FLUX_H_ diff --git a/Source/Initialization/InjectorMomentum.H b/Source/Initialization/InjectorMomentum.H index 2b8b9065a36..52fedcd779f 100644 --- a/Source/Initialization/InjectorMomentum.H +++ b/Source/Initialization/InjectorMomentum.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_MOMENTUM_H_ -#define INJECTOR_MOMENTUM_H_ +#ifndef WARPX_INJECTOR_MOMENTUM_H_ +#define WARPX_INJECTOR_MOMENTUM_H_ #include "GetTemperature.H" #include "GetVelocity.H" @@ -771,4 +771,4 @@ struct InjectorMomentumDeleter { } }; -#endif +#endif //WARPX_INJECTOR_MOMENTUM_H_ diff --git a/Source/Initialization/InjectorPosition.H b/Source/Initialization/InjectorPosition.H index 7168ac93ebb..1b3be9e28a7 100644 --- a/Source/Initialization/InjectorPosition.H +++ b/Source/Initialization/InjectorPosition.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef INJECTOR_POSITION_H_ -#define INJECTOR_POSITION_H_ +#ifndef WARPX_INJECTOR_POSITION_H_ +#define WARPX_INJECTOR_POSITION_H_ #include "InjectorPosition_fwd.H" @@ -255,4 +255,4 @@ private: amrex::Real zmin, zmax; }; -#endif +#endif //WARPX_INJECTOR_POSITION_H_ diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H index 2651aad455f..b9fe2323290 100644 --- a/Source/Initialization/PlasmaInjector.H +++ b/Source/Initialization/PlasmaInjector.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PLASMA_INJECTOR_H_ -#define PLASMA_INJECTOR_H_ +#ifndef WARPX_PLASMA_INJECTOR_H_ +#define WARPX_PLASMA_INJECTOR_H_ #include "InjectorDensity.H" #include "InjectorFlux.H" @@ -202,4 +202,4 @@ protected: void parseFlux (amrex::ParmParse const& pp_species); }; -#endif +#endif //WARPX_PLASMA_INJECTOR_H_ diff --git a/Source/Initialization/SampleGaussianFluxDistribution.H b/Source/Initialization/SampleGaussianFluxDistribution.H index 407c9df632a..65cad141e8f 100644 --- a/Source/Initialization/SampleGaussianFluxDistribution.H +++ b/Source/Initialization/SampleGaussianFluxDistribution.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H -#define SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H +#ifndef WARPX_SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H +#define WARPX_SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H #include @@ -78,4 +78,4 @@ namespace { } } -#endif +#endif //WARPX_SAMPLE_GAUSSIAN_FLUX_DISTRIBUTION_H diff --git a/Source/Initialization/TemperatureProperties.H b/Source/Initialization/TemperatureProperties.H index 4d30b67c851..78a967a32fa 100644 --- a/Source/Initialization/TemperatureProperties.H +++ b/Source/Initialization/TemperatureProperties.H @@ -6,8 +6,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef TEMPERATURE_PROPERTIES_H_ -#define TEMPERATURE_PROPERTIES_H_ +#ifndef WARPX_TEMPERATURE_PROPERTIES_H_ +#define WARPX_TEMPERATURE_PROPERTIES_H_ #include #include @@ -43,4 +43,4 @@ struct TemperatureProperties std::unique_ptr m_ptr_temperature_parser; }; -#endif +#endif //WARPX_TEMPERATURE_PROPERTIES_H_ diff --git a/Source/Initialization/VelocityProperties.H b/Source/Initialization/VelocityProperties.H index 453d97725e2..87730acbc3a 100644 --- a/Source/Initialization/VelocityProperties.H +++ b/Source/Initialization/VelocityProperties.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef VELOCITY_PROPERTIES_H_ -#define VELOCITY_PROPERTIES_H_ +#ifndef WARPX_VELOCITY_PROPERTIES_H_ +#define WARPX_VELOCITY_PROPERTIES_H_ #include #include @@ -50,4 +50,4 @@ struct VelocityProperties std::unique_ptr m_ptr_velocity_parser; }; -#endif +#endif //WARPX_VELOCITY_PROPERTIES_H_ diff --git a/Source/Initialization/WarpXAMReXInit.H b/Source/Initialization/WarpXAMReXInit.H index 613250b5c2f..ab65fa2687c 100644 --- a/Source/Initialization/WarpXAMReXInit.H +++ b/Source/Initialization/WarpXAMReXInit.H @@ -33,4 +33,4 @@ namespace warpx::initialization ); } -#endif +#endif //WARPX_AMREX_INIT_H_ diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 5c530441fc9..341db01bef6 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef GUARDCELLMANAGER_H_ -#define GUARDCELLMANAGER_H_ +#ifndef WARPX_GUARDCELLMANAGER_H_ +#define WARPX_GUARDCELLMANAGER_H_ #include #include @@ -108,4 +108,4 @@ public: amrex::IntVect ng_depos_rho = amrex::IntVect::TheZeroVector(); }; -#endif // GUARDCELLMANAGER_H_ +#endif // WARPX_GUARDCELLMANAGER_H_ diff --git a/Source/Particles/Algorithms/KineticEnergy.H b/Source/Particles/Algorithms/KineticEnergy.H index 87da83f9d8d..f097b6436e1 100644 --- a/Source/Particles/Algorithms/KineticEnergy.H +++ b/Source/Particles/Algorithms/KineticEnergy.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLES_KINETIC_ENERGY_H_ -#define PARTICLES_KINETIC_ENERGY_H_ +#ifndef WARPX_PARTICLES_KINETIC_ENERGY_H_ +#define WARPX_PARTICLES_KINETIC_ENERGY_H_ #include "Utils/WarpXConst.H" @@ -68,4 +68,4 @@ namespace Algorithms{ } -#endif // PARTICLES_ALGORITHMS_H_ +#endif // WARPX_PARTICLES_KINETIC_ENERGY_H_ diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index fdc88a275a0..091bd41955f 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef BINARY_COLLISION_UTILS_H_ -#define BINARY_COLLISION_UTILS_H_ +#ifndef WARPX_BINARY_COLLISION_UTILS_H_ +#define WARPX_BINARY_COLLISION_UTILS_H_ #include @@ -128,4 +128,4 @@ namespace BinaryCollisionUtils{ } } -#endif // BINARY_COLLISION_UTILS_H_ +#endif // WARPX_BINARY_COLLISION_UTILS_H_ diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index 79fe1d42b7d..26782c2e42a 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PAIRWISE_COULOMB_COLLISION_FUNC_H_ -#define PAIRWISE_COULOMB_COLLISION_FUNC_H_ +#ifndef WARPX_PAIRWISE_COULOMB_COLLISION_FUNC_H_ +#define WARPX_PAIRWISE_COULOMB_COLLISION_FUNC_H_ #include "ElasticCollisionPerez.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -117,4 +117,4 @@ private: Executor m_exe; }; -#endif // PAIRWISE_COULOMB_COLLISION_FUNC_H_ +#endif // WARPX_PAIRWISE_COULOMB_COLLISION_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H index bc11e5e4a99..46b228b049e 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/CollisionFilterFunc.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef COLLISION_FILTER_FUNC_H_ -#define COLLISION_FILTER_FUNC_H_ +#ifndef WARPX_COLLISION_FILTER_FUNC_H_ +#define WARPX_COLLISION_FILTER_FUNC_H_ #include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Particles/Collision/ScatteringProcess.H" @@ -109,4 +109,4 @@ void CollisionPairFilter (const amrex::ParticleReal u1x, const amrex::ParticleRe } } -#endif // COLLISION_FILTER_FUNC_H_ +#endif // WARPX_COLLISION_FILTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H index 3c3849580d1..f28f3fb984c 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -7,8 +7,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef DSMC_FUNC_H_ -#define DSMC_FUNC_H_ +#ifndef WARPX_DSMC_FUNC_H_ +#define WARPX_DSMC_FUNC_H_ #include "CollisionFilterFunc.H" @@ -201,4 +201,4 @@ private: Executor m_exe; }; -#endif // DSMC_FUNC_H_ +#endif // WARPX_DSMC_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index a44ab90c4cc..86a2c70374e 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -6,8 +6,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SPLIT_AND_SCATTER_FUNC_H_ -#define SPLIT_AND_SCATTER_FUNC_H_ +#ifndef WARPX_SPLIT_AND_SCATTER_FUNC_H_ +#define WARPX_SPLIT_AND_SCATTER_FUNC_H_ #include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Particles/Collision/ScatteringProcess.H" @@ -291,4 +291,4 @@ private: amrex::Gpu::HostVector m_num_products_host; CollisionType m_collision_type; }; -#endif // SPLIT_AND_SCATTER_FUNC_H_ +#endif // WARPX_SPLIT_AND_SCATTER_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H index 718d386d5ba..e408600480e 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/BoschHaleFusionCrossSection.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef BOSCH_HALE_FUSION_CROSS_SECTION_H -#define BOSCH_HALE_FUSION_CROSS_SECTION_H +#ifndef WARPX_BOSCH_HALE_FUSION_CROSS_SECTION_H +#define WARPX_BOSCH_HALE_FUSION_CROSS_SECTION_H #include "Particles/Collision/BinaryCollision/BinaryCollisionUtils.H" #include "Utils/WarpXConst.H" @@ -110,4 +110,4 @@ amrex::ParticleReal BoschHaleFusionCrossSection ( return millibarn_to_sqm * astrophysical_factor/E_keV * std::exp(-B_G/std::sqrt(E_keV)); } -#endif // BOSCH_HALE_FUSION_CROSS_SECTION_H +#endif // WARPX_BOSCH_HALE_FUSION_CROSS_SECTION_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index a7dbc67b862..b6d6fe171de 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef NUCLEAR_FUSION_FUNC_H_ -#define NUCLEAR_FUSION_FUNC_H_ +#ifndef WARPX_NUCLEAR_FUSION_FUNC_H_ +#define WARPX_NUCLEAR_FUSION_FUNC_H_ #include "SingleNuclearFusionEvent.H" @@ -255,4 +255,4 @@ private: Executor m_exe; }; -#endif // NUCLEAR_FUSION_FUNC_H_ +#endif // WARPX_NUCLEAR_FUSION_FUNC_H_ diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H index ec9a1989189..a5549f30b6f 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PROTON_BORON_FUSION_CROSS_SECTION_H -#define PROTON_BORON_FUSION_CROSS_SECTION_H +#ifndef WARPX_PROTON_BORON_FUSION_CROSS_SECTION_H +#define WARPX_PROTON_BORON_FUSION_CROSS_SECTION_H #include "Utils/WarpXConst.H" @@ -154,4 +154,4 @@ amrex::ParticleReal ProtonBoronFusionCrossSection (const amrex::ParticleReal& E_ return cross_section_b*barn_to_sqm; } -#endif // PROTON_BORON_FUSION_CROSS_SECTION_H +#endif // WARPX_PROTON_BORON_FUSION_CROSS_SECTION_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H index 0b51d6b4b61..375cc1e6d51 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionInitializeMomentum.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H -#define PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H +#ifndef WARPX_PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H +#define WARPX_PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H #include "TwoProductFusionUtil.H" #include "Particles/WarpXParticleContainer.H" @@ -177,4 +177,4 @@ namespace { } -#endif // PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H +#endif // WARPX_PROTON_BORON_FUSION_INITIALIZE_MOMENTUM_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H index 07e0174438b..9bd4441eaa2 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/SingleNuclearFusionEvent.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SINGLE_NUCLEAR_FUSION_EVENT_H_ -#define SINGLE_NUCLEAR_FUSION_EVENT_H_ +#ifndef WARPX_SINGLE_NUCLEAR_FUSION_EVENT_H_ +#define WARPX_SINGLE_NUCLEAR_FUSION_EVENT_H_ #include "BoschHaleFusionCrossSection.H" #include "ProtonBoronFusionCrossSection.H" @@ -131,4 +131,4 @@ void SingleNuclearFusionEvent (const amrex::ParticleReal& u1x, const amrex::Part } } -#endif // SINGLE_NUCLEAR_FUSION_EVENT_H_ +#endif // WARPX_SINGLE_NUCLEAR_FUSION_EVENT_H_ diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H index 52e9db8aa94..6ca87b9390d 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionInitializeMomentum.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H -#define TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H +#ifndef WARPX_TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H +#define WARPX_TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H #include "TwoProductFusionUtil.H" #include "Particles/WarpXParticleContainer.H" @@ -96,4 +96,4 @@ namespace { } } -#endif // TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H +#endif // WARPX_TWO_PRODUCT_FUSION_INITIALIZE_MOMENTUM_H diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H index 09afe1e13a2..dbdef531a16 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/TwoProductFusionUtil.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef TWO_PRODUCT_FUSION_UTIL_H -#define TWO_PRODUCT_FUSION_UTIL_H +#ifndef WARPX_TWO_PRODUCT_FUSION_UTIL_H +#define WARPX_TWO_PRODUCT_FUSION_UTIL_H #include "Utils/ParticleUtils.H" #include "Utils/WarpXConst.H" @@ -165,4 +165,4 @@ namespace { } } -#endif // TWO_PRODUCT_FUSION_UTIL_H +#endif // WARPX_TWO_PRODUCT_FUSION_UTIL_H diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index 83312017e44..bf31076d975 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLE_CREATION_FUNC_H_ -#define PARTICLE_CREATION_FUNC_H_ +#ifndef WARPX_PARTICLE_CREATION_FUNC_H_ +#define WARPX_PARTICLE_CREATION_FUNC_H_ #include "BinaryCollisionUtils.H" @@ -338,4 +338,4 @@ public: } }; -#endif // PARTICLE_CREATION_FUNC_H_ +#endif // WARPX_PARTICLE_CREATION_FUNC_H_ diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index 8ff712eefa2..2912b769c57 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef CHARGEDEPOSITION_H_ -#define CHARGEDEPOSITION_H_ +#ifndef WARPX_CHARGEDEPOSITION_H_ +#define WARPX_CHARGEDEPOSITION_H_ #include "Particles/Deposition/SharedDepositionUtils.H" #include "ablastr/parallelization/KernelTimer.H" @@ -468,4 +468,4 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif } -#endif // CHARGEDEPOSITION_H_ +#endif // WARPX_CHARGEDEPOSITION_H_ diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index e722839c9c6..3dce7d28534 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef CURRENTDEPOSITION_H_ -#define CURRENTDEPOSITION_H_ +#ifndef WARPX_CURRENTDEPOSITION_H_ +#define WARPX_CURRENTDEPOSITION_H_ #include "Particles/Deposition/SharedDepositionUtils.H" #include "ablastr/parallelization/KernelTimer.H" @@ -2378,4 +2378,4 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, #endif // #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) } -#endif // CURRENTDEPOSITION_H_ +#endif // WARPX_CURRENTDEPOSITION_H_ diff --git a/Source/Particles/Deposition/SharedDepositionUtils.H b/Source/Particles/Deposition/SharedDepositionUtils.H index e28835a57df..1e2294be3a2 100644 --- a/Source/Particles/Deposition/SharedDepositionUtils.H +++ b/Source/Particles/Deposition/SharedDepositionUtils.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SHAREDDEPOSITIONUTILS_H_ -#define SHAREDDEPOSITIONUTILS_H_ +#ifndef WARPX_SHAREDDEPOSITIONUTILS_H_ +#define WARPX_SHAREDDEPOSITIONUTILS_H_ #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/ShapeFactors.H" @@ -281,4 +281,4 @@ void depositComponent (const GetParticlePosition& GetPosition, } #endif -#endif // SHAREDDEPOSITIONUTILS_H_ +#endif // WARPX_SHAREDDEPOSITIONUTILS_H_ diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index 61aeb19aea1..cee7ec07eb5 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef IONIZATION_H_ -#define IONIZATION_H_ +#ifndef WARPX_IONIZATION_H_ +#define WARPX_IONIZATION_H_ #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" @@ -171,4 +171,4 @@ struct IonizationTransformFunc } }; -#endif +#endif //WARPX_IONIZATION_H_ diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index fb723f0b79a..d550c5f03ee 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef QED_PAIR_GENERATION_H_ -#define QED_PAIR_GENERATION_H_ +#ifndef WARPX_QED_PAIR_GENERATION_H_ +#define WARPX_QED_PAIR_GENERATION_H_ #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" @@ -208,4 +208,4 @@ private: amrex::Dim3 m_lo; }; -#endif //QED_PAIR_GENERATION_H_ +#endif //WARPX_QED_PAIR_GENERATION_H_ diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index 567b260d0e4..dd5f57a8f24 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef QED_PHOTON_EMISSION_H_ -#define QED_PHOTON_EMISSION_H_ +#ifndef WARPX_QED_PHOTON_EMISSION_H_ +#define WARPX_QED_PHOTON_EMISSION_H_ #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" @@ -267,4 +267,4 @@ void cleanLowEnergyPhotons( } -#endif //QED_PHOTON_EMISSION_H_ +#endif //WARPX_QED_PHOTON_EMISSION_H_ diff --git a/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H b/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H index bf4c566b3ba..32b58dc50dc 100644 --- a/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H +++ b/Source/Particles/ElementaryProcess/QEDSchwingerProcess.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef QED_SCHWINGER_PROCESS_H_ -#define QED_SCHWINGER_PROCESS_H_ +#ifndef WARPX_QED_SCHWINGER_PROCESS_H_ +#define WARPX_QED_SCHWINGER_PROCESS_H_ #include "Particles/ElementaryProcess/QEDInternals/SchwingerProcessWrapper.H" #include "Utils/TextMsg.H" @@ -95,4 +95,4 @@ struct SchwingerTransformFunc } }; -#endif // QED_SCHWINGER_PROCESS_H_ +#endif // WARPX_QED_SCHWINGER_PROCESS_H_ diff --git a/Source/Particles/Filter/FilterFunctors.H b/Source/Particles/Filter/FilterFunctors.H index d1206eb022c..982eeb0d23a 100644 --- a/Source/Particles/Filter/FilterFunctors.H +++ b/Source/Particles/Filter/FilterFunctors.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef FILTERFUNCTORS_H -#define FILTERFUNCTORS_H +#ifndef WARPX_FILTERFUNCTORS_H +#define WARPX_FILTERFUNCTORS_H #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/WarpXParticleContainer.H" @@ -176,4 +176,4 @@ private: const amrex::RealBox m_domain; }; -#endif // FILTERFUNCTORS_H +#endif // WARPX_FILTERFUNCTORS_H diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index e51cfa41c58..b5bd4376ba1 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef FIELDGATHER_H_ -#define FIELDGATHER_H_ +#ifndef WARPX_FIELDGATHER_H_ +#define WARPX_FIELDGATHER_H_ #include "Particles/Gather/GetExternalFields.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -1852,4 +1852,4 @@ void doGatherShapeNImplicit ( } } -#endif // FIELDGATHER_H_ +#endif // WARPX_FIELDGATHER_H_ diff --git a/Source/Particles/Gather/ScaleFields.H b/Source/Particles/Gather/ScaleFields.H index 4cddfa5a342..5731bc047f4 100644 --- a/Source/Particles/Gather/ScaleFields.H +++ b/Source/Particles/Gather/ScaleFields.H @@ -60,4 +60,4 @@ struct ScaleFields } }; -#endif +#endif //WARPX_PARTICLES_GATHER_SCALEFIELDS_H_ diff --git a/Source/Particles/NamedComponentParticleContainer.H b/Source/Particles/NamedComponentParticleContainer.H index 983d417d9c0..02f4c44314a 100644 --- a/Source/Particles/NamedComponentParticleContainer.H +++ b/Source/Particles/NamedComponentParticleContainer.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef NamedComponentParticleContainer_H_ -#define NamedComponentParticleContainer_H_ +#ifndef WARPX_NamedComponentParticleContainer_H_ +#define WARPX_NamedComponentParticleContainer_H_ #include "Utils/TextMsg.H" @@ -219,4 +219,4 @@ protected: std::map particle_runtime_icomps; }; -#endif +#endif //WARPX_NamedComponentParticleContainer_H_ diff --git a/Source/Particles/ParticleBoundaries.H b/Source/Particles/ParticleBoundaries.H index c056436770d..a1b31f61cd2 100644 --- a/Source/Particles/ParticleBoundaries.H +++ b/Source/Particles/ParticleBoundaries.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARIES_H_ -#define PARTICLEBOUNDARIES_H_ +#ifndef WARPX_PARTICLEBOUNDARIES_H_ +#define WARPX_PARTICLEBOUNDARIES_H_ #include "Utils/WarpXAlgorithmSelection.H" @@ -70,4 +70,4 @@ struct ParticleBoundaries ParticleBoundariesData data; }; -#endif /*PARTICLEBOUNDARIES_H_*/ +#endif /*WARPX_PARTICLEBOUNDARIES_H_*/ diff --git a/Source/Particles/ParticleBoundaries_K.H b/Source/Particles/ParticleBoundaries_K.H index 1481dca7684..71b95aae95a 100644 --- a/Source/Particles/ParticleBoundaries_K.H +++ b/Source/Particles/ParticleBoundaries_K.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARIES_K_H_ -#define PARTICLEBOUNDARIES_K_H_ +#ifndef WARPX_PARTICLEBOUNDARIES_K_H_ +#define WARPX_PARTICLEBOUNDARIES_K_H_ #include "ParticleBoundaries.H" #include "Initialization/SampleGaussianFluxDistribution.H" @@ -186,4 +186,4 @@ namespace ApplyParticleBoundaries { } } -#endif +#endif //WARPX_PARTICLEBOUNDARIES_K_H_ diff --git a/Source/Particles/ParticleBoundaryBuffer.H b/Source/Particles/ParticleBoundaryBuffer.H index 895a3810852..d33834309ab 100644 --- a/Source/Particles/ParticleBoundaryBuffer.H +++ b/Source/Particles/ParticleBoundaryBuffer.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEBOUNDARYBUFFER_H_ -#define PARTICLEBOUNDARYBUFFER_H_ +#ifndef WARPX_PARTICLEBOUNDARYBUFFER_H_ +#define WARPX_PARTICLEBOUNDARYBUFFER_H_ #include "Particles/MultiParticleContainer_fwd.H" #include "Particles/WarpXParticleContainer.H" @@ -82,4 +82,4 @@ private: mutable std::vector m_species_names; }; -#endif /*PARTICLEBOUNDARYBUFFER_H_*/ +#endif /*WARPX_PARTICLEBOUNDARYBUFFER_H_*/ diff --git a/Source/Particles/ParticleCreation/DefaultInitialization.H b/Source/Particles/ParticleCreation/DefaultInitialization.H index 3f0cbab79d5..88b23905481 100644 --- a/Source/Particles/ParticleCreation/DefaultInitialization.H +++ b/Source/Particles/ParticleCreation/DefaultInitialization.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef DEFAULTINITIALIZATION_H_ -#define DEFAULTINITIALIZATION_H_ +#ifndef WARPX_DEFAULTINITIALIZATION_H_ +#define WARPX_DEFAULTINITIALIZATION_H_ #include #ifdef WARPX_QED @@ -280,4 +280,4 @@ void DefaultInitializeRuntimeAttributes (PTile& ptile, } -#endif +#endif //WARPX_DEFAULTINITIALIZATION_H_ diff --git a/Source/Particles/ParticleCreation/FilterCopyTransform.H b/Source/Particles/ParticleCreation/FilterCopyTransform.H index 095ebbf6a85..4815a98ca31 100644 --- a/Source/Particles/ParticleCreation/FilterCopyTransform.H +++ b/Source/Particles/ParticleCreation/FilterCopyTransform.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef FILTER_COPY_TRANSFORM_H_ -#define FILTER_COPY_TRANSFORM_H_ +#ifndef WARPX_FILTER_COPY_TRANSFORM_H_ +#define WARPX_FILTER_COPY_TRANSFORM_H_ #include "Particles/ParticleCreation/DefaultInitialization.H" @@ -360,4 +360,4 @@ Index filterCopyTransformParticles (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi std::forward(transform)); } -#endif +#endif //WARPX_FILTER_COPY_TRANSFORM_H_ diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 2a4c9fccad0..1fe44bdfa7f 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef FILTER_CREATE_TRANSFORM_FROM_FAB_H_ -#define FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#ifndef WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#define WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ #include "Particles/ParticleCreation/DefaultInitialization.H" #include "WarpX.H" @@ -244,4 +244,4 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi std::forward(transform)); } -#endif // FILTER_CREATE_TRANSFORM_FROM_FAB_H_ +#endif // WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ diff --git a/Source/Particles/ParticleCreation/SmartCopy.H b/Source/Particles/ParticleCreation/SmartCopy.H index 6a6ceb3d290..e1d944e9c30 100644 --- a/Source/Particles/ParticleCreation/SmartCopy.H +++ b/Source/Particles/ParticleCreation/SmartCopy.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SMART_COPY_H_ -#define SMART_COPY_H_ +#ifndef WARPX_SMART_COPY_H_ +#define WARPX_SMART_COPY_H_ #include "DefaultInitialization.H" #include "SmartUtils.H" @@ -163,4 +163,4 @@ public: [[nodiscard]] bool isDefined () const noexcept { return m_defined; } }; -#endif +#endif //WARPX_SMART_COPY_H_ diff --git a/Source/Particles/ParticleCreation/SmartCreate.H b/Source/Particles/ParticleCreation/SmartCreate.H index b4f25d5daad..fe4cb5929e0 100644 --- a/Source/Particles/ParticleCreation/SmartCreate.H +++ b/Source/Particles/ParticleCreation/SmartCreate.H @@ -5,8 +5,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SMART_CREATE_H_ -#define SMART_CREATE_H_ +#ifndef WARPX_SMART_CREATE_H_ +#define WARPX_SMART_CREATE_H_ #include "DefaultInitialization.H" @@ -112,4 +112,4 @@ public: [[nodiscard]] bool isDefined () const noexcept { return m_defined; } }; -#endif //SMART_CREATE_H_ +#endif //WARPX_SMART_CREATE_H_ diff --git a/Source/Particles/ParticleCreation/SmartUtils.H b/Source/Particles/ParticleCreation/SmartUtils.H index dbac563ca28..652a3aecd17 100644 --- a/Source/Particles/ParticleCreation/SmartUtils.H +++ b/Source/Particles/ParticleCreation/SmartUtils.H @@ -6,8 +6,8 @@ * License: BSD-3-Clause-LBNL */ -#ifndef SMART_UTILS_H_ -#define SMART_UTILS_H_ +#ifndef WARPX_SMART_UTILS_H_ +#define WARPX_SMART_UTILS_H_ #include "DefaultInitialization.H" @@ -70,4 +70,4 @@ void setNewParticleIDs (PTile& ptile, amrex::Long old_size, amrex::Long num_adde }); } -#endif //SMART_UTILS_H_ +#endif //WARPX_SMART_UTILS_H_ diff --git a/Source/Particles/ParticleIO.H b/Source/Particles/ParticleIO.H index 022ab471362..d5fc68f4097 100644 --- a/Source/Particles/ParticleIO.H +++ b/Source/Particles/ParticleIO.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef PARTICLEIO_H_ -#define PARTICLEIO_H_ +#ifndef WARPX_PARTICLEIO_H_ +#define WARPX_PARTICLEIO_H_ #include "Particles/WarpXParticleContainer.H" #include "Particles/Pusher/GetAndSetPosition.H" @@ -92,4 +92,4 @@ void storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, int electrostatic_solver_id, bool is_full_diagnostic ); -#endif /* PARTICLEIO_H_ */ +#endif /* WARPX_PARTICLEIO_H_ */ diff --git a/Source/Particles/PinnedMemoryParticleContainer.H b/Source/Particles/PinnedMemoryParticleContainer.H index eb8fe835c73..402c621eb9a 100644 --- a/Source/Particles/PinnedMemoryParticleContainer.H +++ b/Source/Particles/PinnedMemoryParticleContainer.H @@ -1,8 +1,8 @@ -#ifndef PinnedMemoryParticleContainer_H_ -#define PinnedMemoryParticleContainer_H_ +#ifndef WARPX_PinnedMemoryParticleContainer_H_ +#define WARPX_PinnedMemoryParticleContainer_H_ #include "NamedComponentParticleContainer.H" using PinnedMemoryParticleContainer = NamedComponentParticleContainer; -#endif +#endif //WARPX_PinnedMemoryParticleContainer_H_ diff --git a/Source/Particles/ShapeFactors.H b/Source/Particles/ShapeFactors.H index 7c56da457ed..e711cce15ff 100644 --- a/Source/Particles/ShapeFactors.H +++ b/Source/Particles/ShapeFactors.H @@ -4,8 +4,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef SHAPEFACTORS_H_ -#define SHAPEFACTORS_H_ +#ifndef WARPX_SHAPEFACTORS_H_ +#define WARPX_SHAPEFACTORS_H_ #include "Utils/TextMsg.H" @@ -243,4 +243,4 @@ struct Compute_shape_factor_pair } }; -#endif // SHAPEFACTORS_H_ +#endif // WARPX_SHAPEFACTORS_H_ diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 9768b36cb84..c34516aedc5 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -5,8 +5,8 @@ * * License: BSD-3-Clause-LBNL */ -#ifndef UTILS_WARPXALGORITHMSELECTION_H_ -#define UTILS_WARPXALGORITHMSELECTION_H_ +#ifndef WARPX_UTILS_WARPXALGORITHMSELECTION_H_ +#define WARPX_UTILS_WARPXALGORITHMSELECTION_H_ #include @@ -207,4 +207,4 @@ GetParticleBCTypeInteger( std::string BCType ); std::string GetFieldBCTypeString( FieldBoundaryType fb_type ); -#endif // UTILS_WARPXALGORITHMSELECTION_H_ +#endif // WARPX_UTILS_WARPXALGORITHMSELECTION_H_ From ec175e55fa06fbc7134f551b9042a42d90020094 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Tue, 23 Apr 2024 22:17:26 -0700 Subject: [PATCH 065/190] Implement grid based merging algorithm (#4820) * initial implementation of the grid based merging routine * add compiler directives for different dimensions * avoid implicit capture of `this` in lambda * code clean-up * use `PIdx::x` for r-coordinate while 4667 is still being reviewed * fix clang-tidy errors * another clang-tidy fix * improve doc-strings * use iterative heap sort since "SYCL kernel cannot call a recursive function" * fix clang-tidy error * fix sorting bug; add documentation and CI test * add dummy CI benchmark file to get proper values * update benchmark values based on Azure results * rename `GridBasedMerging` -> `VelocityCoincidenceThinning` * use `Algorithms::KineticEnergy` * reorganize merging loop * update CI benchmark values * Relativistic correction in product particles' velocity calculation * update benchmark values after changing energy calculation * handle edge case with zero cluster momentum * call redistribute after particle resampling to remove invalid particles * use unsigned ints for indexing * Revert "use unsigned ints for indexing" This reverts commit abe027f3402cf0e7c51c4599a0a50b9e8a6b817a. * call `Redistribute` before merging * code clean-up * also check for `std::isnan` in edge case handling * add reference for grid based merging * check that cluster has total weight > 0 before merging * add defense against numerical error leading to nan * make resampling message more verbose * use `deleteInvalidParticles` instead of `Redistribute` * remove default values for merging parameters * remove doc-string default specifications in picmi.py * apply suggestions from code review * update benchmark values; avoid possible nans * add assert to prevent merging of photons * add `BackwardCompatibility` check to `LevelingThinning` --- Docs/source/refs.bib | 11 + Docs/source/usage/parameters.rst | 28 +- .../inputs_1d_velocity_coincidence_thinning | 46 +++ .../Tests/resampling/inputs_leveling_thinning | 2 +- Python/pywarpx/picmi.py | 31 +- ...esample_velocity_coincidence_thinning.json | 20 ++ Regression/WarpX-tests.ini | 17 ++ .../Particles/PhysicalParticleContainer.cpp | 14 +- Source/Particles/Resampling/CMakeLists.txt | 1 + .../Particles/Resampling/LevelingThinning.H | 6 + .../Particles/Resampling/LevelingThinning.cpp | 14 +- Source/Particles/Resampling/Make.package | 1 + Source/Particles/Resampling/Resampling.cpp | 5 + .../Resampling/VelocityCoincidenceThinning.H | 115 ++++++++ .../VelocityCoincidenceThinning.cpp | 278 ++++++++++++++++++ 15 files changed, 574 insertions(+), 15 deletions(-) create mode 100644 Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning create mode 100644 Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json create mode 100644 Source/Particles/Resampling/VelocityCoincidenceThinning.H create mode 100644 Source/Particles/Resampling/VelocityCoincidenceThinning.cpp diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 2de27908c5d..84b34b17351 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -430,3 +430,14 @@ @book{Wiedemann2015 doi = {10.1007/978-3-319-18317-6}, year = {2015} } + +@article{Vranic2015, + title = {Particle merging algorithm for PIC codes}, + author = {M. Vranic and T. Grismayer and J.L. Martins and R.A. Fonseca and L.O. Silva}, + journal = {Computer Physics Communications}, + volume = {191}, + pages = {65-73}, + year = {2015}, + issn = {0010-4655}, + doi = {https://doi.org/10.1016/j.cpc.2015.01.020}, +} diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 0a089358d21..9f88f2d8c6c 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1269,25 +1269,39 @@ Particle initialization * ``.do_resampling`` (`0` or `1`) optional (default `0`) If `1` resampling is performed for this species. This means that the number of macroparticles will be reduced at specific timesteps while preserving the distribution function as much as - possible (in particular the weight of the remaining particles will be increased on average). + possible (details depend on the chosen resampling algorithm). This can be useful in situations with continuous creation of particles (e.g. with ionization or with QED effects). At least one resampling trigger (see below) must be specified to actually perform resampling. * ``.resampling_algorithm`` (`string`) optional (default `leveling_thinning`) - The algorithm used for resampling. Currently there is only one option, which is already set by - default: + The algorithm used for resampling: * ``leveling_thinning`` This algorithm is defined in :cite:t:`param-MuravievCPC2021`. - It has two parameters: + It has one parameter: * ``.resampling_algorithm_target_ratio`` (`float`) optional (default `1.5`) This **roughly** corresponds to the ratio between the number of particles before and after resampling. - * ``.resampling_algorithm_min_ppc`` (`int`) optional (default `1`) - Resampling is not performed in cells with a number of macroparticles strictly smaller - than this parameter. + * ``velocity_coincidence_thinning``` The particles are sorted into phase space + cells and merged, similar to the approach described in :cite:t:`param-Vranic2015`. + It has three parameters: + + * ``.resampling_algorithm_delta_ur`` (`float`) + The width of momentum cells used in clustering particles, in m/s. + + * ``.resampling_algorithm_n_theta`` (`int`) + The number of cell divisions to use in the :math:`\theta` direction + when clustering the particle velocities. + + * ``.resampling_algorithm_n_phi`` (`int`) + The number of cell divisions to use in the :math:`\phi` direction + when clustering the particle velocities. + +* ``.resampling_min_ppc`` (`int`) optional (default `1`) + Resampling is not performed in cells with a number of macroparticles strictly smaller + than this parameter. * ``.resampling_trigger_intervals`` (`string`) optional (default `0`) Using the `Intervals parser`_ syntax, this string defines timesteps at which resampling is diff --git a/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning new file mode 100644 index 00000000000..1610c8b1221 --- /dev/null +++ b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning @@ -0,0 +1,46 @@ +max_step = 4 +warpx.verbose = 1 +warpx.const_dt = 1e-10 +amr.n_cell = 256 +amr.max_grid_size = 64 +amr.max_level = 0 +geometry.dims = 1 +geometry.prob_lo = 0 +geometry.prob_hi = 0.1 + +# Boundary condition and field solver +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic +algo.particle_shape = 1 +algo.maxwell_solver = none + +particles.species_names = hydrogen +hydrogen.mass = 1.67262192369e-27 +hydrogen.charge = 1.602176634e-19 +hydrogen.injection_style = nrandompercell +hydrogen.initialize_self_fields = 0 +hydrogen.do_not_push = 1 +hydrogen.do_resampling = 1 +hydrogen.resampling_min_ppc = 10 +hydrogen.resampling_trigger_intervals = 1::2 +hydrogen.resampling_algorithm = velocity_coincidence_thinning +hydrogen.resampling_algorithm_delta_ur = 100000000.0 +hydrogen.resampling_algorithm_n_phi = 3 +hydrogen.resampling_algorithm_n_theta = 120 +hydrogen.num_particles_per_cell = 2500 +hydrogen.momentum_distribution_type = gaussian +hydrogen.ux_m = 0.0 +hydrogen.uy_m = 0.0 +hydrogen.uz_m = 0.0 +hydrogen.ux_th = 0.000326 +hydrogen.uy_th = 0.000326 +hydrogen.uz_th = 0.000326 +hydrogen.profile = constant +hydrogen.density = 1e+19 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 4 +diag1.diag_type = Full diff --git a/Examples/Tests/resampling/inputs_leveling_thinning b/Examples/Tests/resampling/inputs_leveling_thinning index 9ed49ae358e..90e0d6802d6 100644 --- a/Examples/Tests/resampling/inputs_leveling_thinning +++ b/Examples/Tests/resampling/inputs_leveling_thinning @@ -57,7 +57,7 @@ resampled_part2.do_resampling = 1 resampled_part2.resampling_algorithm = leveling_thinning resampled_part2.resampling_algorithm_target_ratio = 1.3 # This should prevent actual resampling at timestep 7 -resampled_part2.resampling_algorithm_min_ppc = 80000 +resampled_part2.resampling_min_ppc = 80000 # This should trigger resampling at timestep 6 and 7 in this case. The rest is here to test the # intervals parser syntax. resampled_part2.resampling_trigger_intervals = 100 :120:3, 6:6:6 , 7 diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 4bf9d4b4163..2859d6fd468 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -132,12 +132,31 @@ class Species(picmistandard.PICMI_Species): warpx_do_resampling: bool, default=False Whether particles will be resampled + warpx_resampling_min_ppc: int, default=1 + Cells with fewer particles than this number will be + skipped during resampling. + warpx_resampling_trigger_intervals: bool, default=0 Timesteps at which to resample warpx_resampling_trigger_max_avg_ppc: int, default=infinity Resampling will be done when the average number of particles per cell exceeds this number + + warpx_resampling_algorithm: str, default="leveling_thinning" + Resampling algorithm to use. + + warpx_resampling_algorithm_delta_ur: float + Size of velocity window used for clustering particles during grid-based + merging. + + warpx_resampling_algorithm_n_theta: int + Number of bins to use in theta when clustering particle velocities + during grid-based merging. + + warpx_resampling_algorithm_n_phi: int + Number of bins to use in phi when clustering particle velocities + during grid-based merging. """ def init(self, kw): @@ -215,8 +234,13 @@ def init(self, kw): # Resampling settings self.do_resampling = kw.pop('warpx_do_resampling', None) + self.resampling_algorithm = kw.pop('warpx_resampling_algorithm', None) + self.resampling_min_ppc = kw.pop('warpx_resampling_min_ppc', None) self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) + self.resampling_algorithm_delta_ur = kw.pop('warpx_resampling_algorithm_delta_ur', None) + self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) + self.resampling_algorithm_n_phi = kw.pop('warpx_resampling_algorithm_n_phi', None) def species_initialize_inputs(self, layout, initialize_self_fields = False, @@ -255,8 +279,13 @@ def species_initialize_inputs(self, layout, do_not_gather = self.do_not_gather, random_theta = self.random_theta, do_resampling=self.do_resampling, + resampling_algorithm=self.resampling_algorithm, + resampling_min_ppc=self.resampling_min_ppc, resampling_trigger_intervals=self.resampling_trigger_intervals, - resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc) + resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc, + resampling_algorithm_delta_ur=self.resampling_algorithm_delta_ur, + resampling_algorithm_n_theta=self.resampling_algorithm_n_theta, + resampling_algorithm_n_phi=self.resampling_algorithm_n_phi) # add reflection models self.species.add_new_attr("reflection_model_xlo(E)", self.reflection_model_xlo) diff --git a/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json new file mode 100644 index 00000000000..1f6f076f021 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 447073.7680171198, + "jy": 336566.4197455162, + "jz": 642997.1912520501 + }, + "hydrogen": { + "particle_momentum_x": 1.7562033925112653e-17, + "particle_momentum_y": 1.756681695333869e-17, + "particle_momentum_z": 2.0662018455459364e-17, + "particle_position_x": 6600.315303519964, + "particle_weight": 1.0000000000000003e+18 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 1a157d7e022..1f656960789 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4189,6 +4189,23 @@ compileTest = 0 doVis = 0 analysisRoutine = Examples/Tests/repelling_particles/analysis_repelling.py +[resample_velocity_coincidence_thinning] +buildDir = . +inputFile = Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning +runtime_params = +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/analysis_default_regression.py + [restart] buildDir = . inputFile = Examples/Tests/restart/inputs diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index f8afb1d39af..c7f44903d9b 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -3362,11 +3362,7 @@ void PhysicalParticleContainer::resample (const int timestep, const bool verbose WARPX_PROFILE_VAR_START(blp_resample_actual); if (m_resampler.triggered(timestep, global_numparts)) { - if (verbose) { - amrex::Print() << Utils::TextMsg::Info( - "Resampling " + species_name + " at step " + std::to_string(timestep) - ); - } + Redistribute(); for (int lev = 0; lev <= maxLevel(); lev++) { for (WarpXParIter pti(*this, lev); pti.isValid(); ++pti) @@ -3374,6 +3370,14 @@ void PhysicalParticleContainer::resample (const int timestep, const bool verbose m_resampler(pti, lev, this); } } + deleteInvalidParticles(); + if (verbose) { + amrex::Print() << Utils::TextMsg::Info( + "Resampled " + species_name + " at step " + std::to_string(timestep) + + ": macroparticle count decreased by " + + std::to_string(static_cast(global_numparts - TotalNumberOfParticles())) + ); + } } WARPX_PROFILE_VAR_STOP(blp_resample_actual); } diff --git a/Source/Particles/Resampling/CMakeLists.txt b/Source/Particles/Resampling/CMakeLists.txt index 13eb8a8c4b2..8fca0176b86 100644 --- a/Source/Particles/Resampling/CMakeLists.txt +++ b/Source/Particles/Resampling/CMakeLists.txt @@ -5,5 +5,6 @@ foreach(D IN LISTS WarpX_DIMS) Resampling.cpp ResamplingTrigger.cpp LevelingThinning.cpp + VelocityCoincidenceThinning.cpp ) endforeach() diff --git a/Source/Particles/Resampling/LevelingThinning.H b/Source/Particles/Resampling/LevelingThinning.H index 9e3e63fc8be..88e8aa79b4b 100644 --- a/Source/Particles/Resampling/LevelingThinning.H +++ b/Source/Particles/Resampling/LevelingThinning.H @@ -38,6 +38,12 @@ public: */ LevelingThinning (const std::string& species_name); + /** + * This function queries deprecated input parameters and aborts + * the run if one of them is specified. + */ + void BackwardCompatibility (const std::string& species_name ); + /** * \brief A method that performs leveling thinning for the considered species. * diff --git a/Source/Particles/Resampling/LevelingThinning.cpp b/Source/Particles/Resampling/LevelingThinning.cpp index 71ebe735c78..09531d0e47d 100644 --- a/Source/Particles/Resampling/LevelingThinning.cpp +++ b/Source/Particles/Resampling/LevelingThinning.cpp @@ -47,11 +47,23 @@ LevelingThinning::LevelingThinning (const std::string& species_name) } utils::parser::queryWithParser( - pp_species_name, "resampling_algorithm_min_ppc", m_min_ppc); + pp_species_name, "resampling_min_ppc", m_min_ppc); + BackwardCompatibility(species_name); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_min_ppc >= 1, "Resampling min_ppc should be greater than or equal to 1"); } +void LevelingThinning::BackwardCompatibility (const std::string& species_name ) +{ + const amrex::ParmParse pp_species_name(species_name); + int backward_min_ppc; + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !pp_species_name.query("resampling_algorithm_min_ppc", backward_min_ppc), + ".resampling_algorithm_min_ppc is no longer a valid option. " + "Please use the renamed option .resampling_min_ppc instead." + ); +} + void LevelingThinning::operator() (WarpXParIter& pti, const int lev, WarpXParticleContainer * const pc) const { diff --git a/Source/Particles/Resampling/Make.package b/Source/Particles/Resampling/Make.package index e70e3446c33..12b58dd9d47 100644 --- a/Source/Particles/Resampling/Make.package +++ b/Source/Particles/Resampling/Make.package @@ -1,5 +1,6 @@ CEXE_sources += Resampling.cpp CEXE_sources += ResamplingTrigger.cpp CEXE_sources += LevelingThinning.cpp +CEXE_sources += VelocityCoincidenceThinning.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Particles/Resampling/ diff --git a/Source/Particles/Resampling/Resampling.cpp b/Source/Particles/Resampling/Resampling.cpp index 11f8706dc33..15aba88dd3e 100644 --- a/Source/Particles/Resampling/Resampling.cpp +++ b/Source/Particles/Resampling/Resampling.cpp @@ -6,6 +6,7 @@ */ #include "Resampling.H" +#include "VelocityCoincidenceThinning.H" #include "LevelingThinning.H" #include "Utils/TextMsg.H" @@ -22,6 +23,10 @@ Resampling::Resampling (const std::string& species_name) { m_resampling_algorithm = std::make_unique(species_name); } + else if (resampling_algorithm_string == "velocity_coincidence_thinning") + { + m_resampling_algorithm = std::make_unique(species_name); + } else { WARPX_ABORT_WITH_MESSAGE("Unknown resampling algorithm."); } diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.H b/Source/Particles/Resampling/VelocityCoincidenceThinning.H new file mode 100644 index 00000000000..9462f4b35ed --- /dev/null +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.H @@ -0,0 +1,115 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_VELOCITY_COINCIDENCE_THINNING_H_ +#define WARPX_VELOCITY_COINCIDENCE_THINNING_H_ + +#include "Resampling.H" + + +/** + * \brief This class implements a particle merging scheme wherein particles + * are clustered in phase space and particles in the same cluster is merged + * into two remaining particles. The scheme conserves linear momentum and + * kinetic energy within each cluster. + */ +class VelocityCoincidenceThinning: public ResamplingAlgorithm { +public: + + /** + * \brief Default constructor of the VelocityCoincidenceThinning class. + */ + VelocityCoincidenceThinning () = default; + + /** + * \brief Constructor of the VelocityCoincidenceThinning class + * + * @param[in] species_name the name of the resampled species + */ + VelocityCoincidenceThinning (const std::string& species_name); + + /** + * \brief A method that performs merging for the considered species. + * + * @param[in] pti WarpX particle iterator of the particles to resample. + * @param[in] lev the index of the refinement level. + * @param[in] pc a pointer to the particle container. + */ + void operator() (WarpXParIter& pti, int lev, WarpXParticleContainer* pc) const final; + + /** + * \brief This merging routine requires functionality to sort a GPU vector + * based on another GPU vector's values. The heap-sort functions below were + * obtained from https://www.geeksforgeeks.org/iterative-heap-sort/ and + * modified for the current purpose. It achieves the same as + * ``` + * std::sort( + * sorted_indices_data + cell_start, sorted_indices_data + cell_stop, + * [&momentum_bin_number_data](size_t i1, size_t i2) { + * return momentum_bin_number_data[i1] < momentum_bin_number_data[i2]; + * } + * ); + * ``` + * but with support for device execution. + */ + struct HeapSort { + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void swap(int &a, int &b) const + { + const auto temp = b; + b = a; + a = temp; + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void operator() (int index_array[], const int bin_array[], const int start, const int n) const + { + // sort index_array into a max heap structure + for (int i = 1; i < n; i++) + { + auto j = i; + // move child through heap if it is bigger than its parent + while (j > 0 && bin_array[index_array[j+start]] > bin_array[index_array[(j - 1)/2 + start]]) { + // swap child and parent until branch is properly ordered + swap(index_array[j+start], index_array[(j - 1)/2 + start]); + j = (j - 1) / 2; + } + } + + for (int i = n - 1; i > 0; i--) + { + // swap value of first (now the largest value) to the new end point + swap(index_array[start], index_array[i+start]); + + // remake the max heap + int j = 0, index; + while (j < i) { + index = 2 * j + 1; + + // if left child is smaller than right child, point index variable to right child + if (index + 1 < i && bin_array[index_array[index+start]] < bin_array[index_array[index+1+start]]) { + index++; + } + // if parent is smaller than child, swap parent with child having higher value + if (index < i && bin_array[index_array[j+start]] < bin_array[index_array[index+start]]) { + swap(index_array[j+start], index_array[index+start]); + } + j = index; + } + } + } + }; + +private: + int m_min_ppc = 1; + int m_ntheta; + int m_nphi; + amrex::ParticleReal m_delta_ur; +}; +#endif // WARPX_VELOCITY_COINCIDENCE_THINNING_H_ diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp new file mode 100644 index 00000000000..66176c48b4d --- /dev/null +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp @@ -0,0 +1,278 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Roelof Groenewald (TAE Technologies) + * + * License: BSD-3-Clause-LBNL + */ + +#include "VelocityCoincidenceThinning.H" + +#include "Particles/Algorithms/KineticEnergy.H" +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" + + +VelocityCoincidenceThinning::VelocityCoincidenceThinning (const std::string& species_name) +{ + using namespace amrex::literals; + + const amrex::ParmParse pp_species_name(species_name); + + utils::parser::queryWithParser( + pp_species_name, "resampling_min_ppc", m_min_ppc + ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_min_ppc >= 1, + "Resampling min_ppc should be greater than or equal to 1" + ); + + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_delta_ur", m_delta_ur + ); + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_n_theta", m_ntheta + ); + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_n_phi", m_nphi + ); +} + +void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, + WarpXParticleContainer * const pc) const +{ + using namespace amrex::literals; + + auto& ptile = pc->ParticlesAt(lev, pti); + auto& soa = ptile.GetStructOfArrays(); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) + auto * const AMREX_RESTRICT x = soa.GetRealData(PIdx::x).data(); +#elif defined(WARPX_DIM_RZ) + auto * const AMREX_RESTRICT x = soa.GetRealData(PIdx::x).data(); // rename to PIdx::r after PR #4667 +#endif +#if defined(WARPX_DIM_3D) + auto * const AMREX_RESTRICT y = soa.GetRealData(PIdx::y).data(); +#endif + auto * const AMREX_RESTRICT z = soa.GetRealData(PIdx::z).data(); + auto * const AMREX_RESTRICT ux = soa.GetRealData(PIdx::ux).data(); + auto * const AMREX_RESTRICT uy = soa.GetRealData(PIdx::uy).data(); + auto * const AMREX_RESTRICT uz = soa.GetRealData(PIdx::uz).data(); + auto * const AMREX_RESTRICT w = soa.GetRealData(PIdx::w).data(); + auto * const AMREX_RESTRICT idcpu = soa.GetIdCPUData().data(); + + // Using this function means that we must loop over the cells in the ParallelFor. + auto bins = ParticleUtils::findParticlesInEachCell(lev, pti, ptile); + + const auto n_cells = static_cast(bins.numBins()); + auto *const indices = bins.permutationPtr(); + auto *const cell_offsets = bins.offsetsPtr(); + + const auto min_ppc = m_min_ppc; + + const auto mass = pc->getMass(); + + // check if species mass > 0 + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + mass > 0, + "VelocityCoincidenceThinning does not yet work for massless particles." + ); + + // create a GPU vector to hold the momentum cluster index for each particle + amrex::Gpu::DeviceVector momentum_bin_number(bins.numItems()); + auto* momentum_bin_number_data = momentum_bin_number.data(); + + // create a GPU vector to hold the index sorting for the momentum bins + amrex::Gpu::DeviceVector sorted_indices(bins.numItems()); + auto* sorted_indices_data = sorted_indices.data(); + + const auto Ntheta = m_ntheta; + const auto Nphi = m_nphi; + + const auto dr = m_delta_ur; + const auto dtheta = 2.0_prt * MathConst::pi / Ntheta; + const auto dphi = MathConst::pi / Nphi; + constexpr auto c2 = PhysConst::c * PhysConst::c; + + auto heapSort = HeapSort(); + + // Loop over cells + amrex::ParallelForRNG( n_cells, + [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept + { + // The particles that are in the cell `i_cell` are + // given by the `indices[cell_start:cell_stop]` + const auto cell_start = static_cast(cell_offsets[i_cell]); + const auto cell_stop = static_cast(cell_offsets[i_cell+1]); + const auto cell_numparts = cell_stop - cell_start; + + // do nothing for cells with less particles than min_ppc + // (this intentionally includes skipping empty cells, too) + if (cell_numparts < min_ppc) { + return; + } + + // Loop over particles and label them with the appropriate momentum bin + // number. Also assign initial ordering to the sorted_indices array. + for (int i = cell_start; i < cell_stop; ++i) + { + // get polar components of the velocity vector + auto u_mag = std::sqrt( + ux[indices[i]]*ux[indices[i]] + + uy[indices[i]]*uy[indices[i]] + + uz[indices[i]]*uz[indices[i]] + ); + auto u_theta = std::atan2(uy[indices[i]], ux[indices[i]]) + MathConst::pi; + auto u_phi = std::acos(uz[indices[i]]/u_mag); + + const auto ii = static_cast(u_theta / dtheta); + const auto jj = static_cast(u_phi / dphi); + const auto kk = static_cast(u_mag / dr); + + momentum_bin_number_data[i] = ii + jj * Ntheta + kk * Ntheta * Nphi; + sorted_indices_data[i] = i; + } + + // sort indices based on comparing values in momentum_bin_number + heapSort(sorted_indices_data, momentum_bin_number_data, cell_start, cell_numparts); + + // initialize variables used to hold cluster totals + int particles_in_bin = 0; + amrex::ParticleReal total_weight = 0._prt, total_energy = 0._prt; +#if !defined(WARPX_DIM_1D_Z) + amrex::ParticleReal cluster_x = 0._prt; +#endif +#if defined(WARPX_DIM_3D) + amrex::ParticleReal cluster_y = 0._prt; +#endif + amrex::ParticleReal cluster_z = 0._prt; + amrex::ParticleReal cluster_ux = 0._prt, cluster_uy = 0._prt, cluster_uz = 0._prt; + + // Finally, loop through the particles in the cell and merge + // ones in the same momentum bin + for (int i = cell_start; i < cell_stop; ++i) + { + particles_in_bin += 1; + const auto part_idx = indices[sorted_indices_data[i]]; + +#if !defined(WARPX_DIM_1D_Z) + cluster_x += w[part_idx]*x[part_idx]; +#endif +#if defined(WARPX_DIM_3D) + cluster_y += w[part_idx]*y[part_idx]; +#endif + cluster_z += w[part_idx]*z[part_idx]; + cluster_ux += w[part_idx]*ux[part_idx]; + cluster_uy += w[part_idx]*uy[part_idx]; + cluster_uz += w[part_idx]*uz[part_idx]; + total_weight += w[part_idx]; + total_energy += w[part_idx] * Algorithms::KineticEnergy( + ux[part_idx], uy[part_idx], uz[part_idx], mass + ); + + // check if this is the last particle in the current momentum bin + if ( + (i == cell_stop - 1) + || (momentum_bin_number_data[sorted_indices_data[i]] != momentum_bin_number_data[sorted_indices_data[i + 1]]) + ) { + // check if the bin has more than 2 particles in it + if ( particles_in_bin > 2 && total_weight > std::numeric_limits::min() ){ + // get average quantities for the bin +#if !defined(WARPX_DIM_1D_Z) + cluster_x /= total_weight; +#endif +#if defined(WARPX_DIM_3D) + cluster_y /= total_weight; +#endif + cluster_z /= total_weight; + cluster_ux /= total_weight; + cluster_uy /= total_weight; + cluster_uz /= total_weight; + + // perform merging of momentum bin particles + auto u_perp2 = cluster_ux*cluster_ux + cluster_uy*cluster_uy; + auto u_perp = std::sqrt(u_perp2); + auto cluster_u_mag2 = u_perp2 + cluster_uz*cluster_uz; + auto cluster_u_mag = std::sqrt(cluster_u_mag2); + + // calculate required velocity magnitude to achieve + // energy conservation + auto v_mag2 = total_energy / total_weight * ( + (total_energy / total_weight + 2._prt * mass * c2 ) + / (mass * mass * c2) + ); + auto v_perp = (v_mag2 > cluster_u_mag2) ? std::sqrt(v_mag2 - cluster_u_mag2) : 0_prt; + + // choose random angle for new velocity vector + auto phi = amrex::Random(engine) * MathConst::pi; + + // set new velocity components based on chosen phi + auto vx = v_perp * std::cos(phi); + auto vy = v_perp * std::sin(phi); + + // calculate rotation angles to parallel coord. frame + auto cos_theta = (cluster_u_mag > 0._prt) ? cluster_uz / cluster_u_mag : 0._prt; + auto sin_theta = (cluster_u_mag > 0._prt) ? u_perp / cluster_u_mag : 0._prt; + auto cos_phi = (u_perp > 0._prt) ? cluster_ux / u_perp : 0._prt; + auto sin_phi = (u_perp > 0._prt) ? cluster_uy / u_perp : 0._prt; + + // rotate new velocity vector to labframe + auto ux_new = ( + vx * cos_theta * cos_phi - vy * sin_phi + + cluster_u_mag * sin_theta * cos_phi + ); + auto uy_new = ( + vx * cos_theta * sin_phi + vy * cos_phi + + cluster_u_mag * sin_theta * sin_phi + ); + auto uz_new = -vx * sin_theta + cluster_u_mag * cos_theta; + + // set the last two particles' attributes according to + // the bin's aggregate values + const auto part_idx2 = indices[sorted_indices_data[i - 1]]; + + w[part_idx] = total_weight / 2._prt; + w[part_idx2] = total_weight / 2._prt; +#if !defined(WARPX_DIM_1D_Z) + x[part_idx] = cluster_x; + x[part_idx2] = cluster_x; +#endif +#if defined(WARPX_DIM_3D) + y[part_idx] = cluster_y; + y[part_idx2] = cluster_y; +#endif + z[part_idx] = cluster_z; + z[part_idx2] = cluster_z; + + ux[part_idx] = ux_new; + uy[part_idx] = uy_new; + uz[part_idx] = uz_new; + ux[part_idx2] = 2._prt * cluster_ux - ux_new; + uy[part_idx2] = 2._prt * cluster_uy - uy_new; + uz[part_idx2] = 2._prt * cluster_uz - uz_new; + + // set ids of merged particles so they will be removed + for (int j = 2; j < particles_in_bin; ++j){ + idcpu[indices[sorted_indices_data[i - j]]] = amrex::ParticleIdCpus::Invalid; + } + } + + // restart the tallies + particles_in_bin = 0; + total_weight = 0._prt; + total_energy = 0._prt; +#if !defined(WARPX_DIM_1D_Z) + cluster_x = 0_prt; +#endif +#if defined(WARPX_DIM_3D) + cluster_y = 0_prt; +#endif + cluster_z = 0_prt; + cluster_ux = 0_prt; + cluster_uy = 0_prt; + cluster_uz = 0_prt; + } + } + } + ); +} From 486e7939f0dc59b6a81278523093d05519973630 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 24 Apr 2024 08:23:26 +0200 Subject: [PATCH 066/190] Remove friend class PML from WarpX class (#4874) --- Source/BoundaryConditions/PML.H | 4 +-- Source/BoundaryConditions/PML.cpp | 7 +++-- Source/Initialization/WarpXInitData.cpp | 42 +++++++++++++------------ Source/WarpX.H | 4 +-- Source/WarpX.cpp | 2 -- 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 835877f2de9..42a96f3628a 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -136,8 +136,8 @@ private: class PML { public: - PML (int lev, - const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, + PML (int lev, const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm, bool do_similar_dm_pml, const amrex::Geometry* geom, const amrex::Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, amrex::Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 0b2806b3151..09088a24ada 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -542,7 +542,8 @@ MultiSigmaBox::ComputePMLFactorsE (const Real* dx, Real dt) } } -PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& grid_dm, +PML::PML (const int lev, const BoxArray& grid_ba, + const DistributionMapping& grid_dm, const bool do_similar_dm_pml, const Geometry* geom, const Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, @@ -658,7 +659,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri } DistributionMapping dm; - if (WarpX::do_similar_dm_pml) { + if (do_similar_dm_pml) { auto ng_sim = amrex::elemwiseMax(amrex::elemwiseMax(nge, ngb), ngf); dm = amrex::MakeSimilarDM(ba, grid_ba, grid_dm, ng_sim); } else { @@ -806,7 +807,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& gri const BoxArray& cba = MakeBoxArray(is_single_box_domain, cdomain, *cgeom, grid_cba_reduced, cncells, do_pml_in_domain, do_pml_Lo, do_pml_Hi); DistributionMapping cdm; - if (WarpX::do_similar_dm_pml) { + if (do_similar_dm_pml) { auto ng_sim = amrex::elemwiseMax(amrex::elemwiseMax(nge, ngb), ngf); cdm = amrex::MakeSimilarDM(cba, grid_cba_reduced, grid_dm, ng_sim); } else { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 5134a3aaa10..9dff4023f6d 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -587,16 +587,17 @@ WarpX::InitPML () // Note: fill_guards_fields and fill_guards_current are both set to // zero (amrex::IntVect(0)) (what we do with damping BCs does not apply // to the PML, for example in the presence of mesh refinement patches) - pml[0] = std::make_unique(0, boxArray(0), DistributionMap(0), &Geom(0), nullptr, - pml_ncell, pml_delta, amrex::IntVect::TheZeroVector(), - dt[0], nox_fft, noy_fft, noz_fft, grid_type, - do_moving_window, pml_has_particles, do_pml_in_domain, - psatd_solution_type, J_in_time, rho_in_time, - do_pml_dive_cleaning, do_pml_divb_cleaning, - amrex::IntVect(0), amrex::IntVect(0), - guard_cells.ng_FieldSolver.max(), - v_particle_pml, - do_pml_Lo[0], do_pml_Hi[0]); + pml[0] = std::make_unique( + 0, boxArray(0), DistributionMap(0), do_similar_dm_pml, &Geom(0), nullptr, + pml_ncell, pml_delta, amrex::IntVect::TheZeroVector(), + dt[0], nox_fft, noy_fft, noz_fft, grid_type, + do_moving_window, pml_has_particles, do_pml_in_domain, + psatd_solution_type, J_in_time, rho_in_time, + do_pml_dive_cleaning, do_pml_divb_cleaning, + amrex::IntVect(0), amrex::IntVect(0), + guard_cells.ng_FieldSolver.max(), + v_particle_pml, + do_pml_Lo[0], do_pml_Hi[0]); #endif for (int lev = 1; lev <= finest_level; ++lev) @@ -625,16 +626,17 @@ WarpX::InitPML () // Note: fill_guards_fields and fill_guards_current are both set to // zero (amrex::IntVect(0)) (what we do with damping BCs does not apply // to the PML, for example in the presence of mesh refinement patches) - pml[lev] = std::make_unique(lev, boxArray(lev), DistributionMap(lev), - &Geom(lev), &Geom(lev-1), - pml_ncell, pml_delta, refRatio(lev-1), - dt[lev], nox_fft, noy_fft, noz_fft, grid_type, - do_moving_window, pml_has_particles, do_pml_in_domain, - psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, - amrex::IntVect(0), amrex::IntVect(0), - guard_cells.ng_FieldSolver.max(), - v_particle_pml, - do_pml_Lo[lev], do_pml_Hi[lev]); + pml[lev] = std::make_unique( + lev, boxArray(lev), DistributionMap(lev), do_similar_dm_pml, + &Geom(lev), &Geom(lev-1), + pml_ncell, pml_delta, refRatio(lev-1), + dt[lev], nox_fft, noy_fft, noz_fft, grid_type, + do_moving_window, pml_has_particles, do_pml_in_domain, + psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning, + amrex::IntVect(0), amrex::IntVect(0), + guard_cells.ng_FieldSolver.max(), + v_particle_pml, + do_pml_Lo[lev], do_pml_Hi[lev]); } } } diff --git a/Source/WarpX.H b/Source/WarpX.H index 9ca754ea1bb..7d2b8e45907 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -107,8 +107,6 @@ class WARPX_EXPORT WarpX { public: - friend class PML; - static WarpX& GetInstance (); static void ResetInstance (); @@ -1613,7 +1611,7 @@ private: int pml_has_particles = 0; int do_pml_j_damping = 0; int do_pml_in_domain = 0; - static int do_similar_dm_pml; + bool do_similar_dm_pml = true; bool do_pml_dive_cleaning; // default set in WarpX.cpp bool do_pml_divb_cleaning; // default set in WarpX.cpp amrex::Vector do_pml_Lo; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 9a9903f013f..4b693d6cf99 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -212,8 +212,6 @@ int WarpX::n_current_deposition_buffer = -1; short WarpX::grid_type; amrex::IntVect m_rho_nodal_flag; -int WarpX::do_similar_dm_pml = 1; - #ifdef AMREX_USE_GPU bool WarpX::do_device_synchronize = true; #else From 206e6d54a6969379103722be619eed3661ee33df Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 24 Apr 2024 18:23:29 +0200 Subject: [PATCH 067/190] move field type struct in a separate header (#4875) --- Source/BoundaryConditions/PML.cpp | 2 + Source/BoundaryConditions/PML_RZ.cpp | 2 + .../WarpXFieldBoundaries.cpp | 4 +- Source/Diagnostics/BTDiagnostics.cpp | 2 + .../ComputeDiagFunctors/JFunctor.cpp | 3 ++ .../ComputeDiagFunctors/JdispFunctor.cpp | 2 + .../FlushFormats/FlushFormatCheckpoint.cpp | 2 + .../FlushFormats/FlushFormatPlotfile.cpp | 6 ++- Source/Diagnostics/FullDiagnostics.cpp | 3 ++ Source/Diagnostics/ParticleIO.cpp | 2 + .../Diagnostics/ReducedDiags/ChargeOnEB.cpp | 2 + .../ReducedDiags/ColliderRelevant.cpp | 2 + .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 2 + .../Diagnostics/ReducedDiags/FieldMaximum.cpp | 2 + .../ReducedDiags/FieldMomentum.cpp | 2 + .../Diagnostics/ReducedDiags/FieldProbe.cpp | 2 + .../Diagnostics/ReducedDiags/FieldReduction.H | 20 +++++----- .../ReducedDiags/LoadBalanceCosts.cpp | 2 + .../ReducedDiags/ParticleExtrema.cpp | 2 + Source/Diagnostics/SliceDiagnostic.cpp | 5 ++- Source/FieldSolver/ElectrostaticSolver.cpp | 1 + Source/FieldSolver/Fields.H | 40 +++++++++++++++++++ .../HybridPICModel/HybridPICModel.cpp | 3 ++ .../MacroscopicProperties.cpp | 2 + Source/Particles/MultiParticleContainer.cpp | 3 ++ Source/WarpX.H | 38 +++--------------- Source/WarpX.cpp | 1 + 27 files changed, 110 insertions(+), 47 deletions(-) create mode 100644 Source/FieldSolver/Fields.H diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 09088a24ada..25b34818dd1 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -10,6 +10,7 @@ #include "BoundaryConditions/PML.H" #include "BoundaryConditions/PMLComponent.H" +#include "FieldSolver/Fields.H" #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralFieldData.H" #endif @@ -55,6 +56,7 @@ #endif using namespace amrex; +using namespace warpx::fields; namespace { diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index 5eb30f43cca..faa7609b3ab 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -8,6 +8,7 @@ #include "PML_RZ.H" #include "BoundaryConditions/PML_RZ.H" +#include "FieldSolver/Fields.H" #ifdef WARPX_USE_PSATD # include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" #endif @@ -33,6 +34,7 @@ #include using namespace amrex; +using namespace warpx::fields; PML_RZ::PML_RZ (const int lev, const amrex::BoxArray& grid_ba, const amrex::DistributionMapping& grid_dm, const amrex::Geometry* geom, const int ncell, const int do_pml_in_domain) diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index 0e50c560e7c..a5e0004325a 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -15,8 +15,10 @@ #include #include #include -using namespace amrex::literals; + using namespace amrex; +using namespace amrex::literals; +using namespace warpx::fields; namespace { diff --git a/Source/Diagnostics/BTDiagnostics.cpp b/Source/Diagnostics/BTDiagnostics.cpp index 40a76eac946..1cee9909226 100644 --- a/Source/Diagnostics/BTDiagnostics.cpp +++ b/Source/Diagnostics/BTDiagnostics.cpp @@ -14,6 +14,7 @@ #include "Diagnostics/Diagnostics.H" #include "Diagnostics/FlushFormats/FlushFormat.H" #include "ComputeDiagFunctors/BackTransformParticleFunctor.H" +#include "FieldSolver/Fields.H" #include "Utils/Algorithms/IsIn.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" @@ -46,6 +47,7 @@ #include using namespace amrex::literals; +using namespace warpx::fields; namespace { diff --git a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp index a8fd142815a..ebaec47b2f1 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JFunctor.cpp @@ -6,6 +6,7 @@ #include "JFunctor.H" +#include "FieldSolver/Fields.H" #include "Particles/MultiParticleContainer.H" #include "WarpX.H" @@ -14,6 +15,8 @@ #include #include +using namespace warpx::fields; + JFunctor::JFunctor (const int dir, int lev, amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, diff --git a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp index b73931814b6..aac5869da65 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/JdispFunctor.cpp @@ -6,6 +6,7 @@ #include "JdispFunctor.H" #include "WarpX.H" +#include "FieldSolver/Fields.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Particles/MultiParticleContainer.H" @@ -15,6 +16,7 @@ #include using namespace amrex; +using namespace warpx::fields; JdispFunctor::JdispFunctor (int dir, int lev, amrex::IntVect crse_ratio, bool convertRZmodes2cartesian, int ncomp) diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 653ac0c26ba..27d0ad84b9c 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -5,6 +5,7 @@ # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/ParticleDiag/ParticleDiag.H" +#include "FieldSolver/Fields.H" #include "Particles/WarpXParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" @@ -19,6 +20,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp index 7b10edf0d2b..94b9901b06e 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp @@ -1,16 +1,17 @@ #include "FlushFormatPlotfile.H" -#include "Particles/ParticleIO.H" +#include "FieldSolver/Fields.H" +#include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" #include "Particles/Filter/FilterFunctors.H" #include "Particles/WarpXParticleContainer.H" +#include "Particles/ParticleIO.H" #include "Particles/PinnedMemoryParticleContainer.H" #include "Utils/Interpolate.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" #include "WarpX.H" -#include "Diagnostics/MultiDiagnostics.H" #include #include @@ -47,6 +48,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index 63683690483..b25f899e29d 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -1,4 +1,5 @@ #include "FullDiagnostics.H" + #include "ComputeDiagFunctors/CellCenterFunctor.H" #include "ComputeDiagFunctors/DivBFunctor.H" #include "ComputeDiagFunctors/DivEFunctor.H" @@ -10,6 +11,7 @@ #include "ComputeDiagFunctors/RhoFunctor.H" #include "Diagnostics/Diagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" +#include "FieldSolver/Fields.H" #include "FlushFormats/FlushFormat.H" #include "Particles/MultiParticleContainer.H" #include "Utils/Algorithms/IsIn.H" @@ -40,6 +42,7 @@ #include using namespace amrex::literals; +using namespace warpx::fields; FullDiagnostics::FullDiagnostics (int i, const std::string& name): Diagnostics{i, name}, diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index 5cb18602ed9..c4164e383b1 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -7,6 +7,7 @@ * License: BSD-3-Clause-LBNL */ +#include "FieldSolver/Fields.H" #include "Particles/ParticleIO.H" #include "Particles/MultiParticleContainer.H" #include "Particles/PhysicalParticleContainer.H" @@ -41,6 +42,7 @@ #include using namespace amrex; +using namespace warpx::fields; void LaserParticleContainer::ReadHeader (std::istream& is) diff --git a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp index 7e66061fa48..2991831420e 100644 --- a/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp +++ b/Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp @@ -8,6 +8,7 @@ #include "ChargeOnEB.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/Parser/ParserUtils.H" @@ -26,6 +27,7 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor ChargeOnEB::ChargeOnEB (const std::string& rd_name) diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 88dd0c0922b..434ab1d7949 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -8,6 +8,7 @@ #include "ColliderRelevant.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "FieldSolver/Fields.H" #if (defined WARPX_QED) # include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" #endif @@ -58,6 +59,7 @@ #include using namespace amrex; +using namespace warpx::fields; ColliderRelevant::ColliderRelevant (const std::string& rd_name) : ReducedDiags{rd_name} diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index 65b33341d1a..f4b4e2a39a1 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -7,6 +7,7 @@ #include "FieldEnergy.H" +#include "FieldSolver/Fields.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" @@ -28,6 +29,7 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor FieldEnergy::FieldEnergy (const std::string& rd_name) diff --git a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp index 80979cd845e..24d9dda3ea6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMaximum.cpp @@ -7,6 +7,7 @@ #include "FieldMaximum.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" #include "WarpX.H" @@ -38,6 +39,7 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor FieldMaximum::FieldMaximum (const std::string& rd_name) diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index 901748bd6a9..f182acd5ba2 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -7,6 +7,7 @@ #include "FieldMomentum.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "WarpX.H" @@ -37,6 +38,7 @@ #include using namespace amrex; +using namespace warpx::fields; FieldMomentum::FieldMomentum (const std::string& rd_name) : ReducedDiags{rd_name} diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 4be5a928d96..7364d5989f1 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -7,6 +7,7 @@ #include "FieldProbe.H" #include "FieldProbeParticleContainer.H" +#include "FieldSolver/Fields.H" #include "Particles/Gather/FieldGather.H" #include "Particles/Pusher/GetAndSetPosition.H" #include "Particles/Pusher/UpdatePosition.H" @@ -44,6 +45,7 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index a2813bf51e8..a15d6489e2c 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -9,6 +9,7 @@ #define WARPX_DIAGNOSTICS_REDUCEDDIAGS_FIELDREDUCTION_H_ #include "ReducedDiags.H" +#include "FieldSolver/Fields.H" #include "WarpX.H" #include @@ -36,7 +37,6 @@ #include #include - /** * This class contains a function that computes an arbitrary reduction of the fields. The function * used in the reduction is defined by an input file parser expression and the reduction operation @@ -99,15 +99,15 @@ public: const auto dx = geom.CellSizeArray(); // get MultiFab data - const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); - const amrex::MultiFab & jx = warpx.getField(FieldType::current_fp, lev,0); - const amrex::MultiFab & jy = warpx.getField(FieldType::current_fp, lev,1); - const amrex::MultiFab & jz = warpx.getField(FieldType::current_fp, lev,2); + const amrex::MultiFab & Ex = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(warpx::fields::FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(warpx::fields::FieldType::Bfield_aux, lev,2); + const amrex::MultiFab & jx = warpx.getField(warpx::fields::FieldType::current_fp, lev,0); + const amrex::MultiFab & jy = warpx.getField(warpx::fields::FieldType::current_fp, lev,1); + const amrex::MultiFab & jz = warpx.getField(warpx::fields::FieldType::current_fp, lev,2); // General preparation of interpolation and reduction operations diff --git a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp index e5e8a7adc07..882c94b12eb 100644 --- a/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp +++ b/Source/Diagnostics/ReducedDiags/LoadBalanceCosts.cpp @@ -7,6 +7,7 @@ #include "LoadBalanceCosts.H" #include "Diagnostics/ReducedDiags/ReducedDiags.H" +#include "FieldSolver/Fields.H" #include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -35,6 +36,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 79b66a92e66..0cc5429be7a 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -11,6 +11,7 @@ #if (defined WARPX_QED) # include "Particles/ElementaryProcess/QEDInternals/QedChiFunctions.H" #endif +#include "FieldSolver/Fields.H" #include "Particles/Gather/FieldGather.H" #include "Particles/Gather/GetExternalFields.H" #include "Particles/MultiParticleContainer.H" @@ -51,6 +52,7 @@ #include using namespace amrex; +using namespace warpx::fields; // constructor ParticleExtrema::ParticleExtrema (const std::string& rd_name) diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index aaadac52610..14a2d633a0c 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -7,8 +7,9 @@ */ #include "SliceDiagnostic.H" -#include "WarpX.H" +#include "FieldSolver/Fields.H" #include "Utils/TextMsg.H" +#include "WarpX.H" #include #include @@ -40,7 +41,7 @@ #include using namespace amrex; - +using namespace warpx::fields; /* \brief * The functions creates the slice for diagnostics based on the user-input. diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index d0818959b30..5bb50ada278 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -55,6 +55,7 @@ #include using namespace amrex; +using namespace warpx::fields; void WarpX::ComputeSpaceChargeField (bool const reset_fields) diff --git a/Source/FieldSolver/Fields.H b/Source/FieldSolver/Fields.H new file mode 100644 index 00000000000..76ff9a74f7b --- /dev/null +++ b/Source/FieldSolver/Fields.H @@ -0,0 +1,40 @@ +/* Copyright 2024 Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_FIELDS_H_ +#define WARPX_FIELDS_H_ + +namespace warpx::fields +{ + enum struct FieldType : int + { + Efield_aux, + Bfield_aux, + Efield_fp, + Bfield_fp, + current_fp, + current_fp_nodal, + rho_fp, + F_fp, + G_fp, + phi_fp, + vector_potential_fp, + Efield_cp, + Bfield_cp, + current_cp, + rho_cp, + F_cp, + G_cp, + edge_lengths, + face_areas, + Efield_avg_fp, + Bfield_avg_fp, + Efield_avg_cp, + Bfield_avg_cp + }; +} + +#endif //WARPX_FIELDS_H_ diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 6a688ce38b3..8979036fbea 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -9,7 +9,10 @@ #include "HybridPICModel.H" +#include "FieldSolver/Fields.H" + using namespace amrex; +using namespace warpx::fields; HybridPICModel::HybridPICModel ( int nlevs_max ) { diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index c60a0b4c4d5..d9a8ec71fe1 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -1,5 +1,6 @@ #include "MacroscopicProperties.H" +#include "FieldSolver/Fields.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" #include "WarpX.H" @@ -26,6 +27,7 @@ #include using namespace amrex; +using namespace warpx::fields; MacroscopicProperties::MacroscopicProperties () { diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index dda32467925..e4823bfaf6f 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -10,6 +10,8 @@ * License: BSD-3-Clause-LBNL */ #include "MultiParticleContainer.H" + +#include "FieldSolver/Fields.H" #include "Particles/ElementaryProcess/Ionization.H" #ifdef WARPX_QED # include "Particles/ElementaryProcess/QEDInternals/BreitWheelerEngineWrapper.H" @@ -80,6 +82,7 @@ #include using namespace amrex; +using namespace warpx::fields; namespace { diff --git a/Source/WarpX.H b/Source/WarpX.H index 7d2b8e45907..d19843ca636 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -38,6 +38,7 @@ #include "AcceleratorLattice/AcceleratorLattice.H" #include "Evolve/WarpXDtType.H" #include "Evolve/WarpXPushType.H" +#include "FieldSolver/Fields.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" #include "Filter/BilinearFilter.H" @@ -75,33 +76,6 @@ #include #include -enum struct FieldType : int -{ - Efield_aux, - Bfield_aux, - Efield_fp, - Bfield_fp, - current_fp, - current_fp_nodal, - rho_fp, - F_fp, - G_fp, - phi_fp, - vector_potential_fp, - Efield_cp, - Bfield_cp, - current_cp, - rho_cp, - F_cp, - G_cp, - edge_lengths, - face_areas, - Efield_avg_fp, - Bfield_avg_fp, - Efield_avg_cp, - Bfield_avg_cp -}; - class WARPX_EXPORT WarpX : public amrex::AmrCore { @@ -492,7 +466,7 @@ public: * \return true if the field is initialized, false otherwise */ [[nodiscard]] bool - isFieldInitialized (FieldType field_type, int lev, int direction = 0) const; + isFieldInitialized (warpx::fields::FieldType field_type, int lev, int direction = 0) const; /** * \brief @@ -505,7 +479,7 @@ public: * \return the pointer to an amrex::MultiFab containing the field data */ [[nodiscard]] amrex::MultiFab* - getFieldPointer (FieldType field_type, int lev, int direction = 0) const; + getFieldPointer (warpx::fields::FieldType field_type, int lev, int direction = 0) const; /** * \brief @@ -517,7 +491,7 @@ public: * \return an array of three pointers amrex::MultiFab* containing the field data */ [[nodiscard]] std::array - getFieldPointerArray (FieldType field_type, int lev) const; + getFieldPointerArray (warpx::fields::FieldType field_type, int lev) const; /** * \brief @@ -530,7 +504,7 @@ public: * \return a constant refernce to an amrex::MultiFab containing the field data */ [[nodiscard]] const amrex::MultiFab& - getField(FieldType field_type, int lev, int direction = 0) const; + getField(warpx::fields::FieldType field_type, int lev, int direction = 0) const; [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} @@ -1601,7 +1575,7 @@ private: * \return the pointer to an amrex::MultiFab containing the field data */ [[nodiscard]] amrex::MultiFab* - getFieldPointerUnchecked (FieldType field_type, int lev, int direction = 0) const; + getFieldPointerUnchecked (warpx::fields::FieldType field_type, int lev, int direction = 0) const; // PML int do_pml = 0; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 4b693d6cf99..9a29d7c2354 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -85,6 +85,7 @@ #include using namespace amrex; +using namespace warpx::fields; int WarpX::do_moving_window = 0; int WarpX::start_moving_window_step = 0; From 28434e957bf8104479a4774b5653e46e391b572f Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 24 Apr 2024 18:45:15 +0200 Subject: [PATCH 068/190] remove call to WarpX::GetInstance() from FilterCreateTransformFromFAB.H (#4873) --- Source/Particles/MultiParticleContainer.cpp | 4 +- .../FilterCreateTransformFromFAB.H | 38 +++++++++---------- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index e4823bfaf6f..fc496217388 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -1399,10 +1399,12 @@ MultiParticleContainer::doQEDSchwinger () const auto Transform = SchwingerTransformFunc{m_qed_schwinger_y_size, PIdx::w}; + const amrex::Geometry& geom_level_zero = warpx.Geom(level_0); + const auto num_added = filterCreateTransformFromFAB<1>( *pc_product_ele, *pc_product_pos, dst_ele_tile, dst_pos_tile, box, fieldsEB, np_ele_dst, np_pos_dst,Filter, CreateEle, CreatePos, - Transform); + Transform, geom_level_zero); setNewParticleIDs(dst_ele_tile, np_ele_dst, num_added); setNewParticleIDs(dst_pos_tile, np_pos_dst, num_added); diff --git a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H index 1fe44bdfa7f..8a83c60b221 100644 --- a/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H +++ b/Source/Particles/ParticleCreation/FilterCreateTransformFromFAB.H @@ -9,7 +9,6 @@ #define WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ #include "Particles/ParticleCreation/DefaultInitialization.H" -#include "WarpX.H" #include #include @@ -40,6 +39,7 @@ * \param[in] create1 callable that defines what will be done for the create step for dst1. * \param[in] create2 callable that defines what will be done for the create step for dst2. * \param[in] transform callable that defines the transformation to apply on dst1 and dst2. + * \param[in] geom_lev_zero the geometry object associated to level zero * * \return num_added the number of particles that were written to dst1 and dst2. */ @@ -51,34 +51,30 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, const FAB *src_FAB, const Index* mask, const Index dst1_index, const Index dst2_index, CreateFunc1&& create1, CreateFunc2&& create2, - TransFunc&& transform) noexcept + TransFunc&& transform, const amrex::Geometry& geom_lev_zero) noexcept { using namespace amrex; const auto ncells = box.volume(); if (ncells == 0) { return 0; } - auto & warpx = WarpX::GetInstance(); - const int level_0 = 0; - Geometry const & geom = warpx.Geom(level_0); - constexpr int spacedim = AMREX_SPACEDIM; #if defined(WARPX_DIM_1D_Z) - const Real zlo_global = geom.ProbLo(0); - const Real dz = geom.CellSize(0); + const Real zlo_global = geom_lev_zero.ProbLo(0); + const Real dz = geom_lev_zero.CellSize(0); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const Real xlo_global = geom.ProbLo(0); - const Real dx = geom.CellSize(0); - const Real zlo_global = geom.ProbLo(1); - const Real dz = geom.CellSize(1); + const Real xlo_global = geom_lev_zero.ProbLo(0); + const Real dx = geom_lev_zero.CellSize(0); + const Real zlo_global = geom_lev_zero.ProbLo(1); + const Real dz = geom_lev_zero.CellSize(1); #elif defined(WARPX_DIM_3D) - const Real xlo_global = geom.ProbLo(0); - const Real dx = geom.CellSize(0); - const Real ylo_global = geom.ProbLo(1); - const Real dy = geom.CellSize(1); - const Real zlo_global = geom.ProbLo(2); - const Real dz = geom.CellSize(2); + const Real xlo_global = geom_lev_zero.ProbLo(0); + const Real dx = geom_lev_zero.CellSize(0); + const Real ylo_global = geom_lev_zero.ProbLo(1); + const Real dy = geom_lev_zero.CellSize(1); + const Real zlo_global = geom_lev_zero.ProbLo(2); + const Real dz = geom_lev_zero.CellSize(2); #endif const auto arrNumPartCreation = src_FAB->array(); @@ -199,6 +195,7 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, * \param[in] create1 callable that defines what will be done for the create step for dst1. * \param[in] create2 callable that defines what will be done for the create step for dst2. * \param[in] transform callable that defines the transformation to apply on dst1 and dst2. + * \param[in] geom_lev_zero the geometry object associated to level zero * * \return num_added the number of particles that were written to dst1 and dst2. */ @@ -209,7 +206,7 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi const FABs& src_FABs, const Index dst1_index, const Index dst2_index, FilterFunc&& filter, CreateFunc1&& create1, CreateFunc2&& create2, - TransFunc && transform) noexcept + TransFunc && transform, const amrex::Geometry& geom_lev_zero) noexcept { using namespace amrex; @@ -241,7 +238,8 @@ Index filterCreateTransformFromFAB (DstPC& pc1, DstPC& pc2, DstTile& dst1, DstTi mask.dataPtr(), dst1_index, dst2_index, std::forward(create1), std::forward(create2), - std::forward(transform)); + std::forward(transform), + geom_lev_zero); } #endif // WARPX_FILTER_CREATE_TRANSFORM_FROM_FAB_H_ From 7e651179b2b30e01b2011c7d5de3aef86e21ee3a Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 24 Apr 2024 20:25:39 +0200 Subject: [PATCH 069/190] remove calls to WarpX:GetInstance() from MacroscopicProperties.cpp (#4872) --- .../MacroscopicProperties.H | 25 +++++++++++-- .../MacroscopicProperties.cpp | 35 ++++++++----------- Source/Initialization/WarpXInitData.cpp | 11 +++++- 3 files changed, 48 insertions(+), 23 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H index 205b009b2eb..129b2e2f782 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H @@ -13,8 +13,11 @@ #include "Utils/WarpXConst.H" #include +#include +#include #include #include +#include #include #include #include @@ -33,8 +36,26 @@ public: MacroscopicProperties (); // constructor /** Read user-defined macroscopic properties. Called in constructor. */ void ReadParameters (); - /** Initialize multifabs storing macroscopic multifabs */ - void InitData (); + + /** + * \brief Initialize multifabs storing macroscopic multifabs + * + * @param[in] ba the box array associated to the multifabs E and B + * @param[in] dmap the distribution mapping + * @param[in] ng_EB_alloc guard cells allocated for multifabs E and B + * @param[in] geom the geometry + * @param[in] Ex_stag staggering of the Ex field + * @param[in] Ey_stag staggering of the Ey field + * @param[in] Ez_stag staggering of the Ez field + */ + void InitData ( + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dmap, + const amrex::IntVect& ng_EB_alloc, + const amrex::Geometry& geom, + const amrex::IntVect& Ex_stag, + const amrex::IntVect& Ey_stag, + const amrex::IntVect& Ez_stag); /** return MultiFab, sigma (conductivity) of the medium. */ amrex::MultiFab& getsigma_mf () {return (*m_sigma_mf);} diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index d9a8ec71fe1..1467c7a8c0c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -3,18 +3,14 @@ #include "FieldSolver/Fields.H" #include "Utils/Parser/ParserUtils.H" #include "Utils/TextMsg.H" -#include "WarpX.H" #include #include -#include #include -#include #include #include #include -#include #include #include #include @@ -123,16 +119,17 @@ MacroscopicProperties::ReadParameters () } void -MacroscopicProperties::InitData () +MacroscopicProperties::InitData ( + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dmap, + const amrex::IntVect& ng_EB_alloc, + const amrex::Geometry& geom, + const amrex::IntVect& Ex_stag, + const amrex::IntVect& Ey_stag, + const amrex::IntVect& Ez_stag) { amrex::Print() << Utils::TextMsg::Info("we are in init data of macro"); - auto & warpx = WarpX::GetInstance(); - // Get BoxArray and DistributionMap of warpx instance. - const int lev = 0; - const amrex::BoxArray ba = warpx.boxArray(lev); - const amrex::DistributionMapping dmap = warpx.DistributionMap(lev); - const amrex::IntVect ng_EB_alloc = warpx.getngEB(); // Define material property multifabs using ba and dmap from WarpX instance // sigma is cell-centered MultiFab m_sigma_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); @@ -148,7 +145,7 @@ MacroscopicProperties::InitData () } else if (m_sigma_s == "parse_sigma_function") { InitializeMacroMultiFabUsingParser(m_sigma_mf.get(), m_sigma_parser->compile<3>(), - warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); + geom.CellSizeArray(), geom.ProbDomain()); } // Initialize epsilon if (m_epsilon_s == "constant") { @@ -158,7 +155,7 @@ MacroscopicProperties::InitData () } else if (m_epsilon_s == "parse_epsilon_function") { InitializeMacroMultiFabUsingParser(m_eps_mf.get(), m_epsilon_parser->compile<3>(), - warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); + geom.CellSizeArray(), geom.ProbDomain()); } // In the Maxwell solver, `epsilon` is used in the denominator. @@ -175,16 +172,14 @@ MacroscopicProperties::InitData () } else if (m_mu_s == "parse_mu_function") { InitializeMacroMultiFabUsingParser(m_mu_mf.get(), m_mu_parser->compile<3>(), - warpx.Geom(lev).CellSizeArray(), warpx.Geom(lev).ProbDomain()); + geom.CellSizeArray(), geom.ProbDomain()); } - amrex::IntVect sigma_stag = m_sigma_mf->ixType().toIntVect(); - amrex::IntVect epsilon_stag = m_eps_mf->ixType().toIntVect(); - amrex::IntVect mu_stag = m_mu_mf->ixType().toIntVect(); - amrex::IntVect Ex_stag = warpx.getField(FieldType::Efield_fp, 0,0).ixType().toIntVect(); - amrex::IntVect Ey_stag = warpx.getField(FieldType::Efield_fp, 0,1).ixType().toIntVect(); - amrex::IntVect Ez_stag = warpx.getField(FieldType::Efield_fp, 0,2).ixType().toIntVect(); + const amrex::IntVect sigma_stag = m_sigma_mf->ixType().toIntVect(); + const amrex::IntVect epsilon_stag = m_eps_mf->ixType().toIntVect(); + const amrex::IntVect mu_stag = m_mu_mf->ixType().toIntVect(); + for ( int idim = 0; idim < AMREX_SPACEDIM; ++idim) { sigma_IndexType[idim] = sigma_stag[idim]; diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 9dff4023f6d..47eeaa41f86 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -474,7 +474,16 @@ WarpX::InitData () BuildBufferMasks(); if (WarpX::em_solver_medium==1) { - m_macroscopic_properties->InitData(); + const int lev_zero = 0; + m_macroscopic_properties->InitData( + boxArray(lev_zero), + DistributionMap(lev_zero), + getngEB(), + Geom(lev_zero), + getField(FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), + getField(FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), + getField(FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + ); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { From 10f6f675a2e150fdabbbd6339f049ad61b975e82 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 24 Apr 2024 13:07:33 -0700 Subject: [PATCH 070/190] TNSA Example: Link Domain Decomposition (#4885) - link domain decomposition for `numprocs` option - remove outdated warning --- .../usage/workflows/domain_decomposition.rst | 6 ++++++ .../Physics_applications/laser_ion/README.rst | 21 +++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/Docs/source/usage/workflows/domain_decomposition.rst b/Docs/source/usage/workflows/domain_decomposition.rst index 1340ecc10d9..201f2dcbbc5 100644 --- a/Docs/source/usage/workflows/domain_decomposition.rst +++ b/Docs/source/usage/workflows/domain_decomposition.rst @@ -5,6 +5,9 @@ Domain Decomposition WarpX relies on a spatial domain decomposition for MPI parallelization. It provides two different ways for users to specify this decomposition, a `simple` way recommended for most users, and a `flexible` way recommended if more control is desired. The `flexible` method is required for dynamic load balancing to be useful. + +.. _usage_domain_decomposition-simple: + 1. Simple Method ---------------- @@ -14,6 +17,9 @@ The first and simplest method is to provide the ``warpx.numprocs = nx ny nz`` pa If ``warpx.numprocs`` is *not* specified, WarpX will fall back on using the ``amr.max_grid_size`` and ``amr.blocking_factor`` parameters, described below. + +.. _usage_domain_decomposition-general: + 2. More General Method ---------------------- diff --git a/Examples/Physics_applications/laser_ion/README.rst b/Examples/Physics_applications/laser_ion/README.rst index 18a427aa1c7..29862a30518 100644 --- a/Examples/Physics_applications/laser_ion/README.rst +++ b/Examples/Physics_applications/laser_ion/README.rst @@ -15,12 +15,6 @@ Although laser-ion acceleration requires full 3D modeling for adequate descripti This includes spatial and temporal resolution, but also the number of macro-particles per cell representing the target density for proper phase space sampling. You will need a computing cluster for adequate resolution of the target density, see comments in the input file. -.. warning:: - - It is strongly advised to set the parameters ``.zmin / zmax / xmin / ...`` when working with highly dense targets that are limited in one or multiple dimensions. - The particle creation routine will first create particles everywhere between these limits (`defaulting to box size if unset`), setting particles to invalid only afterwards based on the density profile. - Not setting these parameters can quickly lead to memory overflows. - Run --- @@ -30,7 +24,12 @@ This example can be run **either** as: * **Python** script: ``mpiexec -n 2 python3 PICMI_inputs_2d.py`` or * WarpX **executable** using an input file: ``mpiexec -n 2 warpx.2d inputs_2d`` -For `MPI-parallel `__ runs on computing clusters, change the prefix to ``mpiexec -n ...`` or ``srun -n ...``, depending on the system and number of MPI ranks you want to allocate. +.. tip:: + + For `MPI-parallel `__ runs on computing clusters, change the prefix to ``mpiexec -n ...`` or ``srun -n ...``, depending on the system and number of MPI ranks you want to allocate. + + The input option ``warpx_numprocs`` / ``warpx.numprocs`` needs to be adjusted for parallel :ref:`domain decomposition `, to match the number of MPI ranks used. + In order to use dynamic load balancing, use the :ref:`more general method ` of setting blocks. .. tab-set:: @@ -53,18 +52,18 @@ Analyze .. _fig-tnsa-ps-electrons-pinhole: .. figure:: https://user-images.githubusercontent.com/5416860/295003882-c755fd47-4bb3-4439-9319-c48214cbaafd.png - :alt: Longitudinal phase space of forward-flying electrons in a 2 degree opening angle. + :alt: Longitudinal phase space of forward-moving electrons in a 2 degree opening angle. :width: 100% - Longitudinal phase space of forward-flying electrons in a 2 degree opening angle. + Longitudinal phase space of forward-moving electrons in a 2 degree opening angle. .. _fig-tnsa-ps-protons-pinhole: .. figure:: https://user-images.githubusercontent.com/5416860/295003988-dea3dfb7-0d55-4616-b32d-061fb429f9ac.png - :alt: Longitudinal phase space of forward-flying protons in a 2 degree opening angle. + :alt: Longitudinal phase space of forward-moving protons in a 2 degree opening angle. :width: 100% - Longitudinal phase space of forward-flying protons in a 2 degree opening angle. + Longitudinal phase space of forward-moving protons in a 2 degree opening angle. Time-resolved phase electron space analysis as in :numref:`fig-tnsa-ps-electrons-pinhole` gives information about, e.g., how laser energy is locally converted into electron kinetic energy. Later in time, ion phase spaces like :numref:`fig-tnsa-ps-protons-pinhole` can reveal where accelerated ion populations originate. From 94871c7162b835e2cfff699abbd80d85f3156fac Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 24 Apr 2024 16:04:24 -0700 Subject: [PATCH 071/190] fix failing `development` builds (#4890) --- Source/Initialization/WarpXInitData.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 47eeaa41f86..e0c095918fa 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -480,9 +480,9 @@ WarpX::InitData () DistributionMap(lev_zero), getngEB(), Geom(lev_zero), - getField(FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), - getField(FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), - getField(FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() ); } From ed7e824911da5b25c9bc185dc3c164322156ee5e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 24 Apr 2024 23:48:31 -0700 Subject: [PATCH 072/190] Great Lakes Cluster (UMich) (#4869) * Great Lakes Cluster (UMich) Document how to use the Great Lakes Cluster at the University of Michigan. * Fix c-blosc2 typos * Parallel HDF5 for CUDA-aware MPI New module added on the system. * Fix GPU Visibility Ensure one MPI rank sees exactly one, unique GPU. * Add `#SBATCH --gpu-bind=single:1` * Add clean-up message. --- Docs/source/install/hpc.rst | 1 + Docs/source/install/hpc/greatlakes.rst | 240 ++++++++++++++++++ .../greatlakes-umich/greatlakes_v100.sbatch | 38 +++ .../greatlakes_v100_warpx.profile.example | 62 +++++ .../install_v100_dependencies.sh | 133 ++++++++++ 5 files changed, 474 insertions(+) create mode 100644 Docs/source/install/hpc/greatlakes.rst create mode 100644 Tools/machines/greatlakes-umich/greatlakes_v100.sbatch create mode 100644 Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example create mode 100755 Tools/machines/greatlakes-umich/install_v100_dependencies.sh diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst index a7b0f636b56..5efdeae39a8 100644 --- a/Docs/source/install/hpc.rst +++ b/Docs/source/install/hpc.rst @@ -51,6 +51,7 @@ This section documents quick-start guides for a selection of supercomputers that hpc/spock hpc/summit hpc/taurus + hpc/greatlakes .. tip:: diff --git a/Docs/source/install/hpc/greatlakes.rst b/Docs/source/install/hpc/greatlakes.rst new file mode 100644 index 00000000000..d238d498e5c --- /dev/null +++ b/Docs/source/install/hpc/greatlakes.rst @@ -0,0 +1,240 @@ +.. _building-greatlakes: + +Great Lakes (UMich) +=================== + +The `Great Lakes cluster `_ is located at University of Michigan. +The cluster has various partitions, including `GPU nodes and CPU nodes `__. + + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `Great Lakes user guide `__ +* Batch system: `Slurm `__ +* `Jupyter service `__ (`documentation `__) +* `Filesystems `__: + + * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (80GB) + * ``/scratch``: per-project `production directory `__; very fast for parallel jobs; purged every 60 days (10TB default) + + +.. _building-greatlakes-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + +On Great Lakes, you can run either on GPU nodes with `fast V100 GPUs (recommended), the even faster A100 GPUs (only a few available) or CPU nodes `__. + +.. tab-set:: + + .. tab-item:: V100 GPUs + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/greatlakes_v100_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example $HOME/greatlakes_v100_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``iloveplasma``, then run ``nano $HOME/greatlakes_v100_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="iloveplasma" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to Great Lakes, activate these environment settings: + + .. code-block:: bash + + source $HOME/greatlakes_v100_warpx.profile + + Finally, since Great Lakes does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/greatlakes-umich/install_v100_dependencies.sh + source ${HOME}/sw/greatlakes/v100/venvs/warpx-v100/bin/activate + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/greatlakes-umich/install_v100_dependencies.sh + :language: bash + + + .. tab-item:: A100 Nodes + + .. note:: + + This section is TODO. + + + .. tab-item:: CPU Nodes + + .. note:: + + This section is TODO. + + +.. _building-greatlakes-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. tab-set:: + + .. tab-item:: V100 GPUs + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100 + + cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100 -j 8 + + The WarpX application executables are now in ``$HOME/src/warpx/build_v100/bin/``. + Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100_py + + cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100_py -j 8 --target pip_install + + + .. tab-item:: A100 Nodes + + .. note:: + + This section is TODO. + + + .. tab-item:: CPU Nodes + + .. note:: + + This section is TODO. + +Now, you can :ref:`submit Great Lakes compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). +Or, you can use the WarpX executables to submit greatlakes jobs (:ref:`example inputs `). +For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch``. + + +.. _building-greatlakes-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd $HOME/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the greatlakes_v100_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. + + +.. _running-cpp-greatlakes: + +Running +------- + +.. tab-set:: + + .. tab-item:: V100 (16GB) GPUs + + The batch script below can be used to run a WarpX simulation on multiple nodes (change ``-N`` accordingly) on the supercomputer Great Lakes at University of Michigan. + This partition has `20 nodes, each with two V100 GPUs `__. + + Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``plasma_mirror_inputs``. + Note that we run one MPI rank per GPU. + + .. literalinclude:: ../../../../Tools/machines/greatlakes-umich/greatlakes_v100.sbatch + :language: bash + :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch``. + + To run a simulation, copy the lines above to a file ``greatlakes_v100.sbatch`` and run + + .. code-block:: bash + + sbatch greatlakes_v100.sbatch + + to submit the job. + + + .. tab-item:: A100 (80GB) GPUs + + This partition has `2 nodes, each with four A100 GPUs `__ that provide 80 GB HBM per A100 GPU. + To the user, each node will appear as if it has 8 A100 GPUs with 40 GB memory each. + + .. note:: + + This section is TODO. + + + .. tab-item:: CPU Nodes + + The Great Lakes CPU partition as up to `455 nodes `__, each with 2x Intel Xeon Gold 6154 CPUs and 180 GB RAM. + + .. note:: + + This section is TODO. + + +.. _post-processing-greatlakes: + +Post-Processing +--------------- + +For post-processing, many users prefer to use the online `Jupyter service `__ (`documentation `__) that is directly connected to the cluster's fast filesystem. + +.. note:: + + This section is a stub and contributions are welcome. + We can document further details, e.g., which recommended post-processing Python software to install or how to customize Jupyter kernels here. diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch b/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch new file mode 100644 index 00000000000..0353c08456f --- /dev/null +++ b/Tools/machines/greatlakes-umich/greatlakes_v100.sbatch @@ -0,0 +1,38 @@ +#!/bin/bash -l + +# Copyright 2024 The WarpX Community +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +#SBATCH -t 00:10:00 +#SBATCH -N 1 +#SBATCH -J WarpX +#SBATCH -A +#SBATCH --partition=gpu +#SBATCH --exclusive +#SBATCH --ntasks-per-node=2 +#SBATCH --cpus-per-task=20 +#SBATCH --gpus-per-task=v100:1 +#SBATCH --gpu-bind=single:1 +#SBATCH -o WarpX.o%j +#SBATCH -e WarpX.e%j + +# executable & inputs file or python interpreter & PICMI script here +EXE=./warpx +INPUTS=inputs + +# threads for OpenMP and threaded compressors per MPI rank +# per node are 2x 2.4 GHz Intel Xeon Gold 6148 +# note: the system seems to only expose cores (20 per socket), +# not hyperthreads (40 per socket) +export SRUN_CPUS_PER_TASK=20 +export OMP_NUM_THREADS=${SRUN_CPUS_PER_TASK} + +# GPU-aware MPI optimizations +GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" + +# run WarpX +srun --cpu-bind=cores \ + ${EXE} ${INPUTS} ${GPU_AWARE_MPI} \ + > output.txt diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example new file mode 100644 index 00000000000..98476eecfab --- /dev/null +++ b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example @@ -0,0 +1,62 @@ +# please set your project account +export proj="" # change me! + +# remembers the location of this script +export MY_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your $MY_PROFILE file! Please edit its line 2 to continue!"; return; fi + +# required dependencies +module purge +module load gcc/10.3.0 +module load cuda/12.1.1 +module load cmake/3.26.3 +module load openblas/0.3.23 +module load openmpi/4.1.6-cuda + +# optional: for QED support +module load boost/1.78.0 + +# optional: for openPMD and PSATD+RZ support +module load phdf5/1.12.1 + +SW_DIR="${HOME}/sw/greatlakes/v100" +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc2-2.14.4:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.0:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH + +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc2-2.14.4/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.0/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH + +export PATH=${SW_DIR}/adios2-2.10.0/bin:${PATH} + +# optional: for Python bindings or libEnsemble +module load python/3.12.1 + +if [ -d "${SW_DIR}/venvs/warpx-v100" ] +then + source ${SW_DIR}/venvs/warpx-v100/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 --partition=gpu --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --partition=gpu --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" + +# optimize CUDA compilation for V100 +export AMREX_CUDA_ARCH=7.0 + +# optimize CPU microarchitecture for Intel Xeon Gold 6148 +export CXXFLAGS="-march=skylake-avx512" +export CFLAGS="-march=skylake-avx512" + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh new file mode 100755 index 00000000000..7235b18c99f --- /dev/null +++ b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh @@ -0,0 +1,133 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Axel Huebl +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + + +# Check: ###################################################################### +# +# Was greatlakes_v100_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your greatlakes_v100_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi + + +# Remove old dependencies ##################################################### +# +echo "Cleaning up prior installation directory... This may take several minutes." +SW_DIR="${HOME}/sw/greatlakes/v100" +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + + +# General extra dependencies ################################################## +# + +# tmpfs build directory: avoids issues often seen with $HOME and is faster +build_dir=$(mktemp -d) + +# c-blosc (I/O compression) +if [ -d $HOME/src/c-blosc2 ] +then + cd $HOME/src/c-blosc2 + git fetch --prune + git checkout v2.14.4 + cd - +else + git clone -b v2.14.4 https://github.com/Blosc/c-blosc2.git $HOME/src/c-blosc2 +fi +rm -rf $HOME/src/c-blosc2-v100-build +cmake -S $HOME/src/c-blosc2 -B ${build_dir}/c-blosc2-v100-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_EXAMPLES=OFF -DBUILD_FUZZERS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc2-2.14.4 +cmake --build ${build_dir}/c-blosc2-v100-build --target install --parallel 8 +rm -rf ${build_dir}/c-blosc2-v100-build + +# ADIOS2 +if [ -d $HOME/src/adios2 ] +then + cd $HOME/src/adios2 + git fetch --prune + git checkout v2.10.0 + cd - +else + git clone -b v2.10.0 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 +fi +rm -rf $HOME/src/adios2-v100-build +cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-v100-build -DADIOS2_USE_Blosc2=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.0 +cmake --build ${build_dir}/adios2-v100-build --target install -j 8 +rm -rf ${build_dir}/adios2-v100-build + +# BLAS++ (for PSATD+RZ) +if [ -d $HOME/src/blaspp ] +then + cd $HOME/src/blaspp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp +fi +rm -rf $HOME/src/blaspp-v100-build +cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-v100-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake --build ${build_dir}/blaspp-v100-build --target install --parallel 8 +rm -rf ${build_dir}/blaspp-v100-build + +# LAPACK++ (for PSATD+RZ) +if [ -d $HOME/src/lapackpp ] +then + cd $HOME/src/lapackpp + git fetch --prune + git checkout master + git pull + cd - +else + git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp +fi +rm -rf $HOME/src/lapackpp-v100-build +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-v100-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake --build ${build_dir}/lapackpp-v100-build --target install --parallel 8 +rm -rf ${build_dir}/lapackpp-v100-build + + +# Python ###################################################################### +# +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade virtualenv +python3 -m pip cache purge +rm -rf ${SW_DIR}/venvs/warpx-v100 +python3 -m venv ${SW_DIR}/venvs/warpx-v100 +source ${SW_DIR}/venvs/warpx-v100/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt +# install or update WarpX dependencies +python3 -m pip install --upgrade -r $HOME/src/warpx/requirements.txt +python3 -m pip install --upgrade cupy-cuda12x # CUDA 12 compatible wheel +# optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) +python3 -m pip install --upgrade torch # CUDA 12 compatible wheel +python3 -m pip install --upgrade optimas[all] + + +# remove build temporary directory +rm -rf ${build_dir} From 07f239c70fe3678d5b438012df0853222b42cc80 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 24 Apr 2024 23:51:22 -0700 Subject: [PATCH 073/190] Open Governance Model (#4743) * [Draft] Open Governance Model This is a first draft of the open governance model for WarpX, developed after discussion with the current leadership team. The goal of the governance model is to: - document and make transparent our decision structure, - structure and openly credit sensible roles, - clarify appointment and offboarding, - encourage participation, engagement and growth. Thoughts that guided the initial document include: - accurately reflecting our successful project workflows, - evolving it for shared responsibiliy and co-ownership, - avoiding unnecessary overheads and deadlocks, - be flexible enough to enable later introduction of sub-roles and further formalization, depending on community growth. * Review Session today * Clarify SC permissions are for each member. * Fix Typo: Groenewald Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * Added TC Veto Overwrite Rule * Documentation & Link to (Access) Developer Meeting Notes * Added Jean-Luc also to TC --------- Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- Docs/source/governance.rst | 1 + Docs/source/index.rst | 1 + GOVERNANCE.rst | 144 +++++++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+) create mode 120000 Docs/source/governance.rst create mode 100644 GOVERNANCE.rst diff --git a/Docs/source/governance.rst b/Docs/source/governance.rst new file mode 120000 index 00000000000..1b1b99778ab --- /dev/null +++ b/Docs/source/governance.rst @@ -0,0 +1 @@ +../../GOVERNANCE.rst \ No newline at end of file diff --git a/Docs/source/index.rst b/Docs/source/index.rst index 9f2d2ff038e..89bc2e417cb 100644 --- a/Docs/source/index.rst +++ b/Docs/source/index.rst @@ -150,4 +150,5 @@ Epilogue :hidden: glossary + governance acknowledgements diff --git a/GOVERNANCE.rst b/GOVERNANCE.rst new file mode 100644 index 00000000000..b5253b80f9f --- /dev/null +++ b/GOVERNANCE.rst @@ -0,0 +1,144 @@ +.. _governance: + +WarpX Governance +================ + +WarpX is led in an open governance model, described in this file. + + +Steering Committee +------------------ + +Current Roster +^^^^^^^^^^^^^^ + +- Jean-Luc Vay (chair) +- Remi Lehe +- Axel Huebl + +See: `GitHub team `__ + +Role +^^^^ + +Members of the steering committee (SC) can change organizational settings, do administrative operations such as rename/move/archive repositories, change branch protection rules, etc. +SC members can call votes for decisions (technical or governance). + +The SC can veto decisions of the technical committee (TC) by voting in the SC. +The TC can overwrite a veto with a 2/3rd majority vote in the TC. +Decisions are documented in the `weekly developer meeting notes `__ and/or on the GitHub repository. + +The SC can change the governance structure, but only in a unanimous vote. + +Decision Process +^^^^^^^^^^^^^^^^ + +Decision of the SC usually happen in the weekly developer meetings, via e-mail or public chat. + +Decisions are made in a non-confidential manner, by majority on the cast votes of SC members. +Votes can be cast in asynchronous manner, e.g., over the time of 1-2 weeks. +In tie situations, the chair of the SC acts as the tie breaker. + +Appointment Process +^^^^^^^^^^^^^^^^^^^ + +Appointed by current SC members in an unanimous vote. +As a SC member, regularly attending and contributing to the weekly developer meetings is expected. + +SC members can resign or be removed by majority vote, e.g., due to inactivity, bad acting or other reasons. + + +Technical Committee +------------------- + +Current Roster +^^^^^^^^^^^^^^ + +- Luca Fedeli +- Roelof Groenewald +- David Grote +- Axel Huebl +- Revathi Jambunathan +- Remi Lehe +- Andrew Myers +- Maxence Thévenet +- Jean-Luc Vay +- Weiqun Zhang +- Edoardo Zoni + +See: `GitHub team `__ + +Role +^^^^ + +The technical committee (TC) is the core governance body, where under normal operations most ideas are discussed and decisions are made. +Individual TC members can approve and merge code changes. +Usually, they seek approval by another maintainer for their own changes, too. +TC members lead - and weigh in on - technical discussions and, if needed, can call for a vote between TC members for a technical decision. +TC members merge/close PRs and issues, and moderate (including block/mute) bad actors. +The TC can propose governance changes to the SC. + + +Decision Process +^^^^^^^^^^^^^^^^ + +Discussion in the TC usually happens in the weekly developer meetings. + +If someone calls for a vote to make a decision: majority based on the cast votes; we need 50% of the committee participating to vote. In the absence of a quorum, the SC will decide according to its voting rules. + +Votes are cast in a non-confidential manner. +Decisions are documented in the `weekly developer meeting notes `__ and/or on the GitHub repository. + +TC members can individually appoint new contributors, unless a vote is called on an individual. + +Appointment Process +^^^^^^^^^^^^^^^^^^^ + +TC members are the maintainers of WarpX. +As a TC member, regularly attending and contributing to the weekly developer meetings is expected. + +One is appointed to the TC by the steering committee, in a unanimous vote, or by majority vote of the TC. The SC can veto appointments. +Steering committee members can also be TC members. + +TC members can resign or be removed by majority vote by either TC or SC, e.g., due to inactivity, bad acting or other reasons. + + +Contributors +------------ + +Current Roster +^^^^^^^^^^^^^^ + +See: `GitHub team `__ + +Role +^^^^ + +Contributors are valuable, vetted developers of WarpX. +Contributions can be in many forms and not all need to be code contributions. +Examples include code pull requests, support in issues & user discussions, writing and updating documentation, writing tutorials, visualizations, R&D on algorithms, testing and benchmarking, etc. +Contributors can participate in developer meetings and weigh in on discussions. +Contributors can "triage" (add labels) to pull requests, issues, and GitHub discussion pages. +Contributors can comment and review PRs (but not merge). + +Decision Process +^^^^^^^^^^^^^^^^ + +Contributors can individually decide on classification (triage) of pull requests, issues, and GitHub discussion pages. + +Appointment Process +^^^^^^^^^^^^^^^^^^^ + +Appointed after contributing to WarpX (see above) by any member of the TC. + +The role can be lost by resigning or by decision of an individual TC or SC member, e.g., due to inactivity, bad acting or other. + + +Former Members +-------------- + +"Former members" are the giants on whose shoulders we stand. +But, for the purpose of WarpX governance, they are *not* tracked as a governance role in WarpX. +Instead, former (e.g., inactive) contributors are acknowledged separately in GitHub contributor tracking, the WarpX documentation, references, citable Zenodo archives of releases, etc. as appropriate. + +Former members of SC, TC and Contributors are not kept in the roster, since committee role rosters shall reflect currently active members and the responsible governance body. From bc726a5f4d014cbf3551a14061db48d15be3684d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 25 Apr 2024 09:02:41 -0700 Subject: [PATCH 074/190] openPMD-api: 0.15.2 (#4847) * openPMD-api: 0.15.2 This patch release mostly improves reads. https://openpmd-api.readthedocs.io/en/0.15.2/install/changelog.html#id1 * 0.15.1+ - technically also works - we forgot to bump the installed version in 0.15.2 --- .github/workflows/dependencies/icc.sh | 2 +- Docs/source/install/cmake.rst | 2 +- Docs/source/usage/parameters.rst | 14 +++++++------- cmake/dependencies/openPMD.cmake | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dependencies/icc.sh b/.github/workflows/dependencies/icc.sh index 74f0db4b46a..fae6e22d45a 100755 --- a/.github/workflows/dependencies/icc.sh +++ b/.github/workflows/dependencies/icc.sh @@ -58,7 +58,7 @@ export CEI_TMP="/tmp/cei" CXX=$(which icpc) CC=$(which icc) \ cmake-easyinstall \ --prefix=/usr/local \ - git+https://github.com/openPMD/openPMD-api.git@0.15.1 \ + git+https://github.com/openPMD/openPMD-api.git@0.15.2 \ -DopenPMD_USE_PYTHON=OFF \ -DBUILD_TESTING=OFF \ -DBUILD_EXAMPLES=OFF \ diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 2e66c95a4d6..006cd843b31 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -125,7 +125,7 @@ CMake Option Default & Values Des ``WarpX_amrex_internal`` **ON**/OFF Needs a pre-installed AMReX library if set to ``OFF`` ``WarpX_openpmd_src`` *None* Path to openPMD-api source directory (preferred if set) ``WarpX_openpmd_repo`` ``https://github.com/openPMD/openPMD-api.git`` Repository URI to pull and build openPMD-api from -``WarpX_openpmd_branch`` ``0.15.1`` Repository branch for ``WarpX_openpmd_repo`` +``WarpX_openpmd_branch`` ``0.15.2`` Repository branch for ``WarpX_openpmd_repo`` ``WarpX_openpmd_internal`` **ON**/OFF Needs a pre-installed openPMD-api library if set to ``OFF`` ``WarpX_picsar_src`` *None* Path to PICSAR source directory (preferred if set) ``WarpX_picsar_repo`` ``https://github.com/ECP-WarpX/picsar.git`` Repository URI to pull and build PICSAR from diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 9f88f2d8c6c..5f97730e880 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2558,18 +2558,18 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a When WarpX is compiled with openPMD support, the first available backend in the order given above is taken. * ``.openpmd_encoding`` (optional, ``v`` (variable based), ``f`` (file based) or ``g`` (group based) ) only read if ``.format = openpmd``. - openPMD `file output encoding `__. + openPMD `file output encoding `__. File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). - ``variable based`` is an `experimental feature with ADIOS2 `__ and not supported for back-transformed diagnostics. + ``variable based`` is an `experimental feature with ADIOS2 `__ and not supported for back-transformed diagnostics. Default: ``f`` (full diagnostics) * ``.adios2_operator.type`` (``zfp``, ``blosc``) optional, - `ADIOS2 I/O operator type `__ for `openPMD `_ data dumps. + `ADIOS2 I/O operator type `__ for `openPMD `_ data dumps. * ``.adios2_operator.parameters.*`` optional, - `ADIOS2 I/O operator parameters `__ for `openPMD `_ data dumps. + `ADIOS2 I/O operator parameters `__ for `openPMD `_ data dumps. - A typical example for `ADIOS2 output using lossless compression `__ with ``blosc`` using the ``zstd`` compressor and 6 CPU treads per MPI Rank (e.g. for a `GPU run with spare CPU resources `__): + A typical example for `ADIOS2 output using lossless compression `__ with ``blosc`` using the ``zstd`` compressor and 6 CPU treads per MPI Rank (e.g. for a `GPU run with spare CPU resources `__): .. code-block:: text @@ -2588,11 +2588,11 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a .adios2_operator.parameters.precision = 3 * ``.adios2_engine.type`` (``bp4``, ``sst``, ``ssc``, ``dataman``) optional, - `ADIOS2 Engine type `__ for `openPMD `_ data dumps. + `ADIOS2 Engine type `__ for `openPMD `_ data dumps. See full list of engines at `ADIOS2 readthedocs `__ * ``.adios2_engine.parameters.*`` optional, - `ADIOS2 Engine parameters `__ for `openPMD `_ data dumps. + `ADIOS2 Engine parameters `__ for `openPMD `_ data dumps. An example for parameters for the BP engine are setting the number of writers (``NumAggregators``), transparently redirecting data to burst buffers etc. A detailed list of engine-specific parameters are available at the official `ADIOS2 documentation `__ diff --git a/cmake/dependencies/openPMD.cmake b/cmake/dependencies/openPMD.cmake index 93599d7505e..f58d37ee92e 100644 --- a/cmake/dependencies/openPMD.cmake +++ b/cmake/dependencies/openPMD.cmake @@ -13,7 +13,7 @@ function(find_openpmd) if(WarpX_openpmd_internal OR WarpX_openpmd_src) set(CMAKE_POLICY_DEFAULT_CMP0077 NEW) - # see https://openpmd-api.readthedocs.io/en/0.15.1/dev/buildoptions.html + # see https://openpmd-api.readthedocs.io/en/0.15.2/dev/buildoptions.html set(openPMD_USE_ADIOS1 OFF CACHE INTERNAL "") set(openPMD_USE_MPI ${WarpX_MPI} CACHE INTERNAL "") set(openPMD_USE_PYTHON OFF CACHE INTERNAL "") @@ -92,7 +92,7 @@ if(WarpX_OPENPMD) set(WarpX_openpmd_repo "https://github.com/openPMD/openPMD-api.git" CACHE STRING "Repository URI to pull and build openPMD-api from if(WarpX_openpmd_internal)") - set(WarpX_openpmd_branch "0.15.1" + set(WarpX_openpmd_branch "0.15.2" CACHE STRING "Repository branch for WarpX_openpmd_repo if(WarpX_openpmd_internal)") From 92fddfcd2beeb47a714474c10ba3d393bb8f15ba Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 25 Apr 2024 19:17:08 +0200 Subject: [PATCH 075/190] Add two missing includes (found with iwyu) (#4895) * add missing include * add missing include --- Source/Diagnostics/OpenPMDHelpFunction.cpp | 3 +++ Source/Initialization/WarpXAMReXInit.cpp | 2 ++ 2 files changed, 5 insertions(+) diff --git a/Source/Diagnostics/OpenPMDHelpFunction.cpp b/Source/Diagnostics/OpenPMDHelpFunction.cpp index 6170249b52b..cc798adc29e 100644 --- a/Source/Diagnostics/OpenPMDHelpFunction.cpp +++ b/Source/Diagnostics/OpenPMDHelpFunction.cpp @@ -6,8 +6,11 @@ * License: BSD-3-Clause-LBNL */ #include "OpenPMDHelpFunction.H" + #include "Utils/TextMsg.H" +#include + std::string WarpXOpenPMDFileType () { diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index 8cfe78c740f..4036a49235c 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -11,6 +11,8 @@ #include #include +#include + namespace { /** Overwrite defaults in AMReX Inputs * From 2aee638758948dca16ba832ed300e6adc28b61ae Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 29 Apr 2024 06:30:22 -0700 Subject: [PATCH 076/190] `isort`: First Party Modules (#4902) * `isort`: First Party Modules Declare `amrex` and `pywarpx` our first party modules, even if they are not installed. This resolves inconsistencies when sorting imports in developer or remote runs (pre-commit.com CI). * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Docs/source/usage/workflows/ml_materials/run_warpx_training.py | 1 + .../capacitive_discharge/PICMI_inputs_1d.py | 3 ++- .../capacitive_discharge/PICMI_inputs_2d.py | 3 ++- .../spacecraft_charging/PICMI_inputs_rz.py | 1 + .../Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py | 1 + Examples/Tests/langmuir/PICMI_inputs_rz.py | 1 + Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py | 1 + Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py | 1 + Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py | 1 + Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py | 1 + Examples/Tests/ohm_solver_EM_modes/analysis.py | 1 + Examples/Tests/ohm_solver_EM_modes/analysis_rz.py | 3 ++- Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py | 1 + Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py | 1 + Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py | 1 + Examples/Tests/ohm_solver_ion_beam_instability/analysis.py | 1 + .../Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py | 1 + .../Tests/particle_boundary_interaction/PICMI_inputs_rz.py | 1 + Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py | 1 + Examples/Tests/particle_data_python/PICMI_inputs_2d.py | 1 + .../Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py | 1 + Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py | 1 + Examples/Tests/python_wrappers/PICMI_inputs_2d.py | 1 + Examples/Tests/restart/PICMI_inputs_id_cpu_read.py | 1 + .../Tests/restart/PICMI_inputs_runtime_component_analyze.py | 1 + Python/pywarpx/picmi.py | 1 + pyproject.toml | 1 + 27 files changed, 30 insertions(+), 3 deletions(-) diff --git a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py index 23fc49e7431..054c1b4cfc0 100644 --- a/Docs/source/usage/workflows/ml_materials/run_warpx_training.py +++ b/Docs/source/usage/workflows/ml_materials/run_warpx_training.py @@ -2,6 +2,7 @@ import math import numpy as np + from pywarpx import picmi # Physical constants diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py index 0b59516ccc7..8b58790d2e6 100644 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_1d.py @@ -8,10 +8,11 @@ import sys import numpy as np -from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi from scipy.sparse import csc_matrix from scipy.sparse import linalg as sla +from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi + constants = picmi.constants diff --git a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py index 61229a17295..65baabba605 100755 --- a/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py +++ b/Examples/Physics_applications/capacitive_discharge/PICMI_inputs_2d.py @@ -6,10 +6,11 @@ # --- used for the field solve step. import numpy as np -from pywarpx import callbacks, fields, picmi from scipy.sparse import csc_matrix from scipy.sparse import linalg as sla +from pywarpx import callbacks, fields, picmi + constants = picmi.constants ########################## diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index 6eefdca942d..d541c2393ca 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -15,6 +15,7 @@ import numpy as np import scipy.constants as scc from mpi4py import MPI as mpi + from pywarpx import picmi from pywarpx.callbacks import installafterEsolve, installafterInitEsolve from pywarpx.fields import ExWrapper, EzWrapper, PhiFPWrapper, RhoFPWrapper diff --git a/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py b/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py index 6104e08ef27..faec3ed4668 100755 --- a/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py +++ b/Examples/Tests/embedded_boundary_python_api/PICMI_inputs_EB_API.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import numpy as np + from pywarpx import fields, picmi max_steps = 1 diff --git a/Examples/Tests/langmuir/PICMI_inputs_rz.py b/Examples/Tests/langmuir/PICMI_inputs_rz.py index 106b8de5a94..8328aba5185 100755 --- a/Examples/Tests/langmuir/PICMI_inputs_rz.py +++ b/Examples/Tests/langmuir/PICMI_inputs_rz.py @@ -9,6 +9,7 @@ matplotlib.use('Agg') import matplotlib.pyplot as plt import numpy as np + from pywarpx import fields, picmi constants = picmi.constants diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py index f61823bf223..8f205724563 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_3d.py @@ -27,6 +27,7 @@ import matplotlib.pyplot as plt import numpy as np import scipy.constants as con + from pywarpx import fields, picmi ########################## diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py index 6f7d6daa453..1268f7a02b0 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py @@ -27,6 +27,7 @@ import matplotlib.pyplot as plt import numpy as np import scipy.constants as con + from pywarpx import fields, picmi ########################## diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py index 6711331d538..ec67282fd88 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs.py @@ -14,6 +14,7 @@ import dill import numpy as np from mpi4py import MPI as mpi + from pywarpx import callbacks, fields, libwarpx, picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py index e6f9ccc8e65..89c5230a6ad 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py @@ -12,6 +12,7 @@ import dill import numpy as np from mpi4py import MPI as mpi + from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis.py b/Examples/Tests/ohm_solver_EM_modes/analysis.py index 77be546937c..ad832dc2f50 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis.py @@ -6,6 +6,7 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np + from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py index de7c9966b20..58e615c5332 100755 --- a/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/analysis_rz.py @@ -8,10 +8,11 @@ import scipy.fft as fft from matplotlib import colors from openpmd_viewer import OpenPMDTimeSeries -from pywarpx import picmi from scipy.interpolate import RegularGridInterpolator from scipy.special import j1, jn, jn_zeros +from pywarpx import picmi + constants = picmi.constants # load simulation parameters diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py index 8249bdb504f..d4430502f42 100644 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/PICMI_inputs.py @@ -13,6 +13,7 @@ import dill import numpy as np from mpi4py import MPI as mpi + from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py b/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py index 93a4962d7e5..ade7187ebe3 100755 --- a/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py +++ b/Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py @@ -6,6 +6,7 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np + from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py index 186ef6cede3..4f8b5edcc3e 100644 --- a/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/PICMI_inputs.py @@ -14,6 +14,7 @@ import dill import numpy as np from mpi4py import MPI as mpi + from pywarpx import callbacks, fields, libwarpx, particle_containers, picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py index 6388a4c4085..6a57b1c1046 100755 --- a/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py +++ b/Examples/Tests/ohm_solver_ion_beam_instability/analysis.py @@ -7,6 +7,7 @@ import matplotlib import matplotlib.pyplot as plt import numpy as np + from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py index 5d4631e9c1e..56173893b7f 100644 --- a/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py +++ b/Examples/Tests/ohm_solver_magnetic_reconnection/PICMI_inputs.py @@ -15,6 +15,7 @@ import dill import numpy as np from mpi4py import MPI as mpi + from pywarpx import callbacks, fields, libwarpx, picmi constants = picmi.constants diff --git a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py index b9ea76e4a09..4017a05d413 100644 --- a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py +++ b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py @@ -4,6 +4,7 @@ # --- This input is a simple case of reflection # --- of one electron on the surface of a sphere. import numpy as np + from pywarpx import callbacks, particle_containers, picmi ########################## diff --git a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py index 63a805c2612..e5a9a58f597 100755 --- a/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py +++ b/Examples/Tests/particle_boundary_scrape/PICMI_inputs_scrape.py @@ -4,6 +4,7 @@ # --- to access the buffer of scraped particles. import numpy as np + from pywarpx import libwarpx, particle_containers, picmi ########################## diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py index f43c2215b79..572871b8ed5 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_2d.py @@ -3,6 +3,7 @@ import sys import numpy as np + from pywarpx import callbacks, libwarpx, particle_containers, picmi # Create the parser and add the argument diff --git a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py index 6f70b69fc10..5de9879f0f8 100755 --- a/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py +++ b/Examples/Tests/particle_data_python/PICMI_inputs_prev_pos_2d.py @@ -3,6 +3,7 @@ # --- Input file to test the saving of old particle positions import numpy as np + from pywarpx import particle_containers, picmi constants = picmi.constants diff --git a/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py b/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py index 095702a1145..66f259da2ef 100755 --- a/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py +++ b/Examples/Tests/pass_mpi_communicator/PICMI_inputs_2d.py @@ -6,6 +6,7 @@ # --- if the correct amount of processors are initialized in AMReX. from mpi4py import MPI + from pywarpx import picmi constants = picmi.constants diff --git a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py index d01835f9fdf..db1cc7dcad8 100755 --- a/Examples/Tests/python_wrappers/PICMI_inputs_2d.py +++ b/Examples/Tests/python_wrappers/PICMI_inputs_2d.py @@ -3,6 +3,7 @@ import matplotlib.pyplot as plt import numpy as np from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable + from pywarpx import picmi # Number of time steps diff --git a/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py b/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py index d76cef82fec..d400924a378 100755 --- a/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py +++ b/Examples/Tests/restart/PICMI_inputs_id_cpu_read.py @@ -6,6 +6,7 @@ import sys import numpy as np + from pywarpx import callbacks, particle_containers, picmi ########################## diff --git a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py index daac6e1b5ac..32c9f4e5808 100755 --- a/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py +++ b/Examples/Tests/restart/PICMI_inputs_runtime_component_analyze.py @@ -7,6 +7,7 @@ import sys import numpy as np + from pywarpx import callbacks, particle_containers, picmi ########################## diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 2859d6fd468..902d6150773 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -14,6 +14,7 @@ import numpy as np import periodictable + import picmistandard import pywarpx diff --git a/pyproject.toml b/pyproject.toml index 5173216a0f4..f9e615f9b83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,4 +8,5 @@ requires = [ build-backend = "setuptools.build_meta" [tool.isort] +known_first_party = ["amrex", "picmistandard", "pywarpx", "warpx"] profile = "black" From a294ff2fca63890d59d6613f41cb211d44aa72dc Mon Sep 17 00:00:00 2001 From: John Ryan Westernacher-Schneider Date: Mon, 29 Apr 2024 17:54:57 +0200 Subject: [PATCH 077/190] Clarify ParticleContainerWrapper.add_particles() docstring (#4903) --- Python/pywarpx/particle_containers.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Python/pywarpx/particle_containers.py b/Python/pywarpx/particle_containers.py index f5330d05f84..70471e24d1a 100644 --- a/Python/pywarpx/particle_containers.py +++ b/Python/pywarpx/particle_containers.py @@ -42,17 +42,18 @@ def add_particles(self, x=None, y=None, z=None, ux=None, uy=None, The type of species for which particles will be added x, y, z : arrays or scalars - The particle positions (default = 0.) + The particle positions (m) (default = 0.) ux, uy, uz : arrays or scalars - The particle momenta (default = 0.) + The particle proper velocities (m/s) (default = 0.) w : array or scalars Particle weights (default = 0.) unique_particles : bool - Whether the particles are unique or duplicated on several processes - (default = True) + True means the added particles are duplicated by each process; + False means the number of added particles is independent of + the number of processes (default = True) kwargs : dict Containing an entry for all the extra particle attribute arrays. If From 155336b513bbb455fec33624b84242df73f2eae3 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 29 Apr 2024 13:27:28 -0700 Subject: [PATCH 078/190] AMReX/pyAMReX/PICSAR: Weekly Update (#4904) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index abd7e54c013..f73c39b6fc3 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach b752027c1aebdfb4be339b1e30932b4108286a7a && cd - + cd ../amrex && git checkout --detach d857968968ae2cdf18f39ee50df96529e326ff39 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 048efbc06be..94b334d02da 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = b752027c1aebdfb4be339b1e30932b4108286a7a +branch = d857968968ae2cdf18f39ee50df96529e326ff39 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 1f656960789..f8859ef7df5 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = b752027c1aebdfb4be339b1e30932b4108286a7a +branch = d857968968ae2cdf18f39ee50df96529e326ff39 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 022939dcbee..cbde4bca135 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "b752027c1aebdfb4be339b1e30932b4108286a7a" +set(WarpX_amrex_branch "d857968968ae2cdf18f39ee50df96529e326ff39" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 038b98c191b..fcf7c91bce2 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "28897d26f00707ee33d2423b8c24a020cd0767e0" +set(WarpX_pyamrex_branch "f370d38ab410dbf1b3a90683c97291841ebeb3f5" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 9bfc3ba5fbf..75e56e6964f 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach b752027c1aebdfb4be339b1e30932b4108286a7a && cd - +cd amrex && git checkout --detach d857968968ae2cdf18f39ee50df96529e326ff39 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 8dbd3bdfef2aa9b75e36a0c9ef5281994a5d549d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 29 Apr 2024 16:48:49 -0700 Subject: [PATCH 079/190] Doc: Great Lakes ADIOS2 Update (#4905) Update the ADIOS2 v2.10.0 installation, do not create campaign directories (we do not use them yet). --- .../greatlakes-umich/install_v100_dependencies.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh index 7235b18c99f..9361fd6c64b 100755 --- a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh +++ b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh @@ -63,7 +63,15 @@ else git clone -b v2.10.0 https://github.com/ornladios/ADIOS2.git $HOME/src/adios2 fi rm -rf $HOME/src/adios2-v100-build -cmake -S $HOME/src/adios2 -B ${build_dir}/adios2-v100-build -DADIOS2_USE_Blosc2=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.0 +cmake \ + -S $HOME/src/adios2 \ + -B ${build_dir}/adios2-v100-build \ + -DADIOS2_USE_Blosc2=ON \ + -DADIOS2_USE_Campaign=OFF \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.0 cmake --build ${build_dir}/adios2-v100-build --target install -j 8 rm -rf ${build_dir}/adios2-v100-build From c5a5732721b4e3e0b16a61a56fa29548109d9b9f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 30 Apr 2024 08:55:08 -0700 Subject: [PATCH 080/190] Fix external fields on grid in 1D (#4906) --- Source/Initialization/WarpXInitData.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index e0c095918fa..ff2ca01deec 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1024,7 +1024,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; + const amrex::Real z = i*dx_lev[0] + real_box.lo(0) + fac_z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) const amrex::Real fac_x = (1._rt - x_nodal_flag[0]) * dx_lev[0] * 0.5_rt; const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; @@ -1059,7 +1059,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; + const amrex::Real z = i*dx_lev[0] + real_box.lo(0) + fac_z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) const amrex::Real fac_x = (1._rt - y_nodal_flag[0]) * dx_lev[0] * 0.5_rt; const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; @@ -1090,7 +1090,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser ( const amrex::Real x = 0._rt; const amrex::Real y = 0._rt; const amrex::Real fac_z = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; - const amrex::Real z = j*dx_lev[0] + real_box.lo(0) + fac_z; + const amrex::Real z = i*dx_lev[0] + real_box.lo(0) + fac_z; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) const amrex::Real fac_x = (1._rt - z_nodal_flag[0]) * dx_lev[0] * 0.5_rt; const amrex::Real x = i*dx_lev[0] + real_box.lo(0) + fac_x; From 8a9bedc394a48b4c002d356be4e54b13a064356d Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 2 May 2024 14:43:50 -0700 Subject: [PATCH 081/190] Add option for Cartesian velocity grid with `VelocityCoincidenceThinning` (#4886) * initial implementation of the grid based merging routine * add compiler directives for different dimensions * avoid implicit capture of `this` in lambda * code clean-up * use `PIdx::x` for r-coordinate while 4667 is still being reviewed * fix clang-tidy errors * another clang-tidy fix * improve doc-strings * use iterative heap sort since "SYCL kernel cannot call a recursive function" * fix clang-tidy error * fix sorting bug; add documentation and CI test * add dummy CI benchmark file to get proper values * update benchmark values based on Azure results * rename `GridBasedMerging` -> `VelocityCoincidenceThinning` * use `Algorithms::KineticEnergy` * reorganize merging loop * update CI benchmark values * Relativistic correction in product particles' velocity calculation * update benchmark values after changing energy calculation * handle edge case with zero cluster momentum * call redistribute after particle resampling to remove invalid particles * use unsigned ints for indexing * Revert "use unsigned ints for indexing" This reverts commit abe027f3402cf0e7c51c4599a0a50b9e8a6b817a. * call `Redistribute` before merging * code clean-up * also check for `std::isnan` in edge case handling * add reference for grid based merging * check that cluster has total weight > 0 before merging * add defense against numerical error leading to nan * make resampling message more verbose * use `deleteInvalidParticles` instead of `Redistribute` * remove default values for merging parameters * remove doc-string default specifications in picmi.py * apply suggestions from code review * update benchmark values; avoid possible nans * add assert to prevent merging of photons * add `BackwardCompatibility` check to `LevelingThinning` * implement option for Cartesian velocity mesh * use `enum` for velocity grid types * fix Windows `uint` issue and clang-tidy error * use `Reduce` functions for GPU compatibility * merge kernels finding min and max velocities * take array of du values in picmi * avoid checking velocity grid type for every particle * fix issue in picmi * use array input for velocity bin sizes --- Python/pywarpx/picmi.py | 24 +++- .../Resampling/VelocityCoincidenceThinning.H | 95 ++++++++++++++- .../VelocityCoincidenceThinning.cpp | 113 ++++++++++++------ 3 files changed, 187 insertions(+), 45 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 902d6150773..03f2ed14732 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -147,17 +147,27 @@ class Species(picmistandard.PICMI_Species): warpx_resampling_algorithm: str, default="leveling_thinning" Resampling algorithm to use. + warpx_resampling_algorithm_velocity_grid_type: str, default="spherical" + Type of grid to use when clustering particles in velocity space. Only + applicable with the `velocity_coincidence_thinning` algorithm. + warpx_resampling_algorithm_delta_ur: float Size of velocity window used for clustering particles during grid-based - merging. + merging, with `velocity_grid_type == "spherical"`. warpx_resampling_algorithm_n_theta: int Number of bins to use in theta when clustering particle velocities - during grid-based merging. + during grid-based merging, with `velocity_grid_type == "spherical"`. warpx_resampling_algorithm_n_phi: int Number of bins to use in phi when clustering particle velocities - during grid-based merging. + during grid-based merging, with `velocity_grid_type == "spherical"`. + + warpx_resampling_algorithm_delta_u: array of floats or float + Size of velocity window used in ux, uy and uz for clustering particles + during grid-based merging, with `velocity_grid_type == "cartesian"`. If + a single number is given the same du value will be used in all three + directions. """ def init(self, kw): @@ -239,9 +249,13 @@ def init(self, kw): self.resampling_min_ppc = kw.pop('warpx_resampling_min_ppc', None) self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) + self.resampling_algorithm_velocity_grid_type = kw.pop('warpx_resampling_algorithm_velocity_grid_type', None) self.resampling_algorithm_delta_ur = kw.pop('warpx_resampling_algorithm_delta_ur', None) self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) self.resampling_algorithm_n_phi = kw.pop('warpx_resampling_algorithm_n_phi', None) + self.resampling_algorithm_delta_u = kw.pop('warpx_resampling_algorithm_delta_u', None) + if np.size(self.resampling_algorithm_delta_u) == 1: + self.resampling_algorithm_delta_u = [self.resampling_algorithm_delta_u]*3 def species_initialize_inputs(self, layout, initialize_self_fields = False, @@ -284,9 +298,11 @@ def species_initialize_inputs(self, layout, resampling_min_ppc=self.resampling_min_ppc, resampling_trigger_intervals=self.resampling_trigger_intervals, resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc, + resampling_algorithm_velocity_grid_type=self.resampling_algorithm_velocity_grid_type, resampling_algorithm_delta_ur=self.resampling_algorithm_delta_ur, resampling_algorithm_n_theta=self.resampling_algorithm_n_theta, - resampling_algorithm_n_phi=self.resampling_algorithm_n_phi) + resampling_algorithm_n_phi=self.resampling_algorithm_n_phi, + resampling_algorithm_delta_u=self.resampling_algorithm_delta_u) # add reflection models self.species.add_new_attr("reflection_model_xlo(E)", self.reflection_model_xlo) diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.H b/Source/Particles/Resampling/VelocityCoincidenceThinning.H index 9462f4b35ed..b8d67d198a4 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.H +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.H @@ -9,8 +9,10 @@ #ifndef WARPX_VELOCITY_COINCIDENCE_THINNING_H_ #define WARPX_VELOCITY_COINCIDENCE_THINNING_H_ +#include "Particles/Algorithms/KineticEnergy.H" #include "Resampling.H" - +#include "Utils/Parser/ParserUtils.H" +#include "Utils/ParticleUtils.H" /** * \brief This class implements a particle merging scheme wherein particles @@ -33,6 +35,11 @@ public: */ VelocityCoincidenceThinning (const std::string& species_name); + enum struct VelocityGridType { + Spherical = 0, + Cartesian = 1 + }; + /** * \brief A method that performs merging for the considered species. * @@ -106,10 +113,92 @@ public: } }; + /** + * \brief Struct used to assign velocity space bin numbers to a given set + * of particles. + */ + struct VelocityBinCalculator { + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void labelOnSphericalVelocityGrid (const amrex::ParticleReal ux[], + const amrex::ParticleReal uy[], + const amrex::ParticleReal uz[], + const unsigned int indices[], + int bin_array[], int index_array[], + const int cell_start, const int cell_stop ) const + { + for (int i = cell_start; i < cell_stop; ++i) + { + // get polar components of the velocity vector + auto u_mag = std::sqrt( + ux[indices[i]]*ux[indices[i]] + + uy[indices[i]]*uy[indices[i]] + + uz[indices[i]]*uz[indices[i]] + ); + auto u_theta = std::atan2(uy[indices[i]], ux[indices[i]]) + MathConst::pi; + auto u_phi = std::acos(uz[indices[i]]/u_mag); + + const int ii = static_cast(u_theta / dutheta); + const int jj = static_cast(u_phi / duphi); + const int kk = static_cast(u_mag / dur); + + bin_array[i] = ii + jj * n1 + kk * n1 * n2; + index_array[i] = i; + } + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void labelOnCartesianVelocityGrid (const amrex::ParticleReal ux[], + const amrex::ParticleReal uy[], + const amrex::ParticleReal uz[], + const unsigned int indices[], + int bin_array[], int index_array[], + const int cell_start, const int cell_stop ) const + { + for (int i = cell_start; i < cell_stop; ++i) + { + const int ii = static_cast((ux[indices[i]] - ux_min) / dux); + const int jj = static_cast((uy[indices[i]] - uy_min) / duy); + const int kk = static_cast((uz[indices[i]] - uz_min) / duz); + + bin_array[i] = ii + jj * n1 + kk * n1 * n2; + index_array[i] = i; + } + } + + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + void operator() (const amrex::ParticleReal ux[], const amrex::ParticleReal uy[], + const amrex::ParticleReal uz[], const unsigned int indices[], + int bin_array[], int index_array[], + const int cell_start, const int cell_stop) const + { + if (velocity_grid_type == VelocityGridType::Spherical) { + labelOnSphericalVelocityGrid( + ux, uy, uz, indices, bin_array, index_array, cell_start, + cell_stop + ); + } + else if (velocity_grid_type == VelocityGridType::Cartesian) { + labelOnCartesianVelocityGrid( + ux, uy, uz, indices, bin_array, index_array, cell_start, + cell_stop + ); + } + } + + VelocityGridType velocity_grid_type; + int n1, n2; + amrex::ParticleReal dur, dutheta, duphi; + amrex::ParticleReal dux, duy, duz; + amrex::ParticleReal ux_min, uy_min, uz_min, ux_max, uy_max; + }; + private: + VelocityGridType m_velocity_grid_type; + int m_min_ppc = 1; - int m_ntheta; - int m_nphi; + int m_ntheta, m_nphi; amrex::ParticleReal m_delta_ur; + amrex::Vector m_delta_u; }; #endif // WARPX_VELOCITY_COINCIDENCE_THINNING_H_ diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp index 66176c48b4d..db4ddb801c6 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp @@ -9,10 +9,6 @@ #include "VelocityCoincidenceThinning.H" -#include "Particles/Algorithms/KineticEnergy.H" -#include "Utils/Parser/ParserUtils.H" -#include "Utils/ParticleUtils.H" - VelocityCoincidenceThinning::VelocityCoincidenceThinning (const std::string& species_name) { @@ -28,15 +24,33 @@ VelocityCoincidenceThinning::VelocityCoincidenceThinning (const std::string& spe "Resampling min_ppc should be greater than or equal to 1" ); - utils::parser::getWithParser( - pp_species_name, "resampling_algorithm_delta_ur", m_delta_ur - ); - utils::parser::getWithParser( - pp_species_name, "resampling_algorithm_n_theta", m_ntheta - ); - utils::parser::getWithParser( - pp_species_name, "resampling_algorithm_n_phi", m_nphi + std::string velocity_grid_type_str = "spherical"; + pp_species_name.query( + "resampling_algorithm_velocity_grid_type", velocity_grid_type_str ); + if (velocity_grid_type_str == "spherical") { + m_velocity_grid_type = VelocityGridType::Spherical; + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_delta_ur", m_delta_ur + ); + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_n_theta", m_ntheta + ); + utils::parser::getWithParser( + pp_species_name, "resampling_algorithm_n_phi", m_nphi + ); + } + else if (velocity_grid_type_str == "cartesian") { + m_velocity_grid_type = VelocityGridType::Cartesian; + utils::parser::getArrWithParser( + pp_species_name, "resampling_algorithm_delta_u", m_delta_u + ); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_delta_u.size() == 3, + "resampling_algorithm_delta_u must have three components."); + } + else { + WARPX_ABORT_WITH_MESSAGE("Unkown velocity grid type."); + } } void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, @@ -45,6 +59,7 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, using namespace amrex::literals; auto& ptile = pc->ParticlesAt(lev, pti); + const auto n_parts_in_tile = pti.numParticles(); auto& soa = ptile.GetStructOfArrays(); #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) auto * const AMREX_RESTRICT x = soa.GetRealData(PIdx::x).data(); @@ -79,21 +94,57 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, ); // create a GPU vector to hold the momentum cluster index for each particle - amrex::Gpu::DeviceVector momentum_bin_number(bins.numItems()); + amrex::Gpu::DeviceVector momentum_bin_number(n_parts_in_tile); auto* momentum_bin_number_data = momentum_bin_number.data(); // create a GPU vector to hold the index sorting for the momentum bins - amrex::Gpu::DeviceVector sorted_indices(bins.numItems()); + amrex::Gpu::DeviceVector sorted_indices(n_parts_in_tile); auto* sorted_indices_data = sorted_indices.data(); - const auto Ntheta = m_ntheta; - const auto Nphi = m_nphi; - - const auto dr = m_delta_ur; - const auto dtheta = 2.0_prt * MathConst::pi / Ntheta; - const auto dphi = MathConst::pi / Nphi; constexpr auto c2 = PhysConst::c * PhysConst::c; + auto velocityBinCalculator = VelocityBinCalculator(); + velocityBinCalculator.velocity_grid_type = m_velocity_grid_type; + if (m_velocity_grid_type == VelocityGridType::Spherical) { + velocityBinCalculator.dur = m_delta_ur; + velocityBinCalculator.n1 = m_ntheta; + velocityBinCalculator.n2 = m_nphi; + velocityBinCalculator.dutheta = 2.0_prt * MathConst::pi / m_ntheta; + velocityBinCalculator.duphi = MathConst::pi / m_nphi; + } + else if (m_velocity_grid_type == VelocityGridType::Cartesian) { + velocityBinCalculator.dux = m_delta_u[0]; + velocityBinCalculator.duy = m_delta_u[1]; + velocityBinCalculator.duz = m_delta_u[2]; + + // get the minimum and maximum velocities to determine the velocity space + // grid boundaries + { + using ReduceOpsT = amrex::TypeMultiplier; + using ReduceDataT = amrex::TypeMultiplier; + ReduceOpsT reduce_op; + ReduceDataT reduce_data(reduce_op); + using ReduceTuple = typename ReduceDataT::Type; + reduce_op.eval(n_parts_in_tile, reduce_data, [=] AMREX_GPU_DEVICE(int i) -> ReduceTuple { + return {ux[i], uy[i], uz[i], ux[i], uy[i]}; + }); + auto hv = reduce_data.value(reduce_op); + velocityBinCalculator.ux_min = amrex::get<0>(hv); + velocityBinCalculator.uy_min = amrex::get<1>(hv); + velocityBinCalculator.uz_min = amrex::get<2>(hv); + velocityBinCalculator.ux_max = amrex::get<3>(hv); + velocityBinCalculator.uy_max = amrex::get<4>(hv); + } + + velocityBinCalculator.n1 = static_cast( + std::ceil((velocityBinCalculator.ux_max - velocityBinCalculator.ux_min) / m_delta_u[0]) + ); + velocityBinCalculator.n2 = static_cast( + std::ceil((velocityBinCalculator.uy_max - velocityBinCalculator.uy_min) / m_delta_u[1]) + ); + } auto heapSort = HeapSort(); // Loop over cells @@ -114,24 +165,10 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, // Loop over particles and label them with the appropriate momentum bin // number. Also assign initial ordering to the sorted_indices array. - for (int i = cell_start; i < cell_stop; ++i) - { - // get polar components of the velocity vector - auto u_mag = std::sqrt( - ux[indices[i]]*ux[indices[i]] + - uy[indices[i]]*uy[indices[i]] + - uz[indices[i]]*uz[indices[i]] - ); - auto u_theta = std::atan2(uy[indices[i]], ux[indices[i]]) + MathConst::pi; - auto u_phi = std::acos(uz[indices[i]]/u_mag); - - const auto ii = static_cast(u_theta / dtheta); - const auto jj = static_cast(u_phi / dphi); - const auto kk = static_cast(u_mag / dr); - - momentum_bin_number_data[i] = ii + jj * Ntheta + kk * Ntheta * Nphi; - sorted_indices_data[i] = i; - } + velocityBinCalculator( + ux, uy, uz, indices, momentum_bin_number_data, sorted_indices_data, + cell_start, cell_stop + ); // sort indices based on comparing values in momentum_bin_number heapSort(sorted_indices_data, momentum_bin_number_data, cell_start, cell_numparts); From bc5d0a2cb39b375ac23ec99146bd241bdef62f7f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 3 May 2024 10:09:08 -0700 Subject: [PATCH 082/190] Fix Python installation instructions on Perlmutter (#4907) --- Docs/source/install/hpc/perlmutter.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst index e9ebcd4e1de..14972566abc 100644 --- a/Docs/source/install/hpc/perlmutter.rst +++ b/Docs/source/install/hpc/perlmutter.rst @@ -76,7 +76,7 @@ On Perlmutter, you can run either on GPU nodes with fast A100 GPUs (recommended) .. code-block:: bash bash $HOME/src/warpx/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh - source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx/bin/activate + source ${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/venvs/warpx-gpu/bin/activate .. dropdown:: Script Details :color: light @@ -126,7 +126,7 @@ On Perlmutter, you can run either on GPU nodes with fast A100 GPUs (recommended) .. code-block:: bash bash $HOME/src/warpx/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh - source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx/bin/activate + source ${CFS}/${proj}/${USER}/sw/perlmutter/cpu/venvs/warpx-cpu/bin/activate .. dropdown:: Script Details :color: light From 3f413a3ee4e777780feefe619cb1dd49adf788f0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 6 May 2024 22:30:21 -0700 Subject: [PATCH 083/190] Docs: Fix PASC Paper Formatting --- Docs/source/highlights.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 91c99ee50ca..c94662d531f 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -27,7 +27,7 @@ Scientific works in laser-plasma and beam-plasma acceleration. #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint __` + `preprint `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. @@ -104,7 +104,7 @@ Scientific works in particle and beam modeling. #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint __` + `preprint `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. From 0e2fc85fd0de1e5357d84c362dd0f275b7278f50 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 7 May 2024 10:17:24 -0700 Subject: [PATCH 084/190] Highlights: Fix Formatting Issue Render issue in RST in PASC24 entry. --- Docs/source/highlights.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index c94662d531f..b243f62cd97 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -25,9 +25,9 @@ Scientific works in laser-plasma and beam-plasma acceleration. `DOI:10.1103/PhysRevResearch.5.033112 `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. - **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint `__ + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. + `preprint `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. @@ -102,9 +102,9 @@ Particle Accelerator & Beam Physics Scientific works in particle and beam modeling. #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. - **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint `__ + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. + `preprint `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. From 4a6ccdce898889fbff2c4df36cd7d270b371953f Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 9 May 2024 01:17:17 +0200 Subject: [PATCH 085/190] make do_current_centering a regular member variable of the WarpX class (#4901) --- Source/Evolve/WarpXEvolve.cpp | 4 ++-- Source/Parallelization/WarpXComm.cpp | 4 ++-- Source/WarpX.H | 2 +- Source/WarpX.cpp | 2 -- 4 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 51fcb5c262c..de586895c2d 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -645,7 +645,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // (dt[0] denotes the time step on mesh refinement level 0) if (J_in_time == JInTime::Linear) { - auto& current = (WarpX::do_current_centering) ? current_fp_nodal : current_fp; + auto& current = (do_current_centering) ? current_fp_nodal : current_fp; mypc->DepositCurrent(current, dt[0], -dt[0]); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', @@ -679,7 +679,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) // Deposit new J at relative time t_deposit_current with time step dt // (dt[0] denotes the time step on mesh refinement level 0) - auto& current = (WarpX::do_current_centering) ? current_fp_nodal : current_fp; + auto& current = (do_current_centering) ? current_fp_nodal : current_fp; mypc->DepositCurrent(current, dt[0], t_deposit_current); // Synchronize J: filter, exchange boundary, and interpolate across levels. // With current centering, the nodal current is deposited in 'current', diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 9daf37af3f8..f3bff69546b 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -882,7 +882,7 @@ WarpX::SyncCurrent ( WARPX_PROFILE("WarpX::SyncCurrent()"); // If warpx.do_current_centering = 1, center currents from nodal grid to staggered grid - if (WarpX::do_current_centering) + if (do_current_centering) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(finest_level <= 1, "warpx.do_current_centering=1 not supported with more than one fine levels"); @@ -1178,7 +1178,7 @@ void WarpX::SumBoundaryJ ( const amrex::IntVect ng = J.nGrowVect(); amrex::IntVect ng_depos_J = get_ng_depos_J(); - if (WarpX::do_current_centering) + if (do_current_centering) { #if defined(WARPX_DIM_1D_Z) ng_depos_J[0] += WarpX::current_centering_noz / 2; diff --git a/Source/WarpX.H b/Source/WarpX.H index d19843ca636..6c6538325e6 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -211,7 +211,7 @@ public: //! If true, the current is deposited on a nodal grid and then centered onto a staggered grid //! using finite centering of order given by #current_centering_nox, #current_centering_noy, //! and #current_centering_noz - static bool do_current_centering; + bool do_current_centering = false; //! If true, a correction is applied to the current in Fourier space, // to satisfy the continuity equation and charge conservation diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 9a29d7c2354..8c3a6f15b1c 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -143,8 +143,6 @@ amrex::Vector WarpX::field_boundary_hi(AMREX_SPACEDIM,FieldBo amrex::Vector WarpX::particle_boundary_lo(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); amrex::Vector WarpX::particle_boundary_hi(AMREX_SPACEDIM,ParticleBoundaryType::Absorbing); -bool WarpX::do_current_centering = false; - int WarpX::n_rz_azimuthal_modes = 1; int WarpX::ncomps = 1; From 98bd725860fad3d69cc21a6f9d60548ef9fe43ca Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 9 May 2024 21:38:16 -0700 Subject: [PATCH 086/190] Add CI test for velocity coincidence thinning (#4919) * add CI test for Cartesian velocity grid in resampling * avoid unneeded picmi input parameters * set checksum values for new test * use seperate input file for new example --- ...1d_velocity_coincidence_thinning_cartesian | 45 +++++++++++++++++++ Python/pywarpx/picmi.py | 2 +- ...locity_coincidence_thinning_cartesian.json | 20 +++++++++ Regression/WarpX-tests.ini | 17 +++++++ 4 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian create mode 100644 Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json diff --git a/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian new file mode 100644 index 00000000000..97830d8bb9e --- /dev/null +++ b/Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian @@ -0,0 +1,45 @@ +max_step = 4 +warpx.verbose = 1 +warpx.const_dt = 1e-10 +amr.n_cell = 256 +amr.max_grid_size = 64 +amr.max_level = 0 +geometry.dims = 1 +geometry.prob_lo = 0 +geometry.prob_hi = 0.1 + +# Boundary condition and field solver +boundary.field_lo = periodic +boundary.field_hi = periodic +boundary.particle_lo = periodic +boundary.particle_hi = periodic +algo.particle_shape = 1 +algo.maxwell_solver = none + +particles.species_names = hydrogen +hydrogen.mass = 1.67262192369e-27 +hydrogen.charge = 1.602176634e-19 +hydrogen.injection_style = nrandompercell +hydrogen.initialize_self_fields = 0 +hydrogen.do_not_push = 1 +hydrogen.do_resampling = 1 +hydrogen.resampling_min_ppc = 10 +hydrogen.resampling_trigger_intervals = 1::2 +hydrogen.resampling_algorithm = velocity_coincidence_thinning +hydrogen.resampling_algorithm_velocity_grid_type = cartesian +hydrogen.resampling_algorithm_delta_u = 10000000.0 5000000.0 7000000.0 +hydrogen.num_particles_per_cell = 2500 +hydrogen.momentum_distribution_type = gaussian +hydrogen.ux_m = 0.0 +hydrogen.uy_m = 0.0 +hydrogen.uz_m = 0.0 +hydrogen.ux_th = 0.000326 +hydrogen.uy_th = 0.000326 +hydrogen.uz_th = 0.000326 +hydrogen.profile = constant +hydrogen.density = 1e+19 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 4 +diag1.diag_type = Full diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 03f2ed14732..e7575f83b26 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -254,7 +254,7 @@ def init(self, kw): self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) self.resampling_algorithm_n_phi = kw.pop('warpx_resampling_algorithm_n_phi', None) self.resampling_algorithm_delta_u = kw.pop('warpx_resampling_algorithm_delta_u', None) - if np.size(self.resampling_algorithm_delta_u) == 1: + if self.resampling_algorithm_delta_u is not None and np.size(self.resampling_algorithm_delta_u) == 1: self.resampling_algorithm_delta_u = [self.resampling_algorithm_delta_u]*3 def species_initialize_inputs(self, layout, diff --git a/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json new file mode 100644 index 00000000000..9f1dae16e7f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/resample_velocity_coincidence_thinning_cartesian.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 434240.88525516074, + "jy": 315768.9402331213, + "jz": 642846.5589763735 + }, + "hydrogen": { + "particle_momentum_x": 6.919253235905925e-20, + "particle_momentum_y": 7.069867215063566e-20, + "particle_momentum_z": 7.606884436016626e-20, + "particle_position_x": 25.60006676227001, + "particle_weight": 1.0000000000000001e+18 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index f8859ef7df5..edbd97b0acb 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4206,6 +4206,23 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py +[resample_velocity_coincidence_thinning_cartesian] +buildDir = . +inputFile = Examples/Tests/resampling/inputs_1d_velocity_coincidence_thinning_cartesian +runtime_params = +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/analysis_default_regression.py + [restart] buildDir = . inputFile = Examples/Tests/restart/inputs From fee6c4e204add92d3676b715343943ec7a694a9b Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 10 May 2024 10:09:24 -0700 Subject: [PATCH 087/190] Amend Notices for Non-Endorsement (#4924) Amend and move NOTICES to NOTICE.txt. Appended Text: Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or otherwise does not necessarily constitute or imply its endorsement, recommendation, or favoring by the United States Government, The Regents of the University of California, or Lawrence Berkeley National Laboratory. The views and opinions of authors expressed herein do not necessarily state or reflect those of the United States Government, The Regents of the University of California, or Lawrence Berkeley National Laboratory, and shall not be used for advertising or product endorsement purposes. --- NOTICE.txt | 19 +++++++++++++++++++ README.md | 9 +-------- 2 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 NOTICE.txt diff --git a/NOTICE.txt b/NOTICE.txt new file mode 100644 index 00000000000..e556dddec6c --- /dev/null +++ b/NOTICE.txt @@ -0,0 +1,19 @@ +This Software was developed under funding from the U.S. Department +of Energy and the U.S. Government consequently retains certain rights. As +such, the U.S. Government has been granted for itself and others acting on +its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the +Software to reproduce, distribute copies to the public, prepare derivative +works, and perform publicly and display publicly, and to permit other to do +so. + +Reference herein to any specific commercial product, process, or service +by trade name, trademark, manufacturer, or otherwise does not necessarily +constitute or imply its endorsement, recommendation, or favoring by the +United States Government, The Regents of the University of California, or +Lawrence Berkeley National Laboratory. + +The views and opinions of authors expressed herein do not necessarily +state or reflect those of the United States Government, +The Regents of the University of California, or +Lawrence Berkeley National Laboratory, and shall not be used for advertising +or product endorsement purposes. diff --git a/README.md b/README.md index 87cf2b012c0..b2de2211b60 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,6 @@ If you have questions about your rights to use or distribute this software, please contact Berkeley Lab's Innovation & Partnerships Office at IPO@lbl.gov. -NOTICE. This Software was developed under funding from the U.S. Department -of Energy and the U.S. Government consequently retains certain rights. As -such, the U.S. Government has been granted for itself and others acting on -its behalf a paid-up, nonexclusive, irrevocable, worldwide license in the -Software to reproduce, distribute copies to the public, prepare derivative -works, and perform publicly and display publicly, and to permit other to do -so. - Please see the full license agreement in [LICENSE.txt](LICENSE.txt). +Please see the notices in [NOTICE.txt](NOTICE.txt). The SPDX license identifier is `BSD-3-Clause-LBNL`. From e6c1eb5289f39e34fc4093074d0439f7fa395abf Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 12 May 2024 12:16:45 -0700 Subject: [PATCH 088/190] Diagnostics: Fix Position Selection (#4914) * Diagnostics: Warn w/o Positions If a user specifies `..variables` without including particle position arguments we now warn. * openPMD: Skip 0-positions for deselected comps * Clang-Tidy: Braces * Update Examples * RZ: x,y,z * Fix: openPMD Low-D position/positionOffset Continue to write defaults for coarse and fine position of omitted dimensions to zero. --- Docs/source/usage/parameters.rst | 5 +- .../laser_acceleration/inputs_rz | 4 +- .../uniform_plasma/inputs_3d | 2 +- Examples/Tests/Implicit/inputs_1d | 4 +- .../Tests/Implicit/inputs_1d_semiimplicit | 4 +- Examples/Tests/Implicit/inputs_vandb_2d | 4 +- Examples/Tests/electrostatic_sphere/inputs_rz | 2 +- Examples/Tests/langmuir/inputs_3d | 4 +- Examples/Tests/photon_pusher/inputs_3d | 32 ++++---- Examples/Tests/plasma_lens/inputs_3d | 2 +- Examples/Tests/plasma_lens/inputs_boosted_3d | 2 +- Examples/Tests/plasma_lens/inputs_lattice_3d | 2 +- Examples/Tests/plasma_lens/inputs_short_3d | 2 +- Examples/Tests/qed/breit_wheeler/inputs_2d | 28 +++---- Examples/Tests/qed/breit_wheeler/inputs_3d | 28 +++---- .../Tests/qed/quantum_synchrotron/inputs_2d | 18 ++--- .../Tests/qed/quantum_synchrotron/inputs_3d | 18 ++--- Regression/WarpX-tests.ini | 40 +++++----- .../Diagnostics/ParticleDiag/ParticleDiag.cpp | 25 ++++++- Source/Diagnostics/WarpXOpenPMD.H | 6 +- Source/Diagnostics/WarpXOpenPMD.cpp | 73 +++++++++++-------- 21 files changed, 169 insertions(+), 136 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 5f97730e880..40c16a06df5 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2689,10 +2689,11 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``..variables`` (list of `strings` separated by spaces, optional) List of particle quantities to write to output. - Choices are ``w`` for the particle weight and ``ux`` ``uy`` ``uz`` for the particle momenta. + Choices are ``x``, ``y``, ``z`` for the particle positions (3D and RZ), ``x`` & ``z`` in 2D, ``z`` in 1D, + ``w`` for the particle weight and ``ux``, ``uy``, ``uz`` for the particle momenta. When using the lab-frame electrostatic solver, ``phi`` (electrostatic potential, on the macroparticles) is also available. By default, all particle quantities (except ``phi``) are written. - If ``..variables = none``, no particle data are written, except for particle positions, which are always included. + If ``..variables = none``, no particle data are written. * ``..random_fraction`` (`float`) optional If provided ``..random_fraction = a``, only `a` fraction of the particle data of this species will be dumped randomly in diag ````, i.e. if `rand() < a`, this particle will be dumped, where `rand()` denotes a random number generator. diff --git a/Examples/Physics_applications/laser_acceleration/inputs_rz b/Examples/Physics_applications/laser_acceleration/inputs_rz index 7365523b574..b61ed6f83ec 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_rz +++ b/Examples/Physics_applications/laser_acceleration/inputs_rz @@ -89,5 +89,5 @@ diagnostics.diags_names = diag1 diag1.intervals = 10 diag1.diag_type = Full diag1.fields_to_plot = Er Et Ez Br Bt Bz jr jt jz rho -diag1.electrons.variables = w ux uy uz orig_x orig_z -diag1.beam.variables = w ux uy uz +diag1.electrons.variables = x y z w ux uy uz orig_x orig_z +diag1.beam.variables = x y z w ux uy uz diff --git a/Examples/Physics_applications/uniform_plasma/inputs_3d b/Examples/Physics_applications/uniform_plasma/inputs_3d index b0275f6eed6..83310088176 100644 --- a/Examples/Physics_applications/uniform_plasma/inputs_3d +++ b/Examples/Physics_applications/uniform_plasma/inputs_3d @@ -45,7 +45,7 @@ electrons.uz_th = 0.01 # uth the std of the (unitless) momentum diagnostics.diags_names = diag1 chk diag1.intervals = 4 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz w +diag1.electrons.variables = x y z ux uy uz w diag1.fields_to_plot = Bx By Bz Ex Ey Ez jx jy jz rho chk.intervals = 6 diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d index 50d28a2db75..465d9dd6965 100644 --- a/Examples/Tests/Implicit/inputs_1d +++ b/Examples/Tests/Implicit/inputs_1d @@ -71,8 +71,8 @@ diagnostics.diags_names = diag1 diag1.intervals = 100 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = w ux uy uz -diag1.protons.variables = w ux uy uz +diag1.electrons.variables = z w ux uy uz +diag1.protons.variables = z w ux uy uz warpx.reduced_diags_names = particle_energy field_energy particle_energy.type = ParticleEnergy diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit index 2271a0bb1bc..4008a559588 100644 --- a/Examples/Tests/Implicit/inputs_1d_semiimplicit +++ b/Examples/Tests/Implicit/inputs_1d_semiimplicit @@ -71,8 +71,8 @@ diagnostics.diags_names = diag1 diag1.intervals = 100 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = w ux uy uz -diag1.protons.variables = w ux uy uz +diag1.electrons.variables = z w ux uy uz +diag1.protons.variables = z w ux uy uz warpx.reduced_diags_names = particle_energy field_energy particle_energy.type = ParticleEnergy diff --git a/Examples/Tests/Implicit/inputs_vandb_2d b/Examples/Tests/Implicit/inputs_vandb_2d index 2dc57323efe..33ce964710a 100644 --- a/Examples/Tests/Implicit/inputs_vandb_2d +++ b/Examples/Tests/Implicit/inputs_vandb_2d @@ -82,8 +82,8 @@ diagnostics.diags_names = diag1 diag1.intervals = 20 diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz rho divE -diag1.electrons.variables = w ux uy uz -diag1.protons.variables = w ux uy uz +diag1.electrons.variables = x z w ux uy uz +diag1.protons.variables = x z w ux uy uz warpx.reduced_diags_names = particle_energy field_energy particle_energy.type = ParticleEnergy diff --git a/Examples/Tests/electrostatic_sphere/inputs_rz b/Examples/Tests/electrostatic_sphere/inputs_rz index 26d8b0c7afc..2b6151e6d8c 100644 --- a/Examples/Tests/electrostatic_sphere/inputs_rz +++ b/Examples/Tests/electrostatic_sphere/inputs_rz @@ -39,5 +39,5 @@ diag1.fields_to_plot = Er Et Ez rho diag2.intervals = 30 diag2.diag_type = Full diag2.fields_to_plot = none -diag2.electron.variables = ux uy uz w phi +diag2.electron.variables = x y z ux uy uz w phi diag2.format = openpmd diff --git a/Examples/Tests/langmuir/inputs_3d b/Examples/Tests/langmuir/inputs_3d index 432b4788171..604ed9bc514 100644 --- a/Examples/Tests/langmuir/inputs_3d +++ b/Examples/Tests/langmuir/inputs_3d @@ -93,5 +93,5 @@ diagnostics.diags_names = diag1 diag1.intervals = max_step diag1.diag_type = Full diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho -diag1.electrons.variables = w ux -diag1.positrons.variables = uz +diag1.electrons.variables = x y z w ux +diag1.positrons.variables = x y z uz diff --git a/Examples/Tests/photon_pusher/inputs_3d b/Examples/Tests/photon_pusher/inputs_3d index e2c75981e0e..3977a187e1b 100644 --- a/Examples/Tests/photon_pusher/inputs_3d +++ b/Examples/Tests/photon_pusher/inputs_3d @@ -140,19 +140,19 @@ p_dn_10.single_particle_weight = 1.0 diagnostics.diags_names = diag1 diag1.intervals = 50 diag1.diag_type = Full -diag1.p_xp_1.variables = ux uy uz -diag1.p_xn_1.variables = ux uy uz -diag1.p_yp_1.variables = ux uy uz -diag1.p_yn_1.variables = ux uy uz -diag1.p_zp_1.variables = ux uy uz -diag1.p_zn_1.variables = ux uy uz -diag1.p_dp_1.variables = ux uy uz -diag1.p_dn_1.variables = ux uy uz -diag1.p_xp_10.variables = ux uy uz -diag1.p_xn_10.variables = ux uy uz -diag1.p_yp_10.variables = ux uy uz -diag1.p_yn_10.variables = ux uy uz -diag1.p_zp_10.variables = ux uy uz -diag1.p_zn_10.variables = ux uy uz -diag1.p_dp_10.variables = ux uy uz -diag1.p_dn_10.variables = ux uy uz +diag1.p_xp_1.variables = x y z ux uy uz +diag1.p_xn_1.variables = x y z ux uy uz +diag1.p_yp_1.variables = x y z ux uy uz +diag1.p_yn_1.variables = x y z ux uy uz +diag1.p_zp_1.variables = x y z ux uy uz +diag1.p_zn_1.variables = x y z ux uy uz +diag1.p_dp_1.variables = x y z ux uy uz +diag1.p_dn_1.variables = x y z ux uy uz +diag1.p_xp_10.variables = x y z ux uy uz +diag1.p_xn_10.variables = x y z ux uy uz +diag1.p_yp_10.variables = x y z ux uy uz +diag1.p_yn_10.variables = x y z ux uy uz +diag1.p_zp_10.variables = x y z ux uy uz +diag1.p_zn_10.variables = x y z ux uy uz +diag1.p_dp_10.variables = x y z ux uy uz +diag1.p_dn_10.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_3d b/Examples/Tests/plasma_lens/inputs_3d index 7763013adb1..a87c1eb7f44 100644 --- a/Examples/Tests/plasma_lens/inputs_3d +++ b/Examples/Tests/plasma_lens/inputs_3d @@ -48,4 +48,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 diagnostics.diags_names = diag1 diag1.intervals = 84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_boosted_3d b/Examples/Tests/plasma_lens/inputs_boosted_3d index ecdf3b3baec..fa18ac439c4 100644 --- a/Examples/Tests/plasma_lens/inputs_boosted_3d +++ b/Examples/Tests/plasma_lens/inputs_boosted_3d @@ -50,4 +50,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 diagnostics.diags_names = diag1 diag1.intervals = 84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_lattice_3d b/Examples/Tests/plasma_lens/inputs_lattice_3d index a41ca8baee9..94e57de8290 100644 --- a/Examples/Tests/plasma_lens/inputs_lattice_3d +++ b/Examples/Tests/plasma_lens/inputs_lattice_3d @@ -70,4 +70,4 @@ plasmalens4.dEdx = 200000. diagnostics.diags_names = diag1 diag1.intervals = 84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/plasma_lens/inputs_short_3d b/Examples/Tests/plasma_lens/inputs_short_3d index 8fa111df3c9..5c91ab26498 100644 --- a/Examples/Tests/plasma_lens/inputs_short_3d +++ b/Examples/Tests/plasma_lens/inputs_short_3d @@ -52,4 +52,4 @@ particles.repeated_plasma_lens_strengths_B = 0.0 0.0 0.0 0.0 diagnostics.diags_names = diag1 diag1.intervals = 1 #84 diag1.diag_type = Full -diag1.electrons.variables = ux uy uz +diag1.electrons.variables = x y z ux uy uz diff --git a/Examples/Tests/qed/breit_wheeler/inputs_2d b/Examples/Tests/qed/breit_wheeler/inputs_2d index 7f507dd08d7..857b3243ac6 100644 --- a/Examples/Tests/qed/breit_wheeler/inputs_2d +++ b/Examples/Tests/qed/breit_wheeler/inputs_2d @@ -201,19 +201,19 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthBW -diag1.p2.variables = ux uy uz w opticalDepthBW -diag1.p3.variables = ux uy uz w opticalDepthBW -diag1.p4.variables = ux uy uz w opticalDepthBW - -diag1.ele1.variables = ux uy uz w opticalDepthQSR -diag1.ele2.variables = ux uy uz w opticalDepthQSR -diag1.ele3.variables = ux uy uz w opticalDepthQSR -diag1.ele4.variables = ux uy uz w opticalDepthQSR - -diag1.pos1.variables = ux uy uz w opticalDepthQSR -diag1.pos2.variables = ux uy uz w opticalDepthQSR -diag1.pos3.variables = ux uy uz w opticalDepthQSR -diag1.pos4.variables = ux uy uz w opticalDepthQSR +diag1.p1.variables = x z ux uy uz w opticalDepthBW +diag1.p2.variables = x z ux uy uz w opticalDepthBW +diag1.p3.variables = x z ux uy uz w opticalDepthBW +diag1.p4.variables = x z ux uy uz w opticalDepthBW + +diag1.ele1.variables = x z ux uy uz w opticalDepthQSR +diag1.ele2.variables = x z ux uy uz w opticalDepthQSR +diag1.ele3.variables = x z ux uy uz w opticalDepthQSR +diag1.ele4.variables = x z ux uy uz w opticalDepthQSR + +diag1.pos1.variables = x z ux uy uz w opticalDepthQSR +diag1.pos2.variables = x z ux uy uz w opticalDepthQSR +diag1.pos3.variables = x z ux uy uz w opticalDepthQSR +diag1.pos4.variables = x z ux uy uz w opticalDepthQSR diag1.format = plotfile diff --git a/Examples/Tests/qed/breit_wheeler/inputs_3d b/Examples/Tests/qed/breit_wheeler/inputs_3d index 2749a60b888..b79dfd9c030 100644 --- a/Examples/Tests/qed/breit_wheeler/inputs_3d +++ b/Examples/Tests/qed/breit_wheeler/inputs_3d @@ -201,19 +201,19 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthBW -diag1.p2.variables = ux uy uz w opticalDepthBW -diag1.p3.variables = ux uy uz w opticalDepthBW -diag1.p4.variables = ux uy uz w opticalDepthBW - -diag1.ele1.variables = ux uy uz w opticalDepthQSR -diag1.ele2.variables = ux uy uz w opticalDepthQSR -diag1.ele3.variables = ux uy uz w opticalDepthQSR -diag1.ele4.variables = ux uy uz w opticalDepthQSR - -diag1.pos1.variables = ux uy uz w opticalDepthQSR -diag1.pos2.variables = ux uy uz w opticalDepthQSR -diag1.pos3.variables = ux uy uz w opticalDepthQSR -diag1.pos4.variables = ux uy uz w opticalDepthQSR +diag1.p1.variables = x y z ux uy uz w opticalDepthBW +diag1.p2.variables = x y z ux uy uz w opticalDepthBW +diag1.p3.variables = x y z ux uy uz w opticalDepthBW +diag1.p4.variables = x y z ux uy uz w opticalDepthBW + +diag1.ele1.variables = x y z ux uy uz w opticalDepthQSR +diag1.ele2.variables = x y z ux uy uz w opticalDepthQSR +diag1.ele3.variables = x y z ux uy uz w opticalDepthQSR +diag1.ele4.variables = x y z ux uy uz w opticalDepthQSR + +diag1.pos1.variables = x y z ux uy uz w opticalDepthQSR +diag1.pos2.variables = x y z ux uy uz w opticalDepthQSR +diag1.pos3.variables = x y z ux uy uz w opticalDepthQSR +diag1.pos4.variables = x y z ux uy uz w opticalDepthQSR diag1.format = plotfile diff --git a/Examples/Tests/qed/quantum_synchrotron/inputs_2d b/Examples/Tests/qed/quantum_synchrotron/inputs_2d index 2fc00231280..2ac2c782ccd 100644 --- a/Examples/Tests/qed/quantum_synchrotron/inputs_2d +++ b/Examples/Tests/qed/quantum_synchrotron/inputs_2d @@ -178,12 +178,12 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthQSR -diag1.p2.variables = ux uy uz w opticalDepthQSR -diag1.p3.variables = ux uy uz w opticalDepthQSR -diag1.p4.variables = ux uy uz w opticalDepthQSR - -diag1.qsp_1.variables = ux uy uz w opticalDepthBW -diag1.qsp_2.variables = ux uy uz w opticalDepthBW -diag1.qsp_3.variables = ux uy uz w opticalDepthBW -diag1.qsp_4.variables = ux uy uz w opticalDepthBW +diag1.p1.variables = x z ux uy uz w opticalDepthQSR +diag1.p2.variables = x z ux uy uz w opticalDepthQSR +diag1.p3.variables = x z ux uy uz w opticalDepthQSR +diag1.p4.variables = x z ux uy uz w opticalDepthQSR + +diag1.qsp_1.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_2.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_3.variables = x z ux uy uz w opticalDepthBW +diag1.qsp_4.variables = x z ux uy uz w opticalDepthBW diff --git a/Examples/Tests/qed/quantum_synchrotron/inputs_3d b/Examples/Tests/qed/quantum_synchrotron/inputs_3d index 9bc963f7e94..429666ef938 100644 --- a/Examples/Tests/qed/quantum_synchrotron/inputs_3d +++ b/Examples/Tests/qed/quantum_synchrotron/inputs_3d @@ -178,12 +178,12 @@ diagnostics.diags_names = diag1 diag1.intervals = 2 diag1.diag_type = Full diag1.fields_to_plot = Ex -diag1.p1.variables = ux uy uz w opticalDepthQSR -diag1.p2.variables = ux uy uz w opticalDepthQSR -diag1.p3.variables = ux uy uz w opticalDepthQSR -diag1.p4.variables = ux uy uz w opticalDepthQSR - -diag1.qsp_1.variables = ux uy uz w opticalDepthBW -diag1.qsp_2.variables = ux uy uz w opticalDepthBW -diag1.qsp_3.variables = ux uy uz w opticalDepthBW -diag1.qsp_4.variables = ux uy uz w opticalDepthBW +diag1.p1.variables = x y z ux uy uz w opticalDepthQSR +diag1.p2.variables = x y z ux uy uz w opticalDepthQSR +diag1.p3.variables = x y z ux uy uz w opticalDepthQSR +diag1.p4.variables = x y z ux uy uz w opticalDepthQSR + +diag1.qsp_1.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_2.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_3.variables = x y z ux uy uz w opticalDepthBW +diag1.qsp_4.variables = x y z ux uy uz w opticalDepthBW diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index edbd97b0acb..d31a042c807 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -554,7 +554,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereLabFrame] buildDir = . inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=ux uy uz w phi +runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=x y z ux uy uz w phi dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 @@ -1189,7 +1189,7 @@ analysisOutputImage = langmuir_fluid_multi_analysis.png [Langmuir_multi_1d] buildDir = . inputFile = Examples/Tests/langmuir/inputs_1d -runtime_params = algo.current_deposition=esirkepov diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz +runtime_params = algo.current_deposition=esirkepov diag1.electrons.variables=z w ux uy uz diag1.positrons.variables=z w ux uy uz dim = 1 addToCompileString = USE_OPENPMD=TRUE QED=FALSE cmakeSetupOpts = -DWarpX_DIMS=1 -DWarpX_OPENPMD=ON -DWarpX_QED=OFF @@ -1208,7 +1208,7 @@ analysisOutputImage = langmuir_multi_1d_analysis.png [Langmuir_multi_2d_MR] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = w ux uy uz diag1.positrons.variables = w ux uy uz +runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1227,7 +1227,7 @@ analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_anisotropic] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio_vect = 4 2 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = w ux uy uz diag1.positrons.variables = w ux uy uz +runtime_params = algo.maxwell_solver = ckc warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio_vect = 4 2 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1246,7 +1246,7 @@ analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_momentum_conserving] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=ckc warpx.use_filter=1 amr.max_level=1 amr.ref_ratio=4 warpx.fine_tag_lo=-10.e-6 -10.e-6 warpx.fine_tag_hi=10.e-6 10.e-6 algo.field_gathering=momentum-conserving diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz +runtime_params = algo.maxwell_solver=ckc warpx.use_filter=1 amr.max_level=1 amr.ref_ratio=4 warpx.fine_tag_lo=-10.e-6 -10.e-6 warpx.fine_tag_hi=10.e-6 10.e-6 algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1265,7 +1265,7 @@ analysisOutputImage = Langmuir_multi_2d_MR_momentum_conserving.png [Langmuir_multi_2d_MR_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver = psatd warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = w ux uy uz diag1.positrons.variables = w ux uy uz psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver = psatd warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1284,7 +1284,7 @@ analysisOutputImage = Langmuir_multi_2d_MR_psatd.png [Langmuir_multi_2d_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz +runtime_params = warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz dim = 2 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=2 @@ -1303,7 +1303,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1322,7 +1322,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1341,7 +1341,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1360,7 +1360,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_momentum_conserving] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1417,7 +1417,7 @@ analysisOutputImage = Langmuir_multi_2d_psatd_multiJ_nodal.png [Langmuir_multi_2d_psatd_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1436,7 +1436,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1455,7 +1455,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1474,7 +1474,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d -runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 warpx.grid_type=collocated algo.current_deposition=vay diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 +runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 warpx.grid_type=collocated algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 addToCompileString = USE_PSATD=TRUE cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON @@ -1492,7 +1492,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_nodal] buildDir = . -inputFile = Examples/Tests/langmuir/inputs_3d +inputFile = runtime_params = warpx.grid_type=collocated algo.current_deposition=direct dim = 3 addToCompileString = @@ -1721,7 +1721,7 @@ analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_rz] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 +runtime_params = diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ @@ -1741,7 +1741,7 @@ aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium +runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON @@ -1761,7 +1761,7 @@ aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_current_correction] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 amr.max_grid_size=128 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot=jr jz Er Ez Bt rho divE electrons.random_theta=0 ions.random_theta=0 +runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 amr.max_grid_size=128 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot=jr jz Er Ez Bt rho divE electrons.random_theta=0 ions.random_theta=0 dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON @@ -1781,7 +1781,7 @@ aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_multiJ] buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz -runtime_params = amr.max_grid_size=32 algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.ions.variables=w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 warpx.n_rz_azimuthal_modes=2 electrons.random_theta=0 electrons.num_particles_per_cell_each_dim=2 4 2 ions.random_theta=0 ions.num_particles_per_cell_each_dim=2 4 2 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=4 warpx.use_filter=1 +runtime_params = amr.max_grid_size=32 algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 warpx.n_rz_azimuthal_modes=2 electrons.random_theta=0 electrons.num_particles_per_cell_each_dim=2 4 2 ions.random_theta=0 ions.num_particles_per_cell_each_dim=2 4 2 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=4 warpx.use_filter=1 dim = 2 addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON diff --git a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp index 7ed64ccf8ec..1a64ae20f0e 100644 --- a/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp +++ b/Source/Diagnostics/ParticleDiag/ParticleDiag.cpp @@ -6,6 +6,8 @@ #include "Utils/TextMsg.H" #include "WarpX.H" +#include + #include #include @@ -13,7 +15,7 @@ using namespace amrex; -ParticleDiag::ParticleDiag( +ParticleDiag::ParticleDiag ( const std::string& diag_name, const std::string& name, WarpXParticleContainer* pc, PinnedMemoryParticleContainer* pinned_pc): m_diag_name(diag_name), m_name(name), m_pc(pc), m_pinned_pc(pinned_pc) @@ -28,12 +30,17 @@ ParticleDiag::ParticleDiag( amrex::Vector variables; const int variables_specified = pp_diag_name_species_name.queryarr("variables", variables); - if (variables_specified){ + if (variables_specified) { // If only specific variables have been specified, fill m_plot_flags with zero and only set // requested variables to one std::fill(m_plot_flags.begin(), m_plot_flags.end(), 0); + bool contains_positions = false; if (variables[0] != "none"){ - const std::map existing_variable_names = pc->getParticleComps(); + std::map existing_variable_names = pc->getParticleComps(); +#ifdef WARPX_DIM_RZ + // we reconstruct to Cartesian x,y,z for RZ particle output + existing_variable_names["y"] = PIdx::theta; +#endif for (const auto& var : variables){ if (var == "phi") { // User requests phi on particle. This is *not* part of the variables that @@ -47,9 +54,21 @@ ParticleDiag::ParticleDiag( "variables argument '" + var +"' is not an existing attribute for this species"); m_plot_flags[existing_variable_names.at(var)] = 1; + + if (var == "x" || var == "y" || var == "z") { + contains_positions = true; + } } } } + + if (!contains_positions) { + ablastr::warn_manager::WMRecordWarning( + "Diagnostics", + diag_name + "." + name + ".variables contains no particle positions!", + ablastr::warn_manager::WarnPriority::high + ); + } } #ifdef WARPX_DIM_RZ diff --git a/Source/Diagnostics/WarpXOpenPMD.H b/Source/Diagnostics/WarpXOpenPMD.H index 12aa46de058..99d6e0682ab 100644 --- a/Source/Diagnostics/WarpXOpenPMD.H +++ b/Source/Diagnostics/WarpXOpenPMD.H @@ -218,11 +218,13 @@ private: /** This function sets up the entries for storing the particle positions and global IDs * * @param[in] currSpecies Corresponding openPMD species + * @param[in] positionComponents user-selected components of the particle position * @param[in] np Number of particles * @param[in] isBTD Is this a back-transformed diagnostics output? */ void SetupPos ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, bool isBTD = false); @@ -231,12 +233,14 @@ private: * Sets the entries for storing particle position offset, constant records (charge, mass) and ED-PIC attributes. * * @param[in] currSpecies Corresponding openPMD species + * @param[in] positionComponents user-selected components of the particle position * @param[in] np Number of particles * @param[in] charge Charge of the particles (note: fix for ions) * @param[in] mass Mass of the particles */ void SetConstParticleRecordsEDPIC ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, amrex::ParticleReal charge, amrex::ParticleReal mass); @@ -284,8 +288,8 @@ private: * @param[in] name species name * @param[in] iteration timestep * @param[in] write_real_comp The real attribute ids, from WarpX - * @param[in] real_comp_names The real attribute names, from WarpX * @param[in] write_int_comp The int attribute ids, from WarpX + * @param[in] real_comp_names The real attribute names, from WarpX * @param[in] int_comp_names The int attribute names, from WarpX * @param[in] charge Charge of the particles (note: fix for ions) * @param[in] mass Mass of the particles diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 965eb82b31e..45d5dc1cdee 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -213,32 +213,29 @@ namespace detail return make_pair(record_name, component_name); } - /** Return the component labels for particle positions + /** Return the user-selected components for particle positions + * + * @param[in] write_real_comp The real attribute ids, from WarpX + * @param[in] real_comp_names The real attribute names, from WarpX */ inline std::vector< std::string > - getParticlePositionComponentLabels (bool ignore_dims=false) + getParticlePositionComponentLabels ( + amrex::Vector const & write_real_comp, + amrex::Vector const & real_comp_names) { - using vs = std::vector< std::string >; - auto positionComponents = vs{"x", "y", "z"}; - if (!ignore_dims) { -#if defined(WARPX_DIM_1D_Z) - positionComponents = vs{"z"}; -#elif defined(WARPX_DIM_XZ) - positionComponents = vs{"x", "z"}; -#elif defined(WARPX_DIM_RZ) - // note: although we internally store particle positions - // for AMReX in r,z and a theta attribute, we - // actually need them for algorithms (e.g. push) - // and I/O in Cartesian. - // Other attributes like momentum are consequently - // stored in x,y,z internally. - positionComponents = vs{"x", "y", "z"}; -#elif defined(WARPX_DIM_3D) - positionComponents = vs{"x", "y", "z"}; -#else -# error Unknown WarpX dimensionality. -#endif + std::vector< std::string > positionComponents; + + int idx = 0; + for (auto const & comp : real_comp_names ) { + if (write_real_comp[idx]) { + if (comp == "position_x" || comp == "position_y" || comp == "position_z") { + std::string const last_letter{comp.back()}; + positionComponents.push_back(last_letter); + } + } + idx++; } + return positionComponents; } @@ -705,16 +702,18 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc, doParticleSetup = is_first_flush_with_particles || is_last_flush_and_never_particles; } + auto const positionComponents = detail::getParticlePositionComponentLabels(write_real_comp, real_comp_names); + // this setup stage also implicitly calls "makeEmpty" if needed (i.e., is_last_flush_and_never_particles) // for BTD, we call this multiple times as we may resize in subsequent dumps if number of particles in the buffer > 0 if (doParticleSetup || is_resizing_flush) { - SetupPos(currSpecies, NewParticleVectorSize, isBTD); + SetupPos(currSpecies, positionComponents, NewParticleVectorSize, isBTD); SetupRealProperties(pc, currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, NewParticleVectorSize, isBTD); } if (is_last_flush_to_step) { - SetConstParticleRecordsEDPIC(currSpecies, NewParticleVectorSize, charge, mass); + SetConstParticleRecordsEDPIC(currSpecies, positionComponents, NewParticleVectorSize, charge, mass); } // open files from all processors, in case some will not contribute below @@ -978,6 +977,7 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, void WarpXOpenPMDPlot::SetupPos ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, bool const isBTD) { @@ -986,7 +986,6 @@ WarpXOpenPMDPlot::SetupPos ( auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}, options); auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), {np}, options); - auto const positionComponents = detail::getParticlePositionComponentLabels(); for( auto const& comp : positionComponents ) { currSpecies["position"][comp].resetDataset( realType ); } @@ -998,6 +997,7 @@ WarpXOpenPMDPlot::SetupPos ( void WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( openPMD::ParticleSpecies& currSpecies, + std::vector const & positionComponents, const unsigned long long& np, amrex::ParticleReal const charge, amrex::ParticleReal const mass) @@ -1006,7 +1006,6 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( const auto *const scalar = openPMD::RecordComponent::SCALAR; // define record shape to be number of particles - auto const positionComponents = detail::getParticlePositionComponentLabels(true); for( auto const& comp : positionComponents ) { currSpecies["positionOffset"][comp].resetDataset( realType ); } @@ -1015,9 +1014,12 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( #if defined(WARPX_DIM_1D_Z) currSpecies["position"]["x"].resetDataset( realType ); currSpecies["position"]["y"].resetDataset( realType ); + currSpecies["positionOffset"]["x"].resetDataset( realType ); + currSpecies["positionOffset"]["y"].resetDataset( realType ); #endif #if defined(WARPX_DIM_XZ) currSpecies["position"]["y"].resetDataset( realType ); + currSpecies["positionOffset"]["y"].resetDataset( realType ); #endif // make constant @@ -1031,22 +1033,29 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC ( #if defined(WARPX_DIM_1D_Z) currSpecies["position"]["x"].makeConstant( 0._prt ); currSpecies["position"]["y"].makeConstant( 0._prt ); + currSpecies["positionOffset"]["x"].makeConstant( 0._prt ); + currSpecies["positionOffset"]["y"].makeConstant( 0._prt ); #endif #if defined(WARPX_DIM_XZ) currSpecies["position"]["y"].makeConstant( 0._prt ); + currSpecies["positionOffset"]["y"].makeConstant( 0._prt ); #endif // meta data - currSpecies["position"].setUnitDimension( detail::getUnitDimension("position") ); - currSpecies["positionOffset"].setUnitDimension( detail::getUnitDimension("positionOffset") ); + if (!positionComponents.empty()) { + currSpecies["position"].setUnitDimension( detail::getUnitDimension("position") ); + currSpecies["positionOffset"].setUnitDimension( detail::getUnitDimension("positionOffset") ); + } currSpecies["charge"].setUnitDimension( detail::getUnitDimension("charge") ); currSpecies["mass"].setUnitDimension( detail::getUnitDimension("mass") ); // meta data for ED-PIC extension - currSpecies["position"].setAttribute( "macroWeighted", 0u ); - currSpecies["position"].setAttribute( "weightingPower", 0.0 ); - currSpecies["positionOffset"].setAttribute( "macroWeighted", 0u ); - currSpecies["positionOffset"].setAttribute( "weightingPower", 0.0 ); + if (!positionComponents.empty()) { + currSpecies["position"].setAttribute( "macroWeighted", 0u ); + currSpecies["position"].setAttribute( "weightingPower", 0.0 ); + currSpecies["positionOffset"].setAttribute( "macroWeighted", 0u ); + currSpecies["positionOffset"].setAttribute( "weightingPower", 0.0 ); + } currSpecies["id"].setAttribute( "macroWeighted", 0u ); currSpecies["id"].setAttribute( "weightingPower", 0.0 ); currSpecies["charge"].setAttribute( "macroWeighted", 0u ); From 02f38975c8104347804bd133df12118e014825b7 Mon Sep 17 00:00:00 2001 From: Olga Shapoval <30510597+oshapoval@users.noreply.github.com> Date: Sun, 12 May 2024 12:18:44 -0700 Subject: [PATCH 089/190] Resetting charge buffers to zero. (#4916) * Resetting charge buffers to zero. * Cleaning up: added braces. * Added CI test ElectrostaticSphereLabFrame_MR_emass_10 (with large particle's mass). * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Reverted ES sphere input script. * Docs: Fix PASC Paper Formatting * Highlights: Fix Formatting Issue Render issue in RST in PASC24 entry. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated benchmarks for ElectrostaticSphereLabFrame_MR_emass_10. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- .../analysis_electrostatic_sphere.py | 17 ++++++++++---- ...ectrostaticSphereLabFrame_MR_emass_10.json | 23 +++++++++++++++++++ Regression/WarpX-tests.ini | 17 ++++++++++++++ Source/FieldSolver/ElectrostaticSolver.cpp | 16 +++++++++++-- 4 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json diff --git a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py index 8e05bfa0c68..e63a3f02ba8 100755 --- a/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py +++ b/Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py @@ -18,6 +18,7 @@ inverse t(r) can be solved for exactly. """ import os +import re import sys import numpy as np @@ -36,6 +37,15 @@ ds = yt.load( filename ) t_max = ds.current_time.item() # time of simulation +# Parse test name and check if particle_shape = 4 is used +emass_10 = True if re.search('emass_10', filename) else False + +if emass_10: + l2_tolerance = 0.096 + m_e = 10 +else: + l2_tolerance = 0.05 + m_e = 9.10938356e-31 #Electron mass in kg ndims = np.count_nonzero(ds.domain_dimensions > 1) if ndims == 2: @@ -60,7 +70,6 @@ # Constants eps_0 = 8.8541878128e-12 #Vacuum Permittivity in C/(V*m) -m_e = 9.10938356e-31 #Electron mass in kg q_e = -1.60217662e-19 #Electron charge in C pi = np.pi #Circular constant of the universe r_0 = 0.1 #Initial radius of sphere @@ -134,9 +143,9 @@ def calculate_error(E_axis, xmin, dx, nx): print("L2 error along y-axis = %s" %L2_error_y) print("L2 error along z-axis = %s" %L2_error_z) -assert L2_error_x < 0.05 -assert L2_error_y < 0.05 -assert L2_error_z < 0.05 +assert L2_error_x < l2_tolerance +assert L2_error_y < l2_tolerance +assert L2_error_z < l2_tolerance # Check conservation of energy def return_energies(iteration): diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json new file mode 100644 index 00000000000..024127a1bd2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json @@ -0,0 +1,23 @@ +{ + "lev=0": { + "Ex": 7.110909624093144, + "Ey": 7.110909624093145, + "Ez": 7.110909624093143, + "rho": 0.0 + }, + "lev=1": { + "Ex": 14.281015560380963, + "Ey": 14.281015560380965, + "Ez": 14.281015560380965, + "rho": 2.6092568008333786e-10 + }, + "electron": { + "particle_momentum_x": 1.80842228672388e-24, + "particle_momentum_y": 1.8084222867238806e-24, + "particle_momentum_z": 1.7598771525647628e-24, + "particle_position_x": 327.46875, + "particle_position_y": 327.46875, + "particle_position_z": 327.46875, + "particle_weight": 6212.501525878906 + } +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index d31a042c807..16382d8f338 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -465,6 +465,23 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py +[ElectrostaticSphereLabFrame_MR_emass_10] +buildDir = . +inputFile = Examples/Tests/electrostatic_sphere/inputs_3d +runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=ux uy uz w warpx.abort_on_warning_threshold=medium electron.mass = 10 amr.max_level = 1 amr.ref_ratio_vect = 2 2 2 warpx.fine_tag_lo = -0.5 -0.5 -0.5 warpx.fine_tag_hi = 0.5 0.5 0.5 max_step = 2 +dim = 3 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=3 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 0 +analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py + [ElectrostaticSphereEB] buildDir = . inputFile = Examples/Tests/electrostatic_sphere_eb/inputs_3d diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 5bb50ada278..a74beae3c91 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -195,7 +195,13 @@ WarpX::AddSpaceChargeField (WarpXParticleContainer& pc) pc.DepositCharge(rho, local, reset, apply_boundary_and_scale_volume, interpolate_across_levels); } - + for (int lev = 0; lev <= max_level; lev++) { + if (lev > 0) { + if (charge_buf[lev]) { + charge_buf[lev]->setVal(0.); + } + } + } SyncRho(rho, rho_coarse, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels // Get the particle beta vector @@ -239,7 +245,13 @@ WarpX::AddSpaceChargeFieldLabFrame () int const lev = 0; myfl->DepositCharge( lev, *rho_fp[lev] ); } - + for (int lev = 0; lev <= max_level; lev++) { + if (lev > 0) { + if (charge_buf[lev]) { + charge_buf[lev]->setVal(0.); + } + } + } SyncRho(rho_fp, rho_cp, charge_buf); // Apply filter, perform MPI exchange, interpolate across levels #ifndef WARPX_DIM_RZ for (int lev = 0; lev <= finestLevel(); lev++) { From d3069053d1eae9c44db5ae92380108eab1d4615d Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Sun, 12 May 2024 13:24:57 -0700 Subject: [PATCH 090/190] Update hip installation script for CI (#4927) It appears that "apt/debian ubuntu main" does not work anymore. We now use "apt/latest ${UBUNTU_CODENAME} main" instead. --- .github/workflows/dependencies/hip.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dependencies/hip.sh b/.github/workflows/dependencies/hip.sh index 2225e670bb0..ae897786acd 100755 --- a/.github/workflows/dependencies/hip.sh +++ b/.github/workflows/dependencies/hip.sh @@ -12,12 +12,24 @@ set -eu -o pipefail # failed files the given number of times. echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries -# Ref.: https://rocmdocs.amd.com/en/latest/Installation_Guide/Installation-Guide.html#ubuntu -wget -q -O - https://repo.radeon.com/rocm/rocm.gpg.key \ - | sudo apt-key add - -echo 'deb [arch=amd64] https://repo.radeon.com/rocm/apt/debian/ ubuntu main' \ - | sudo tee /etc/apt/sources.list.d/rocm.list +# Ref.: https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/native-install/ubuntu.html + +# Make the directory if it doesn't exist yet. +# This location is recommended by the distribution maintainers. +sudo mkdir --parents --mode=0755 /etc/apt/keyrings + +# Download the key, convert the signing-key to a full +# keyring required by apt and store in the keyring directory +wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | \ + gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null +curl -O https://repo.radeon.com/rocm/rocm.gpg.key +sudo apt-key add rocm.gpg.key + +source /etc/os-release # set UBUNTU_CODENAME: focal or jammy or ... + +echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${1-latest} ${UBUNTU_CODENAME} main" \ + | sudo tee /etc/apt/sources.list.d/rocm.list echo 'export PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/opt/rocm/profiler/bin:/opt/rocm/opencl/bin:$PATH' \ | sudo tee -a /etc/profile.d/rocm.sh From 4270ca8521cd9a48d0d2fddcb174153c4d555b83 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 12 May 2024 21:58:45 -0700 Subject: [PATCH 091/190] `ElectrostaticSphereLabFrame_MR_emass_10`: +Positions (#4930) --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 16382d8f338..0f3957a8cce 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -468,7 +468,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sph [ElectrostaticSphereLabFrame_MR_emass_10] buildDir = . inputFile = Examples/Tests/electrostatic_sphere/inputs_3d -runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=ux uy uz w warpx.abort_on_warning_threshold=medium electron.mass = 10 amr.max_level = 1 amr.ref_ratio_vect = 2 2 2 warpx.fine_tag_lo = -0.5 -0.5 -0.5 warpx.fine_tag_hi = 0.5 0.5 0.5 max_step = 2 +runtime_params = warpx.do_electrostatic=labframe diag2.electron.variables=x y z ux uy uz w warpx.abort_on_warning_threshold=medium electron.mass = 10 amr.max_level = 1 amr.ref_ratio_vect = 2 2 2 warpx.fine_tag_lo = -0.5 -0.5 -0.5 warpx.fine_tag_hi = 0.5 0.5 0.5 max_step = 2 dim = 3 addToCompileString = cmakeSetupOpts = -DWarpX_DIMS=3 From 2a3fbdacbfb7edfd279c4c92543597bf0b475dd3 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Mon, 13 May 2024 13:15:14 +0200 Subject: [PATCH 092/190] ablastr::particles::compute_weights : implement 1D and use template parameter to specify if field is nodal (#4846) * use template parameter to specify if field is nodal in compute_weights and implement function also for 1D case * fix bug * remove unused variables * use amrex::IndexType::NODE as template parameter * Add Missing Include * Doc: Remove `(default)` Co-authored-by: Axel Huebl --- Source/Diagnostics/ParticleIO.cpp | 3 +- Source/EmbeddedBoundary/ParticleScraper.H | 7 +-- Source/Particles/ParticleBoundaryBuffer.cpp | 10 ++-- Source/ablastr/particles/NodalFieldGather.H | 58 +++++++++++++-------- 4 files changed, 48 insertions(+), 30 deletions(-) diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index c4164e383b1..f1f4d426a4a 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -283,7 +283,8 @@ storePhiOnParticles ( PinnedMemoryParticleContainer& tmp, getPosition(ip, xp, yp, zp); int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + xp, yp, zp, plo, dxi, i, j, k, W); amrex::Real const phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi_grid); phi_particle_arr[ip] = phi_value; } diff --git a/Source/EmbeddedBoundary/ParticleScraper.H b/Source/EmbeddedBoundary/ParticleScraper.H index ea4d0bc5c0e..1e915b39381 100644 --- a/Source/EmbeddedBoundary/ParticleScraper.H +++ b/Source/EmbeddedBoundary/ParticleScraper.H @@ -183,15 +183,16 @@ scrapeParticlesAtEB (PC& pc, const amrex::Vector& distan int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + xp, yp, zp, plo, dxi, i, j, k, W); amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phi); if (phi_value < 0.0) { int ic, jc, kc; // Cell-centered indices - [[maybe_unused]] int nodal; amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weights - ablastr::particles::compute_weights(xp, yp, zp, plo, dxi, ic, jc, kc, Wc, nodal=0); + ablastr::particles::compute_weights( + xp, yp, zp, plo, dxi, ic, jc, kc, Wc); amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phi, dxi); DistanceToEB::normalize(normal); amrex::RealVect pos; diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index c8c683f0abf..345a0bfa592 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -98,7 +98,8 @@ struct FindEmbeddedBoundaryIntersection { amrex::Real W[AMREX_SPACEDIM][2]; amrex::ParticleReal x_temp=xp, y_temp=yp, z_temp=zp; UpdatePosition(x_temp, y_temp, z_temp, ux, uy, uz, -dt_frac*dt); - ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); amrex::Real phi_value = ablastr::particles::interp_field_nodal(i, j, k, W, phiarr); return phi_value; } ); @@ -115,11 +116,12 @@ struct FindEmbeddedBoundaryIntersection { // record the components of the normal on the destination int i, j, k; amrex::Real W[AMREX_SPACEDIM][2]; - ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); + ablastr::particles::compute_weights( + x_temp, y_temp, z_temp, plo, dxi, i, j, k, W); int ic, jc, kc; // Cell-centered indices - int nodal; amrex::Real Wc[AMREX_SPACEDIM][2]; // Cell-centered weight - ablastr::particles::compute_weights(x_temp, y_temp, z_temp, plo, dxi, ic, jc, kc, Wc, nodal=0); // nodal=0 to calculate the weights with respect to the cell-centered nodes + ablastr::particles::compute_weights( + x_temp, y_temp, z_temp, plo, dxi, ic, jc, kc, Wc); amrex::RealVect normal = DistanceToEB::interp_normal(i, j, k, W, ic, jc, kc, Wc, phiarr, dxi); DistanceToEB::normalize(normal); diff --git a/Source/ablastr/particles/NodalFieldGather.H b/Source/ablastr/particles/NodalFieldGather.H index 5a10c73ae20..f324264db7f 100644 --- a/Source/ablastr/particles/NodalFieldGather.H +++ b/Source/ablastr/particles/NodalFieldGather.H @@ -12,16 +12,19 @@ #include #include #include +#include #include #include + namespace ablastr::particles { + /** * \brief Compute weight of each surrounding node (or cell-centered nodes) in interpolating a nodal ((or a cell-centered node) field - * to the given coordinates. If nodal=1, then the calculations will be done with respect to the nodes (default). If nodal=0, then the calculations will be done with respect to the cell-centered nodal) - * - * This currently only does linear order. + * to the given coordinates. If template parameter IdxType is amrex::IndexType::CellIndex::NODE, then the calculations will be done with respect to the nodes. + * If template parameter IdxType is amrex::IndexType::CellIndex::CELL, then the calculations will be done with respect to the cell-centered nodal. + * This currently only does linear order. * * \param xp,yp,zp Particle position coordinates * \param plo Index lower bounds of domain. @@ -30,24 +33,30 @@ namespace ablastr::particles * \param W 2D array of weights to store each neighbouring node (or cell-centered node) * \param nodal Int that tells if the weights are calculated in respect to the nodes (nodal=1) of the cell-centered nodes (nodal=0) */ +template AMREX_GPU_HOST_DEVICE AMREX_INLINE void compute_weights (const amrex::ParticleReal xp, const amrex::ParticleReal yp, const amrex::ParticleReal zp, amrex::GpuArray const& plo, amrex::GpuArray const& dxi, - int& i, int& j, int& k, amrex::Real W[AMREX_SPACEDIM][2], int nodal=1) noexcept + int& i, int& j, int& k, amrex::Real W[AMREX_SPACEDIM][2]) noexcept { using namespace amrex::literals; -#if !((nodal==0)||(nodal==1)) - ABLASTR_ABORT_WITH_MESSAGE("Error: 'nodal' has to be equal to 0 or 1"); -#endif + constexpr auto half_if_cell_centered = [](){ + if constexpr (IdxType == amrex::IndexType::CellIndex::NODE){ + return 0.0_rt; + } + else{ + return 0.5_rt; + } + }(); #if (defined WARPX_DIM_3D) - const amrex::Real x = (xp - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; - const amrex::Real y = (yp - plo[1]) * dxi[1] + static_cast(nodal-1)*0.5_rt; - const amrex::Real z = (zp - plo[2]) * dxi[2] + static_cast(nodal-1)*0.5_rt; + const amrex::Real x = (xp - plo[0]) * dxi[0] - half_if_cell_centered; + const amrex::Real y = (yp - plo[1]) * dxi[1] - half_if_cell_centered; + const amrex::Real z = (zp - plo[2]) * dxi[2] - half_if_cell_centered; i = static_cast(amrex::Math::floor(x)); j = static_cast(amrex::Math::floor(y)); @@ -64,17 +73,17 @@ void compute_weights (const amrex::ParticleReal xp, #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) # if (defined WARPX_DIM_XZ) - const amrex::Real x = (xp - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; + const amrex::Real x = (xp - plo[0]) * dxi[0] - half_if_cell_centered; amrex::ignore_unused(yp); i = static_cast(amrex::Math::floor(x)); W[0][1] = x - i; # elif (defined WARPX_DIM_RZ) - const amrex::Real r = (std::sqrt(xp*xp+yp*yp) - plo[0]) * dxi[0] + static_cast(nodal-1)*0.5_rt; + const amrex::Real r = (std::sqrt(xp*xp+yp*yp) - plo[0]) * dxi[0] - half_if_cell_centered; i = static_cast(amrex::Math::floor(r)); W[0][1] = r - i; # endif - const amrex::Real z = (zp - plo[1]) * dxi[1] + static_cast(nodal-1)*0.5_rt; + const amrex::Real z = (zp - plo[1]) * dxi[1] - half_if_cell_centered; j = static_cast(amrex::Math::floor(z)); W[1][1] = z - j; @@ -83,8 +92,15 @@ void compute_weights (const amrex::ParticleReal xp, k = 0; #else - amrex::ignore_unused(xp, yp, zp, plo, dxi, i, j, k, W, nodal); - ABLASTR_ABORT_WITH_MESSAGE("Error: compute_weights not yet implemented in 1D"); + const amrex::Real z = (zp - plo[0]) * dxi[0] - half_if_cell_centered; + amrex::ignore_unused(xp, yp); + i = static_cast(amrex::Math::floor(z)); + + W[0][1] = z - i; + W[0][0] = 1.0_rt - W[0][1]; + + j = 0; + k = 0; #endif } @@ -100,8 +116,8 @@ amrex::Real interp_field_nodal (int i, int j, int k, const amrex::Real W[AMREX_SPACEDIM][2], amrex::Array4 const& scalar_field) noexcept { -#if (defined WARPX_DIM_3D) amrex::Real value = 0; +#if (defined WARPX_DIM_3D) value += scalar_field(i, j , k ) * W[0][0] * W[1][0] * W[2][0]; value += scalar_field(i+1, j , k ) * W[0][1] * W[1][0] * W[2][0]; value += scalar_field(i, j+1, k ) * W[0][0] * W[1][1] * W[2][0]; @@ -111,15 +127,13 @@ amrex::Real interp_field_nodal (int i, int j, int k, value += scalar_field(i , j+1, k+1) * W[0][0] * W[1][1] * W[2][1]; value += scalar_field(i+1, j+1, k+1) * W[0][1] * W[1][1] * W[2][1]; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::Real value = 0; value += scalar_field(i, j , k) * W[0][0] * W[1][0]; value += scalar_field(i+1, j , k) * W[0][1] * W[1][0]; value += scalar_field(i, j+1, k) * W[0][0] * W[1][1]; value += scalar_field(i+1, j+1, k) * W[0][1] * W[1][1]; #else - const amrex::Real value = 0; - amrex::ignore_unused(i, j, k, W, scalar_field); - ABLASTR_ABORT_WITH_MESSAGE("Error: interp_field not yet implemented in 1D"); + value += scalar_field(i, j , k) * W[0][0]; + value += scalar_field(i+1, j , k) * W[0][1]; #endif return value; } @@ -144,7 +158,7 @@ amrex::Real doGatherScalarFieldNodal (const amrex::ParticleReal xp, // first find the weight of surrounding nodes to use during interpolation int ii, jj, kk; amrex::Real W[AMREX_SPACEDIM][2]; - compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); + compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); return interp_field_nodal(ii, jj, kk, W, scalar_field); } @@ -172,7 +186,7 @@ doGatherVectorFieldNodal (const amrex::ParticleReal xp, // first find the weight of surrounding nodes to use during interpolation int ii, jj, kk; amrex::Real W[AMREX_SPACEDIM][2]; - compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); + compute_weights(xp, yp, zp, lo, dxi, ii, jj, kk, W); amrex::GpuArray const field_interp = { interp_field_nodal(ii, jj, kk, W, vector_field_x), From ee51424e1a875332967a9302df249cb521448fa1 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 13 May 2024 11:24:19 -0700 Subject: [PATCH 093/190] Remove redundant setVal(0) in ParticleReductionFunctor (#4926) ParticleReductionFunctor uses the routine ParticleToMesh, which by default will zero out the MultiFab passed in. This PR removes the unneeded explicit calls to setVal(0.). --- .../ComputeDiagFunctors/ParticleReductionFunctor.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp index 15473890233..34fbbee5c7c 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/ParticleReductionFunctor.cpp @@ -52,8 +52,6 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, constexpr int ng = 1; // Temporary cell-centered, multi-component MultiFab for storing particles per cell. amrex::MultiFab red_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng); - // Set value to 0, and increment the value in each cell with ppc. - red_mf.setVal(0._rt); auto& pc = warpx.GetPartContainer().GetParticleContainer(m_ispec); // Copy over member variables so they can be captured in the lambda auto map_fn = m_map_fn; @@ -99,7 +97,6 @@ ParticleReductionFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, }); if (m_do_average) { amrex::MultiFab ppc_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 1, ng); - ppc_mf.setVal(0._rt); // Add the weight for each particle -- total number of particles of this species ParticleToMesh(pc, ppc_mf, m_lev, [=] AMREX_GPU_DEVICE (const WarpXParticleContainer::SuperParticleType& p, From c70a6c59f00bb8eb34b4d8d53712fb84d1ebb506 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Mon, 13 May 2024 15:48:07 -0700 Subject: [PATCH 094/190] Code clean-up for binary collisions (#4921) * code clean-up for binary collisions * fix variable shadowing * pass particle tiles by reference --- .../BinaryCollision/BinaryCollision.H | 6 +-- .../BinaryCollision/BinaryCollisionUtils.H | 28 ++++++++++++ .../DSMC/SplitAndScatterFunc.H | 45 ++++--------------- .../BinaryCollision/ParticleCreationFunc.H | 45 +++++-------------- 4 files changed, 48 insertions(+), 76 deletions(-) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index bd6ec84dd69..b3d39a9c581 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -75,7 +75,6 @@ class BinaryCollision final using ParticleTileType = WarpXParticleContainer::ParticleTileType; using ParticleTileDataType = ParticleTileType::ParticleTileDataType; using ParticleBins = amrex::DenseBins; - using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; using index_type = ParticleBins::index_type; public: @@ -298,7 +297,6 @@ public: auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); #endif - /* The following calculations are only required when creating product particles */ @@ -431,7 +429,7 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_1, + ptile_1, ptile_1, product_species_vector, tile_products_data, m1, m1, @@ -648,7 +646,7 @@ public: // Create the new product particles and define their initial values // num_added: how many particles of each product species have been created const amrex::Vector num_added = m_copy_transform_functor(n_total_pairs, - soa_1, soa_2, + ptile_1, ptile_2, product_species_vector, tile_products_data, m1, m2, diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index 091bd41955f..08ddcd27174 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -126,6 +126,34 @@ namespace BinaryCollisionUtils{ // is calculated here. lab_to_COM_lorentz_factor = g1_star*g2_star/static_cast(g1*g2); } + + /** + * \brief Subtract given weight from particle and set its ID to invalid + * if the weight reaches zero. + */ + AMREX_GPU_HOST_DEVICE AMREX_INLINE + void remove_weight_from_colliding_particle ( + amrex::ParticleReal& weight, uint64_t& idcpu, + const amrex::ParticleReal reaction_weight ) + { + // Remove weight from given particle + amrex::Gpu::Atomic::AddNoRet(&weight, -reaction_weight); + + // If the colliding particle weight decreases to zero, remove particle by + // setting its id to invalid + if (weight <= std::numeric_limits::min()) + { +#if defined(AMREX_USE_OMP) +#pragma omp atomic write + idcpu = amrex::ParticleIdCpus::Invalid; +#else + amrex::Gpu::Atomic::Exch( + (unsigned long long *)&idcpu, + (unsigned long long)amrex::ParticleIdCpus::Invalid + ); +#endif + } + } } #endif // WARPX_BINARY_COLLISION_UTILS_H_ diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H index 86a2c70374e..a0c6ca8c8d4 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/SplitAndScatterFunc.H @@ -27,8 +27,7 @@ class SplitAndScatterFunc using ParticleTileDataType = typename ParticleTileType::ParticleTileDataType; using ParticleBins = amrex::DenseBins; using index_type = typename ParticleBins::index_type; - using SoaData_type = typename WarpXParticleContainer::ParticleTileType::ParticleTileDataType; - + using SoaData_type = WarpXParticleContainer::ParticleTileType::ParticleTileDataType; public: /** @@ -53,8 +52,7 @@ public: AMREX_INLINE amrex::Vector operator() ( const index_type& n_total_pairs, - // Tile& ptile1, Tile& ptile2, - const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + ParticleTileType& ptile1, ParticleTileType& ptile2, const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, const amrex::ParticleReal m1, const amrex::ParticleReal m2, @@ -93,9 +91,8 @@ public: tile_products[i]->resize(products_np[i] + num_added); } - // this works for DSMC since the colliding particles are also products - const auto soa_1 = tile_products[0]->getParticleTileData(); - const auto soa_2 = tile_products[1]->getParticleTileData(); + const auto soa_1 = ptile1.getParticleTileData(); + const auto soa_2 = ptile2.getParticleTileData(); amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; @@ -155,36 +152,10 @@ public: soa_products_data[1].m_rdata[PIdx::w][product2_index] = p_pair_reaction_weight[i]; // Remove p_pair_reaction_weight[i] from the colliding particles' weights - amrex::Gpu::Atomic::AddNoRet(&w1[p_pair_indices_1[i]], - -p_pair_reaction_weight[i]); - amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], - -p_pair_reaction_weight[i]); - - // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX - // to replace the following lambda. - auto const atomicSetIdInvalid = [] AMREX_GPU_DEVICE (uint64_t & idcpu) - { -#if defined(AMREX_USE_OMP) -#pragma omp atomic write - idcpu = amrex::ParticleIdCpus::Invalid; -#else - amrex::Gpu::Atomic::Exch( - (unsigned long long *)&idcpu, - (unsigned long long)amrex::ParticleIdCpus::Invalid - ); -#endif - }; - - // If the colliding particle weight decreases to zero, remove particle by - // setting its id to invalid - if (w1[p_pair_indices_1[i]] <= std::numeric_limits::min()) - { - atomicSetIdInvalid(idcpu1[p_pair_indices_1[i]]); - } - if (w2[p_pair_indices_2[i]] <= std::numeric_limits::min()) - { - atomicSetIdInvalid(idcpu2[p_pair_indices_2[i]]); - } + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w1[p_pair_indices_1[i]], idcpu1[p_pair_indices_1[i]], p_pair_reaction_weight[i]); + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w2[p_pair_indices_2[i]], idcpu2[p_pair_indices_2[i]], p_pair_reaction_weight[i]); // Set the child particle properties appropriately auto& ux1 = soa_products_data[0].m_rdata[PIdx::ux][product1_index]; diff --git a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H index bf31076d975..3cb7197b93a 100644 --- a/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H +++ b/Source/Particles/Collision/BinaryCollision/ParticleCreationFunc.H @@ -67,8 +67,7 @@ public: * function specific to the considered binary collision. * * @param[in] n_total_pairs how many binary collisions have been performed in this tile - * @param[in, out] soa_1 struct of array data of the first colliding particle species - * @param[in, out] soa_2 struct of array data of the second colliding particle species + * @param[in, out] ptile1,ptile2 the particle tiles of the two colliding species * @param[out] tile_products array containing tile data of the product particles. * @param[in] m1 mass of the first colliding particle species * @param[in] m2 mass of the second colliding particle species @@ -94,7 +93,7 @@ public: AMREX_INLINE amrex::Vector operator() ( const index_type& n_total_pairs, - const SoaData_type& soa_1, const SoaData_type& soa_2, + ParticleTileType& ptile1, ParticleTileType& ptile2, const amrex::Vector& pc_products, ParticleTileType** AMREX_RESTRICT tile_products, const amrex::ParticleReal& m1, const amrex::ParticleReal& m2, @@ -129,6 +128,9 @@ public: tile_products[i]->resize(products_np[i] + num_added); } + const auto soa_1 = ptile1.getParticleTileData(); + const auto soa_2 = ptile2.getParticleTileData(); + amrex::ParticleReal* AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; amrex::ParticleReal* AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; uint64_t* AMREX_RESTRICT idcpu1 = soa_1.m_idcpu; @@ -196,37 +198,10 @@ public: } // Remove p_pair_reaction_weight[i] from the colliding particles' weights - amrex::Gpu::Atomic::AddNoRet(&w1[p_pair_indices_1[i]], - -p_pair_reaction_weight[i]); - amrex::Gpu::Atomic::AddNoRet(&w2[p_pair_indices_2[i]], - -p_pair_reaction_weight[i]); - - // Note: Particle::atomicSetID should also be provided as a standalone helper function in AMReX - // to replace the following lambda. - auto const atomicSetIdInvalid = [] AMREX_GPU_DEVICE (uint64_t & idcpu) - { -#if defined(AMREX_USE_OMP) -#pragma omp atomic write - idcpu = amrex::ParticleIdCpus::Invalid; -#else - amrex::Gpu::Atomic::Exch( - (unsigned long long *)&idcpu, - (unsigned long long)amrex::ParticleIdCpus::Invalid - ); -#endif - }; - - // If the colliding particle weight decreases to zero, remove particle by - // setting its id to invalid - if (w1[p_pair_indices_1[i]] <= std::numeric_limits::min()) - { - atomicSetIdInvalid(idcpu1[p_pair_indices_1[i]]); - - } - if (w2[p_pair_indices_2[i]] <= std::numeric_limits::min()) - { - atomicSetIdInvalid(idcpu2[p_pair_indices_2[i]]); - } + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w1[p_pair_indices_1[i]], idcpu1[p_pair_indices_1[i]], p_pair_reaction_weight[i]); + BinaryCollisionUtils::remove_weight_from_colliding_particle( + w2[p_pair_indices_2[i]], idcpu2[p_pair_indices_2[i]], p_pair_reaction_weight[i]); // Initialize the product particles' momentum, using a function depending on the // specific collision type @@ -323,7 +298,7 @@ public: AMREX_INLINE amrex::Vector operator() ( const index_type& /*n_total_pairs*/, - const SoaData_type& /*soa_1*/, const SoaData_type& /*soa_2*/, + ParticleTileType& /*ptile1*/, ParticleTileType& /*ptile2*/, amrex::Vector& /*pc_products*/, ParticleTileType** /*tile_products*/, const amrex::ParticleReal& /*m1*/, const amrex::ParticleReal& /*m2*/, From 0a1284c18fcdc97a683fcec4962253a370fdc27e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 13 May 2024 15:48:39 -0700 Subject: [PATCH 095/190] Release 24.05 (#4928) * AMReX: 24.05 * pyAMReX: 24.05 * WarpX: 24.05 * `Python_restart_eb`: Reset Checksum --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- .../benchmarks_json/Python_restart_eb.json | 14 +++++++------- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index f73c39b6fc3..eaff5543c78 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach d857968968ae2cdf18f39ee50df96529e326ff39 && cd - + cd ../amrex && git checkout --detach 24.05 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 92f235d91ec..31b410fc55e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.04) +project(WarpX VERSION 24.05) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 7299b865746..471f6f2b6a3 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.04' +version = u'24.05' # The full version, including alpha/beta/rc tags. -release = u'24.04' +release = u'24.05' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index d1b8ddd27e5..fbf3330ada0 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.04', + version = '24.05', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/Checksum/benchmarks_json/Python_restart_eb.json b/Regression/Checksum/benchmarks_json/Python_restart_eb.json index 25087c5757d..ad0d2cee5a3 100644 --- a/Regression/Checksum/benchmarks_json/Python_restart_eb.json +++ b/Regression/Checksum/benchmarks_json/Python_restart_eb.json @@ -1,10 +1,10 @@ { "lev=0": { - "Bx": 148618.63186220315, - "By": 148618.63186220315, - "Bz": 3385.851454453729, - "Ex": 55362729623335.9, - "Ey": 55362729623335.89, - "Ez": 68396725892689.86 + "Bx": 148673.005859208, + "By": 148673.00585920806, + "Bz": 3371.758117878558, + "Ex": 55378581103426.71, + "Ey": 55378581103426.72, + "Ez": 68412803445328.25 } -} \ No newline at end of file +} diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 94b334d02da..c188bc5ea45 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = d857968968ae2cdf18f39ee50df96529e326ff39 +branch = 24.05 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 0f3957a8cce..085e9e0220e 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = d857968968ae2cdf18f39ee50df96529e326ff39 +branch = 24.05 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index cbde4bca135..c2e1828b737 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.04 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.05 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "d857968968ae2cdf18f39ee50df96529e326ff39" +set(WarpX_amrex_branch "24.05" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index fcf7c91bce2..a42ffa6ba41 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.04 CONFIG REQUIRED) + find_package(pyAMReX 24.05 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "f370d38ab410dbf1b3a90683c97291841ebeb3f5" +set(WarpX_pyamrex_branch "24.05" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 75e56e6964f..001fe1094fe 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach d857968968ae2cdf18f39ee50df96529e326ff39 && cd - +cd amrex && git checkout --detach 24.05 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index 70b333fc6b9..164f4f1395f 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.04', + version = '24.05', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From b8c8d394801c117827a126b66943aa9b832d5d08 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 14 May 2024 05:47:33 -0700 Subject: [PATCH 096/190] LICENSE.txt: Wrap to 80 Characters (#4929) Use modern formatting as we have in newer codes to wrap the `LICENSE.txt` file for readability to 80 characters per line. No content change. --- LICENSE.txt | 60 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index 9d9b939f342..2965985ebb1 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,16 +1,44 @@ -WarpX Copyright (c) 2018, The Regents of the University of California, through Lawrence Berkeley National Laboratory, and Lawrence Livermore National Security, LLC, for the operation of Lawrence Livermore National Laboratory (subject to receipt of any required approvals from the U.S. Dept. of Energy). All rights reserved. - - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - - -(1) Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -(2) Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -(3) Neither the name of the University of California, Lawrence Berkeley National Laboratory, Lawrence Livermore National Security, LLC, Lawrence Livermore National Laboratory, U.S. Dept. of Energy, nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -You are under no obligation whatsoever to provide any bug fixes, patches, or upgrades to the features, functionality or performance of the source code ("Enhancements") to anyone; however, if you choose to make your Enhancements available either publicly, or directly to Lawrence Berkeley National Laboratory, without imposing a separate written license agreement for such Enhancements, then you hereby grant the following license: a non-exclusive, royalty-free perpetual license to install, use, modify, prepare derivative works, incorporate into other computer software, distribute, and sublicense such enhancements or derivative works thereof, in binary and source code form. +WarpX Copyright (c) 2018, The Regents of the University of California, +through Lawrence Berkeley National Laboratory, and Lawrence Livermore National +Security, LLC, for the operation of Lawrence Livermore National Laboratory +(subject to receipt of any required approvals from the U.S. Dept. of Energy). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +(1) Redistributions of source code must retain the above copyright notice, +this list of conditions and the following disclaimer. + +(2) Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +(3) Neither the name of the University of California, Lawrence Berkeley +National Laboratory, U.S. Dept. of Energy nor the names of its contributors +may be used to endorse or promote products derived from this software +without specific prior written permission. + + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + +You are under no obligation whatsoever to provide any bug fixes, patches, +or upgrades to the features, functionality or performance of the source +code ("Enhancements") to anyone; however, if you choose to make your +Enhancements available either publicly, or directly to Lawrence Berkeley +National Laboratory, without imposing a separate written license agreement +for such Enhancements, then you hereby grant the following license: a +non-exclusive, royalty-free perpetual license to install, use, modify, +prepare derivative works, incorporate into other computer software, +distribute, and sublicense such enhancements or derivative works thereof, +in binary and source code form. From 42c3e2802416bf4e97d072be67bfee284c0531b2 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 14 May 2024 08:56:14 -0700 Subject: [PATCH 097/190] AMReX/pyAMReX/PICSAR: Weekly Update (#4931) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index eaff5543c78..b54afde491b 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.05 && cd - + cd ../amrex && git checkout --detach 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index c188bc5ea45..5e95dd04eb8 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.05 +branch = 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 085e9e0220e..fcd35bbbe7d 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.05 +branch = 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index c2e1828b737..146b6003729 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.05" +set(WarpX_amrex_branch "8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index a42ffa6ba41..f85af8727d4 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.05" +set(WarpX_pyamrex_branch "74d93b53d60b25200e36f345f7b0ac5d611512b9" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 001fe1094fe..470a497979b 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.05 && cd - +cd amrex && git checkout --detach 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From ff2516f4cefddfb01c2e9935dc059f9e07d99dde Mon Sep 17 00:00:00 2001 From: Daniel <92740330+PhysicsDan@users.noreply.github.com> Date: Wed, 15 May 2024 20:54:21 +0100 Subject: [PATCH 098/190] Updated proton-boron cross section (#4377) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * pB Fusion: Updated reaction cross sections * Updated the analytical model for pB fusion reaction cross sections to match that described in A. Tentori and F. Belloni Nucl. Fusion 63, 086001 (2023). * Updated python analysis test scripts to match new cross section function * Fixed reference in analysis_proton_boron_fusion.py * Bug: Fixed typo in analysis script Typo in constant C1 for low temp nonresonant pB reactivity * Updated checksums for pB fusion Updated the checksums for pB fusion 2D and 3D tests using the pB cross section data from A. Tentori & F. Belloni, Nuclear Fusion, 63, 086001 (2023) * Update Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py Fixed comment to note the energy range covered by the XS fit in A. Tentori & F. Belloni Nucl. Fusion 63 086001 (2023). Co-authored-by: Neïl Zaim <49716072+NeilZaim@users.noreply.github.com> * Fix pB crosssection for E>9.76 MeV * Added the power law decribed in Buck et al. Nuclear Physics A, 398(2), 189-202 (1983) for E>9.76 MeV * Note: cross section starting value altered from paper to ensure continuity with E<=9.76 MeV * Update Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py Co-authored-by: Neïl Zaim <49716072+NeilZaim@users.noreply.github.com> * update checksum values for pB11 fusion tests --------- Co-authored-by: Neïl Zaim <49716072+NeilZaim@users.noreply.github.com> Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Co-authored-by: roelof-groenewald --- .../analysis_proton_boron_fusion.py | 88 +++---- .../Proton_Boron_Fusion_2D.json | 158 ++++++------ .../Proton_Boron_Fusion_3D.json | 226 +++++++++--------- .../ProtonBoronFusionCrossSection.H | 76 +++--- 4 files changed, 274 insertions(+), 274 deletions(-) diff --git a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py index eec2ba4fffb..543eb62484a 100755 --- a/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py +++ b/Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py @@ -40,8 +40,8 @@ ## the initial number of protons and borons. ## ## The third test corresponds to a Maxwellian plasma with a 44 keV temperature. The alpha yield is -## directly compared to the analytical fits of W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 -## (2000) for a thermal plasma. +## directly compared to the analytical fits of A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 +## (2023) for a thermal plasma. ## ## The fourth test corresponds to a plasma with an extremely small boron density, so that all boron ## macroparticles should have disappeared by the end of the simulation, which we verify. @@ -259,10 +259,10 @@ def check_isotropy(data, relative_tolerance): def astrophysical_factor_lowE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the low energy fit in the range E < 400 keV - ## described in equation (2) of W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000) + ## described in equation (3) of A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023) C0 = 197. - C1 = 0.24 - C2 = 2.31e-4 + C1 = 0.269 + C2 = 2.54e-4 AL = 1.82e4 EL = 148. dEL = 2.35 @@ -271,12 +271,12 @@ def astrophysical_factor_lowE(E): def astrophysical_factor_midE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the mid energy fit in the range - ## 400 keV < E < 642 keV described in equation (3) of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000) - D0 = 330. - D1 = 66.1 - D2 = -20.3 - D5 = -1.58 + ## 400 keV < E < 668 keV described in equation (4) of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023) + D0 = 346. + D1 = 150. + D2 = -59.9 + D5 = -0.460 E_400 = 400. E_100 = 100. E_norm = (E - E_400)/E_100 @@ -285,29 +285,29 @@ def astrophysical_factor_midE(E): def astrophysical_factor_highE(E): ## E is in keV ## Returns astrophysical factor in MeV b using the high energy fit in the range - ## 642 keV < E < 3500 keV described in equation (4) of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000) - A0 = 2.57e6 - A1 = 5.67e5 - A2 = 1.34e5 - A3 = 5.68e5 - E0 = 581.3 - E1 = 1083. - E2 = 2405. - E3 = 3344. - dE0 = 85.7 - dE1 = 234. - dE2 = 138. - dE3 = 309. - B = 4.38 + ## 668 keV < E < 9760 keV described in equation (5) of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023) + A0 = 1.98e6 + A1 = 3.89e6 + A2 = 1.36e6 + A3 = 3.71e6 + E0 = 640.9 + E1 = 1211. + E2 = 2340. + E3 = 3294. + dE0 = 85.5 + dE1 = 414. + dE2 = 221. + dE3 = 351. + B = 0.381 return A0/((E-E0)**2 + dE0**2) + A1/((E-E1)**2 + dE1**2) + \ A2/((E-E2)**2 + dE2**2) + A3/((E-E3)**2 + dE3**2) + B def astrophysical_factor(E): ## E is in keV - ## Returns astrophysical factor in MeV b using the fits described in W.M. Nevins - ## and R. Swain, Nuclear Fusion, 40, 865 (2000) - conditions = [E <= 400, E <= 642, E > 642] + ## Returns astrophysical factor in MeV b using the fits described in A. Tentori + ## and F. Belloni, Nuclear Fusion, 63, 086001 (2023) + conditions = [E <= 400, E <= 668, E > 668] choices = [astrophysical_factor_lowE(E), astrophysical_factor_midE(E), astrophysical_factor_highE(E)] @@ -316,20 +316,20 @@ def astrophysical_factor(E): def pb_cross_section_buck_fit(E): ## E is in MeV ## Returns cross section in b using a power law fit of the data presented in Buck et al., - ## Nuclear Physics A, 398(2), 189-202 (1983) in the range E > 3.5 MeV. - E_start_fit = 3.5 + ## Nuclear Physics A, 398(2), 189-202 (1983) in the range E > 9.76 MeV. + E_start_fit = 9.76 ## Cross section at E = E_start_fit = 3.5 MeV - cross_section_start_fit = 0.2168440845211521 + cross_section_start_fit = 0.01277998 slope_fit = -2.661840717596765 return cross_section_start_fit*(E/E_start_fit)**slope_fit def pb_cross_section(E): ## E is in keV - ## Returns cross section in b using the fits described in W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000) for E < 3.5 MeV and a power law fit of the data presented in - ## Buck et al., Nuclear Physics A, 398(2), 189-202 (1983) for E > 3.5 MeV. + ## Returns cross section in b using the fits described in A. Tentori and F. Belloni, + ## Nucl. Fusion, 63, 086001 (2023) for E < 9.76 MeV otherwise returns a power law fit + ## of the data in Buck et al., Nuclear Physics A, 398(2), 189-202 (1983) E_MeV = E/1.e3 - conditions = [E <= 3500, E > 3500] + conditions = [E <= 9760, E > 9760] choices = [astrophysical_factor(E)/E_MeV * np.exp(-np.sqrt(E_Gamow_MeV / E_MeV)), pb_cross_section_buck_fit(E_MeV)] return np.select(conditions, choices) @@ -598,13 +598,13 @@ def check_xy_isotropy(data): def sigmav_thermal_fit_lowE_nonresonant(T): ## Temperature T is in keV ## Returns the nonresonant average of cross section multiplied by relative velocity in m^3/s, - ## in the range T <= 70 keV, as described by equation 9 of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000). + ## in the range T <= 70 keV, as described by equations 10-14 of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023). E0 = (E_Gamow_keV/4.)**(1./3.) * T**(2./3.) DE0 = 4.*np.sqrt(T*E0/3.) C0 = 197.*1.e3 - C1 = 0.24*1.e3 - C2 = 2.31e-4*1.e3 + C1 = 0.269*1.e3 + C2 = 2.54e-4*1.e3 tau = 3.*E0/T Seff = C0*(1.+5./(12.*tau)) + C1*(E0+35./36.*T) + C2*(E0**2 + 89./36.*E0*T) ## nonresonant sigma times vrel, in barn meter per second @@ -615,21 +615,21 @@ def sigmav_thermal_fit_lowE_nonresonant(T): def sigmav_thermal_fit_lowE_resonant(T): ## Temperature T is in keV ## Returns the resonant average of cross section multiplied by relative velocity in m^3/s, - ## in the range T <= 70 keV, as described by equation 11 of W.M. Nevins and R. Swain, - ## Nuclear Fusion, 40, 865 (2000). + ## in the range T <= 70 keV, as described by equation 15 of A. Tentori and F. Belloni, + ## Nuclear Fusion, 63, 086001 (2023). return 5.41e-21 * np.exp(-148./T) / T**(3./2.) def sigmav_thermal_fit_lowE(T): ## Temperature T is in keV ## Returns the average of cross section multiplied by relative velocity in m^3/s, using the - ## fits described in section 3.1 of W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). + ## fits described in section 2.2 of A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023). ## The fits are valid for T <= 70 keV. return sigmav_thermal_fit_lowE_nonresonant(T) + sigmav_thermal_fit_lowE_resonant(T) def expected_alpha_thermal(T, proton_density, boron_density, dV, dt): ## Computes the expected number of produced alpha particles when the protons and borons follow ## a Maxwellian distribution with a temperature T, in keV. This uses the thermal fits described - ## in W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). + ## in A. Tentori and F. Belloni, Nuclear Fusion, 63, 086001 (2023). ## The fit used here is only valid in the range T <= 70 keV. assert((T >=0) and (T<=70)) diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json index c6bd572ab68..cdee4b078b0 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_2D.json @@ -2,29 +2,29 @@ "lev=0": { "rho": 0.0 }, - "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3723248133690294e-14, - "particle_position_x": 4095630.6981353555, - "particle_position_y": 8192073.551798361, - "particle_weight": 1.2885512789535533e+28 + "alpha2": { + "particle_momentum_x": 4.057984510682467e-15, + "particle_momentum_y": 4.104188139725855e-15, + "particle_momentum_z": 4.17858000090827e-15, + "particle_position_x": 408793.7905852193, + "particle_position_y": 861780.8020495367, + "particle_weight": 5.078061191951185e+18 }, - "alpha4": { - "particle_momentum_x": 2.3381598715816888e-14, - "particle_momentum_y": 2.3435337108040707e-14, - "particle_momentum_z": 2.3554073734365495e-14, - "particle_position_x": 2457367.4582781526, - "particle_position_y": 4915112.044373058, - "particle_weight": 384.0000000000002 + "alpha3": { + "particle_momentum_x": 5.017656304003558e-16, + "particle_momentum_y": 4.935595075276182e-16, + "particle_momentum_z": 4.867133212376827e-16, + "particle_position_x": 52678.192400911765, + "particle_position_y": 105483.59950020742, + "particle_weight": 1.5413633830148085e+27 }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 8192499.7060386725, - "particle_weight": 6.399508749004609e+30 + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 81921136.14476715, + "particle_weight": 128.00000000000261 }, "boron2": { "particle_momentum_x": 0.0, @@ -32,39 +32,7 @@ "particle_momentum_z": 0.0, "particle_position_x": 409798.015821768, "particle_position_y": 819270.9858143466, - "particle_weight": 1.279999999903553e+28 - }, - "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 819198.7077771134, - "particle_weight": 1.2800000000000004e+37 - }, - "alpha2": { - "particle_momentum_x": 4.057984510682467e-15, - "particle_momentum_y": 4.104188139725855e-15, - "particle_momentum_z": 4.17858000090827e-15, - "particle_position_x": 408793.7905852193, - "particle_position_y": 861780.8020495367, - "particle_weight": 2.893405947366684e+18 - }, - "proton3": { - "particle_momentum_x": 1.6847528610914633e-15, - "particle_momentum_y": 1.6827441855152735e-15, - "particle_momentum_z": 1.6802807054711977e-15, - "particle_position_x": 2457284.001605452, - "particle_position_y": 4914327.937654163, - "particle_weight": 1.2795087490046132e+30 - }, - "alpha3": { - "particle_momentum_x": 4.969030674549252e-16, - "particle_momentum_y": 4.889174794354074e-16, - "particle_momentum_z": 4.826711561221412e-16, - "particle_position_x": 52379.50835085382, - "particle_position_y": 104954.74986555413, - "particle_weight": 1.4737529861620076e+27 + "particle_weight": 1.2799999998307316e+28 }, "boron5": { "particle_momentum_x": 0.0, @@ -74,21 +42,21 @@ "particle_position_y": 819118.5558814355, "particle_weight": 127.99999999999999 }, - "proton5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.2877618571, - "particle_position_y": 819101.3225783394, - "particle_weight": 1.2800000000000004e+37 + "alpha1": { + "particle_momentum_x": 4.691909092431811e-15, + "particle_momentum_y": 4.6836958163275755e-15, + "particle_momentum_z": 4.657203546376977e-15, + "particle_position_x": 463465.27131109464, + "particle_position_y": 978207.6061359186, + "particle_weight": 4.977661460251435e-28 }, - "boron1": { + "proton2": { "particle_momentum_x": 0.0, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 81921136.14476715, - "particle_weight": 128.00000000000261 + "particle_momentum_z": 2.3723248133690294e-14, + "particle_position_x": 4095630.6981353555, + "particle_position_y": 8192073.551798361, + "particle_weight": 1.2885512788807322e+28 }, "proton1": { "particle_momentum_x": 0.0, @@ -98,20 +66,52 @@ "particle_position_y": 81919772.69310114, "particle_weight": 128.00000000000261 }, - "alpha1": { - "particle_momentum_x": 4.691909092431811e-15, - "particle_momentum_y": 4.6836958163275755e-15, - "particle_momentum_z": 4.657203546376977e-15, - "particle_position_x": 463465.27131109464, - "particle_position_y": 978207.6061359186, - "particle_weight": 2.964832324596836e-28 - }, "alpha5": { - "particle_momentum_x": 2.3343469003376247e-14, - "particle_momentum_y": 2.3450435277836452e-14, - "particle_momentum_z": 2.357999284036709e-14, + "particle_momentum_x": 2.3343485859070736e-14, + "particle_momentum_y": 2.3451128753701046e-14, + "particle_momentum_z": 2.3579462789062662e-14, "particle_position_x": 2457556.8571638423, - "particle_position_y": 4914659.635379323, + "particle_position_y": 4914659.635379322, "particle_weight": 3.839999999999998e-19 + }, + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 8192499.7060386725, + "particle_weight": 6.399486212205656e+30 + }, + "proton4": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 819198.7077771134, + "particle_weight": 1.2800000000000004e+37 + }, + "proton3": { + "particle_momentum_x": 1.6847290893972251e-15, + "particle_momentum_y": 1.6827074502304075e-15, + "particle_momentum_z": 1.6802489646490975e-15, + "particle_position_x": 2457270.6999197667, + "particle_position_y": 4914315.665267942, + "particle_weight": 1.279486212205663e+30 + }, + "alpha4": { + "particle_momentum_x": 2.338084461204216e-14, + "particle_momentum_y": 2.3436156778849828e-14, + "particle_momentum_z": 2.35535708386288e-14, + "particle_position_x": 2457367.4582781536, + "particle_position_y": 4915112.044373056, + "particle_weight": 384.0000000000002 + }, + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.2877618571, + "particle_position_y": 819101.3225783394, + "particle_weight": 1.2800000000000004e+37 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json index 9a078d754c3..ec7e047c537 100644 --- a/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json +++ b/Regression/Checksum/benchmarks_json/Proton_Boron_Fusion_3D.json @@ -1,131 +1,131 @@ { "lev=0": { - "rho": 0.0 + "rho": 0.0 }, - "alpha2": { - "particle_momentum_x": 4.1179548991012315e-15, - "particle_momentum_y": 4.110026665992801e-15, - "particle_momentum_z": 4.169802553223462e-15, - "particle_position_x": 408575.75269073684, - "particle_position_y": 413407.5155277014, - "particle_position_z": 863983.4313441743, - "particle_weight": 1.901545867395127e+19 + "proton1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40960140.72983793, + "particle_position_y": 40959772.69310104, + "particle_position_z": 81919021.52308556, + "particle_weight": 1024.000000000021 }, - "alpha4": { - "particle_momentum_x": 2.3389474647479255e-14, - "particle_momentum_y": 2.3425362750821152e-14, - "particle_momentum_z": 2.3515097401900785e-14, - "particle_position_x": 2457367.458278154, - "particle_position_y": 2457512.044373057, - "particle_position_z": 4914475.776513075, - "particle_weight": 3072.000000000002 + "boron1": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.524872467113344e-13, + "particle_position_x": 40958301.591654316, + "particle_position_y": 40961136.14476712, + "particle_position_z": 81920546.19181262, + "particle_weight": 1024.000000000021 + }, + "alpha3": { + "particle_momentum_x": 4.764404554793872e-16, + "particle_momentum_y": 4.655900875811434e-16, + "particle_momentum_z": 4.578927372510084e-16, + "particle_position_x": 50987.442011759704, + "particle_position_y": 48999.674675246955, + "particle_position_z": 101142.57224226737, + "particle_weight": 1.0633705344227063e+28 + }, + "alpha1": { + "particle_momentum_x": 4.665933695243743e-15, + "particle_momentum_y": 4.603805875733438e-15, + "particle_momentum_z": 4.706765986105302e-15, + "particle_position_x": 461871.79172011977, + "particle_position_y": 461162.2166206925, + "particle_position_z": 969262.7809050508, + "particle_weight": 3.2387855108185994e-27 + }, + "alpha5": { + "particle_momentum_x": 2.3388206254864998e-14, + "particle_momentum_y": 2.334372885765467e-14, + "particle_momentum_z": 2.363588638941874e-14, + "particle_position_x": 2457556.857163843, + "particle_position_y": 2457059.6353793247, + "particle_position_z": 4915847.043341331, + "particle_weight": 3.0719999999999984e-18 }, "boron5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409547.33129275695, - "particle_position_y": 409518.5558814353, - "particle_position_z": 819306.5006950963, - "particle_weight": 1023.9999999999999 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409547.33129275695, + "particle_position_y": 409518.5558814353, + "particle_position_z": 819306.5006950963, + "particle_weight": 1023.9999999999999 }, "proton2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.3745333755307162e-14, - "particle_position_x": 4095630.698135355, - "particle_position_y": 4096073.5517983637, - "particle_position_z": 8191737.5566503005, - "particle_weight": 1.0227810240721708e+29 - }, - "proton3": { - "particle_momentum_x": 1.6847690263373399e-15, - "particle_momentum_y": 1.682851264092124e-15, - "particle_momentum_z": 1.6803794796027466e-15, - "particle_position_x": 2457363.4111393224, - "particle_position_y": 2457049.324917686, - "particle_position_z": 4914549.051577593, - "particle_weight": 1.0236635064099502e+31 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.3745333755307162e-14, + "particle_position_x": 4095630.698135355, + "particle_position_y": 4096073.5517983637, + "particle_position_z": 8191737.5566503005, + "particle_weight": 1.022781024024315e+29 }, "proton4": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409630.8789482905, - "particle_position_y": 409598.7077771135, - "particle_position_z": 818958.0399127571, - "particle_weight": 1.0240000000000003e+38 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409630.8789482905, + "particle_position_y": 409598.7077771135, + "particle_position_z": 818958.0399127571, + "particle_weight": 1.0240000000000003e+38 }, - "alpha5": { - "particle_momentum_x": 2.3387550695963677e-14, - "particle_momentum_y": 2.33445389960837e-14, - "particle_momentum_z": 2.3635948557453833e-14, - "particle_position_x": 2457556.8571638414, - "particle_position_y": 2457059.6353793237, - "particle_position_z": 4915847.043341329, - "particle_weight": 3.0719999999999984e-18 + "boron3": { + "particle_momentum_x": 9.277692671587846e-15, + "particle_momentum_y": 9.268409636965691e-15, + "particle_momentum_z": 9.279446607709548e-15, + "particle_position_x": 4096178.1664224654, + "particle_position_y": 4096499.7060386725, + "particle_position_z": 8191465.586938233, + "particle_weight": 5.1196455431551905e+31 }, - "alpha3": { - "particle_momentum_x": 4.72203056468584e-16, - "particle_momentum_y": 4.622060921861793e-16, - "particle_momentum_z": 4.5443204949049515e-16, - "particle_position_x": 50603.134609459696, - "particle_position_y": 48681.319634796026, - "particle_position_z": 100449.24933492322, - "particle_weight": 1.0094807701511393e+28 + "alpha4": { + "particle_momentum_x": 2.33898275612641e-14, + "particle_momentum_y": 2.3423797451957437e-14, + "particle_momentum_z": 2.3516107929259732e-14, + "particle_position_x": 2457367.458278153, + "particle_position_y": 2457512.0443730573, + "particle_position_z": 4914475.7765130745, + "particle_weight": 3072.000000000002 }, - "proton5": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7586062624930794e-15, - "particle_position_x": 409638.28776185703, - "particle_position_y": 409501.32257833943, - "particle_position_z": 819309.1804186807, - "particle_weight": 1.0240000000000003e+38 + "proton3": { + "particle_momentum_x": 1.6847282386883186e-15, + "particle_momentum_y": 1.6828065767793222e-15, + "particle_momentum_z": 1.6803456707569493e-15, + "particle_position_x": 2457343.371083716, + "particle_position_y": 2457033.3891170574, + "particle_position_z": 4914529.855222688, + "particle_weight": 1.023645543155193e+31 }, - "proton1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40960140.72983793, - "particle_position_y": 40959772.69310104, - "particle_position_z": 81919021.52308556, - "particle_weight": 1024.000000000021 + "proton5": { + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7586062624930794e-15, + "particle_position_x": 409638.28776185703, + "particle_position_y": 409501.32257833943, + "particle_position_z": 819309.1804186807, + "particle_weight": 1.0240000000000003e+38 }, "boron2": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 0.0, - "particle_position_x": 409798.0158217681, - "particle_position_y": 409670.9858143465, - "particle_position_z": 819255.8152412223, - "particle_weight": 1.0239999999366152e+29 + "particle_momentum_x": 0.0, + "particle_momentum_y": 0.0, + "particle_momentum_z": 0.0, + "particle_position_x": 409798.0158217681, + "particle_position_y": 409670.9858143465, + "particle_position_z": 819255.8152412223, + "particle_weight": 1.0239999998887592e+29 }, - "alpha1": { - "particle_momentum_x": 4.665933695243743e-15, - "particle_momentum_y": 4.603805875733438e-15, - "particle_momentum_z": 4.706765986105302e-15, - "particle_position_x": 461871.79172011977, - "particle_position_y": 461162.2166206925, - "particle_position_z": 969262.7809050508, - "particle_weight": 1.9171578359058453e-27 - }, - "boron1": { - "particle_momentum_x": 0.0, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.524872467113344e-13, - "particle_position_x": 40958301.591654316, - "particle_position_y": 40961136.14476712, - "particle_position_z": 81920546.19181262, - "particle_weight": 1024.000000000021 - }, - "boron3": { - "particle_momentum_x": 9.277692671587846e-15, - "particle_momentum_y": 9.268409636965691e-15, - "particle_momentum_z": 9.279446607709548e-15, - "particle_position_x": 4096178.1664224654, - "particle_position_y": 4096499.7060386725, - "particle_position_z": 8191465.586938233, - "particle_weight": 5.119663506409945e+31 + "alpha2": { + "particle_momentum_x": 4.1179548991012315e-15, + "particle_momentum_y": 4.110026665992801e-15, + "particle_momentum_z": 4.169802553223462e-15, + "particle_position_x": 408575.75269073684, + "particle_position_y": 413407.5155277014, + "particle_position_z": 863983.4313441743, + "particle_weight": 3.3372246639840338e+19 } -} \ No newline at end of file +} diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H index a5549f30b6f..4f0d663388a 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/ProtonBoronFusionCrossSection.H @@ -16,18 +16,15 @@ #include /** - * \brief Computes the total proton-boron fusion cross section in the range 0 < E < 3.5 MeV using - * the analytical fits given in W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). - * For the record, note that there is a typo in equation (1) of this paper: the total cross section - * should read S(E)/E*exp(-sqrt(E_G/E)) instead of S(E)/E*exp(sqrt(E_G/E)) (minus sign in the - * exponential). + * \brief Computes the total proton-boron fusion cross section in the range 0 < E < 9.76 MeV using + * the analytical fits given in A. Tentori & F. Belloni, Nuclear Fusion, 63, 086001 (2023). * * @param[in] E_keV the kinetic energy of the proton-boron pair in its center of mass frame, in * keV. * @return The total cross section in barn. */ AMREX_GPU_HOST_DEVICE AMREX_INLINE -amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleReal& E_keV) +amrex::ParticleReal ProtonBoronFusionCrossSectionTentori (const amrex::ParticleReal& E_keV) { using namespace amrex::literals; using namespace amrex::Math; @@ -54,13 +51,13 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe // Compute astrophysical factor, in MeV barn, using the fits constexpr auto E_lim1 = 400._prt; // Limits between the different fit regions - constexpr auto E_lim2 = 642._prt; + constexpr auto E_lim2 = 668._prt; amrex::ParticleReal astrophysical_factor; if (E_keV < E_lim1) { constexpr auto C0 = 197._prt; - constexpr auto C1 = 0.24_prt; - constexpr auto C2 = 2.31e-4_prt; + constexpr auto C1 = 0.269_prt; + constexpr auto C2 = 2.54e-4_prt; constexpr auto AL = 1.82e4_prt; constexpr auto EL = 148._prt; constexpr auto dEL_sq = 2.35_prt*2.35_prt; @@ -69,28 +66,28 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe } else if (E_keV < E_lim2) { - constexpr auto D0 = 330._prt; - constexpr auto D1 = 66.1_prt; - constexpr auto D2 = -20.3_prt; - constexpr auto D5 = -1.58_prt; + constexpr auto D0 = 346._prt; + constexpr auto D1 = 150._prt; + constexpr auto D2 = -59.9_prt; + constexpr auto D5 = -0.460_prt; const amrex::ParticleReal E_norm = (E_keV-400._prt) * 1.e-2_prt; astrophysical_factor = D0 + D1*E_norm + D2*powi<2>(E_norm) + D5*powi<5>(E_norm); } else { - constexpr auto A0 = 2.57e6_prt; - constexpr auto A1 = 5.67e5_prt; - constexpr auto A2 = 1.34e5_prt; - constexpr auto A3 = 5.68e5_prt; - constexpr auto E0 = 581.3_prt; - constexpr auto E1 = 1083._prt; - constexpr auto E2 = 2405._prt; - constexpr auto E3 = 3344._prt; - constexpr auto dE0_sq = 85.7_prt*85.7_prt; - constexpr auto dE1_sq = 234._prt*234._prt; - constexpr auto dE2_sq = 138._prt*138._prt; - constexpr auto dE3_sq = 309._prt*309._prt; - constexpr auto B = 4.38_prt; + constexpr auto A0 = 1.98e6_prt; + constexpr auto A1 = 3.89e6_prt; + constexpr auto A2 = 1.36e6_prt; + constexpr auto A3 = 3.71e6_prt; + constexpr auto E0 = 640.9_prt; + constexpr auto E1 = 1211._prt; + constexpr auto E2 = 2340._prt; + constexpr auto E3 = 3294._prt; + constexpr auto dE0_sq = 85.5_prt*85.5_prt; + constexpr auto dE1_sq = 414._prt*414._prt; + constexpr auto dE2_sq = 221._prt*221._prt; + constexpr auto dE3_sq = 351._prt*351._prt; + constexpr auto B = 0.381_prt; astrophysical_factor = A0 / ((E_keV-E0)*(E_keV-E0) + dE0_sq) + A1 / ((E_keV-E1)*(E_keV-E1) + dE1_sq) + A2 / ((E_keV-E2)*(E_keV-E2) + dE2_sq) + @@ -102,10 +99,12 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionNevins (const amrex::ParticleRe } /** - * \brief Computes the total proton-boron fusion cross section in the range E > 3.5 MeV using a + * \brief Computes the total proton-boron fusion cross section in the range E > 9.76 MeV using a * simple power law fit of the data presented in Buck et al., Nuclear Physics A, 398(2), 189-202 - * (1983) (data can also be found in the EXFOR database). - * + * (1983) (data can also be found in the EXFOR database). Note: the fit in Buck et al. started + * from 3.5 MeV. The same exponent power has been used here however the cross_section_start_fit + * has been modified to ensure exact continuity with the fit used for lower energies. + * @param[in] E_keV the kinetic energy of the proton-boron pair in its center of mass frame, in * keV. * @return The total cross section in barn. @@ -115,9 +114,9 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionBuck (const amrex::ParticleReal { using namespace amrex::literals; - constexpr amrex::ParticleReal E_start_fit = 3500._prt; // Fit starts at 3.5 MeV + constexpr amrex::ParticleReal E_start_fit = 9760._prt; // Fit starts at 9.76 MeV // cross section at E = E_start_fit, in barn - constexpr amrex::ParticleReal cross_section_start_fit = 0.2168440845211521_prt; + constexpr amrex::ParticleReal cross_section_start_fit = 0.01277998_prt; constexpr amrex::ParticleReal slope_fit = -2.661840717596765_prt; // Compute fitted value @@ -125,12 +124,13 @@ amrex::ParticleReal ProtonBoronFusionCrossSectionBuck (const amrex::ParticleReal } /** - * \brief Computes the total proton-boron fusion cross section. When E_kin_star < 3.5 MeV, we use - * the analytical fits given in W.M. Nevins and R. Swain, Nuclear Fusion, 40, 865 (2000). When - * E_kin_star > 3.5 MeV, we use a simple power law fit of the data presented in Buck et al., + * \brief Computes the total proton-boron fusion cross section using the analytical fit described + * in A. Tentori & F. Belloni, Nuclear Fusion, 63, 086001 (2023). Includes the Breit-Wigner term + * to reconstruct the 148 keV resonance, which is missing from the Sikora, Wellar dataset. When + * E_kin_star > 9.76 MeV, we use a simple power law fit of the data presented in Buck et al., * Nuclear Physics A, 398(2), 189-202 (1983). Both fits return the same value for - * E_kin_star = 3.5 MeV. - * + * E_kin_star = 9.76 MeV. + * @param[in] E_kin_star the kinetic energy of the proton-boron pair in its center of mass frame, * in SI units. * @return The total cross section in SI units (square meters). @@ -143,10 +143,10 @@ amrex::ParticleReal ProtonBoronFusionCrossSection (const amrex::ParticleReal& E_ // Fits use energy in keV constexpr amrex::ParticleReal joule_to_keV = 1.e-3_prt/PhysConst::q_e; const amrex::ParticleReal E_keV = E_kin_star*joule_to_keV; - constexpr amrex::ParticleReal E_threshold = 3500._prt; + constexpr amrex::ParticleReal E_threshold = 9760._prt; const amrex::ParticleReal cross_section_b = (E_keV <= E_threshold) ? - ProtonBoronFusionCrossSectionNevins(E_keV) : + ProtonBoronFusionCrossSectionTentori(E_keV) : ProtonBoronFusionCrossSectionBuck(E_keV); // Convert cross section to SI units: barn to square meter From c55230bf6d56723372f8ad05b1cf8504dffba04b Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Thu, 16 May 2024 08:36:07 -0700 Subject: [PATCH 099/190] Avoid un-needed copy in BackTransformParticleFunctor (#4934) --- .../ComputeDiagFunctors/BackTransformParticleFunctor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp index 8a6f0765664..9c2d10ad819 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/BackTransformParticleFunctor.cpp @@ -90,7 +90,7 @@ BackTransformParticleFunctor::operator () (PinnedMemoryParticleContainer& pc_dst const amrex::Real dt = warpx.getdt(0); for (WarpXParIter pti(*m_pc_src, lev); pti.isValid(); ++pti) { - auto ptile_dst = pc_dst.DefineAndReturnParticleTile(lev, pti.index(), pti.LocalTileIndex() ); + pc_dst.DefineAndReturnParticleTile(lev, pti.index(), pti.LocalTileIndex() ); } auto& particles = m_pc_src->GetParticles(lev); From 6d2c5edede06b176002d494fc475c45788f122e5 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Sun, 19 May 2024 01:20:07 +0200 Subject: [PATCH 100/190] fix race conditions (#4932) --- Source/BoundaryConditions/WarpX_PEC.cpp | 6 +++--- .../Particles/PhysicalParticleContainer.cpp | 19 +++++++++++++------ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Source/BoundaryConditions/WarpX_PEC.cpp b/Source/BoundaryConditions/WarpX_PEC.cpp index a42873db555..0067f54d3ff 100644 --- a/Source/BoundaryConditions/WarpX_PEC.cpp +++ b/Source/BoundaryConditions/WarpX_PEC.cpp @@ -840,6 +840,7 @@ PEC::ApplyReflectiveBoundarytoJfield( } // Each current component is handled separately below, starting with Jx. + grown_domain_box.convert(Jx_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -850,7 +851,6 @@ PEC::ApplyReflectiveBoundarytoJfield( // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jx_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data @@ -875,6 +875,7 @@ PEC::ApplyReflectiveBoundarytoJfield( } // Handle Jy. + grown_domain_box.convert(Jy_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -885,7 +886,6 @@ PEC::ApplyReflectiveBoundarytoJfield( // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jy_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data @@ -910,6 +910,7 @@ PEC::ApplyReflectiveBoundarytoJfield( } // Handle Jz. + grown_domain_box.convert(Jz_nodal); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif @@ -920,7 +921,6 @@ PEC::ApplyReflectiveBoundarytoJfield( // If grown_domain_box contains fabbox it means there are no PEC // boundaries to handle so continue to next box - grown_domain_box.convert(Jz_nodal); if (grown_domain_box.contains(fabbox)) { continue; } // Extract field data diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index c7f44903d9b..3d134393f06 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1003,6 +1003,19 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int const bool radially_weighted = plasma_injector.radially_weighted; #endif + + // User-defined integer and real attributes: prepare parsers + const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); + const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); + amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); + amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_real_attrib_parserexec_pinned(n_user_real_attribs); + for (int ia = 0; ia < n_user_int_attribs; ++ia) { + user_int_attrib_parserexec_pinned[ia] = m_user_int_attrib_parser[ia]->compile<7>(); + } + for (int ia = 0; ia < n_user_real_attribs; ++ia) { + user_real_attrib_parserexec_pinned[ia] = m_user_real_attrib_parser[ia]->compile<7>(); + } + MFItInfo info; if (do_tiling && Gpu::notInLaunchRegion()) { info.EnableTiling(tile_size); @@ -1156,19 +1169,13 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int } uint64_t * AMREX_RESTRICT pa_idcpu = soa.GetIdCPUData().data() + old_size; // user-defined integer and real attributes - const auto n_user_int_attribs = static_cast(m_user_int_attribs.size()); - const auto n_user_real_attribs = static_cast(m_user_real_attribs.size()); amrex::Gpu::PinnedVector pa_user_int_pinned(n_user_int_attribs); amrex::Gpu::PinnedVector pa_user_real_pinned(n_user_real_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_int_attrib_parserexec_pinned(n_user_int_attribs); - amrex::Gpu::PinnedVector< amrex::ParserExecutor<7> > user_real_attrib_parserexec_pinned(n_user_real_attribs); for (int ia = 0; ia < n_user_int_attribs; ++ia) { pa_user_int_pinned[ia] = soa.GetIntData(particle_icomps[m_user_int_attribs[ia]]).data() + old_size; - user_int_attrib_parserexec_pinned[ia] = m_user_int_attrib_parser[ia]->compile<7>(); } for (int ia = 0; ia < n_user_real_attribs; ++ia) { pa_user_real_pinned[ia] = soa.GetRealData(particle_comps[m_user_real_attribs[ia]]).data() + old_size; - user_real_attrib_parserexec_pinned[ia] = m_user_real_attrib_parser[ia]->compile<7>(); } #ifdef AMREX_USE_GPU // To avoid using managed memory, we first define pinned memory vector, initialize on cpu, From 30a8c3e5b44db65489c5a0c002579c2ee5283c9f Mon Sep 17 00:00:00 2001 From: Avigdor Veksler <124003120+aveksler1@users.noreply.github.com> Date: Mon, 20 May 2024 15:49:41 -0700 Subject: [PATCH 101/190] Ensure compilation with double precision particles for DSMC collisions (#4938) * ensure double precision particle with DSMC collisions * Check ParticleReal precision is double Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --------- Co-authored-by: Avigdor Veksler Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp index e7a4eb722fc..ca0818a6041 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp @@ -7,6 +7,7 @@ * License: BSD-3-Clause-LBNL */ #include "DSMCFunc.H" +#include "Utils/TextMsg.H" /** * \brief Constructor of the DSMCFunc class @@ -22,6 +23,9 @@ DSMCFunc::DSMCFunc ( { using namespace amrex::literals; + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( (std::is_same::value), + "Particle precision must be double for DSMC collisions."); + const amrex::ParmParse pp_collision_name(collision_name); // query for a list of collision processes From 559842bd5ae4bda47c7db0a56151078ea5ebc389 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 20 May 2024 17:06:38 -0700 Subject: [PATCH 102/190] AMReX/pyAMReX/PICSAR: Weekly Update (#4940) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index b54afde491b..de1c8237021 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 && cd - + cd ../amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 5e95dd04eb8..855dcff58fd 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 +branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index fcd35bbbe7d..ac321b8b694 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 +branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 146b6003729..349a4fd3f63 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86" +set(WarpX_amrex_branch "7ca419ebb90da60fefc01d8c1816846fff8638a5" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index f85af8727d4..79feddf3184 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "74d93b53d60b25200e36f345f7b0ac5d611512b9" +set(WarpX_pyamrex_branch "1b795c2c9741c8a63a362ceef8bd5e70cb242e92" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 470a497979b..099a59702a2 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 8eff86d32d17bfbfc6b6bf1091a45bdeb6dd5c86 && cd - +cd amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 6c4171a6f4608cca4fa81cdd07f91b8ec3ae78a3 Mon Sep 17 00:00:00 2001 From: Thomas Marks Date: Tue, 21 May 2024 15:42:15 -0400 Subject: [PATCH 103/190] Promote DSMC calculations to double precision to allow use with single-precision particles (#4941) * promote DSMC calculations to double precision * remove commented-out check * double -> auto for clang-tidy --- .../BinaryCollision/BinaryCollisionUtils.H | 62 ++++++++++--------- .../BinaryCollision/DSMC/DSMCFunc.cpp | 3 - 2 files changed, 34 insertions(+), 31 deletions(-) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H index 08ddcd27174..2e24a93a35e 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollisionUtils.H @@ -62,60 +62,66 @@ namespace BinaryCollisionUtils{ using namespace amrex::literals; using namespace amrex::Math; - constexpr auto one_pr = amrex::ParticleReal(1.); - constexpr auto inv_four_pr = amrex::ParticleReal(1./4.); constexpr double c_sq = PhysConst::c * PhysConst::c; constexpr double inv_csq = 1.0 / c_sq; - const amrex::ParticleReal m1_sq = m1*m1; - const amrex::ParticleReal m2_sq = m2*m2; + // Cast input parameters to double before computing collision properties + // This is needed to avoid errors when using single-precision particles + const auto m1_dbl = static_cast(m1); + const auto m2_dbl = static_cast(m2); + const auto u1x_dbl = static_cast(u1x); + const auto u1y_dbl = static_cast(u1y); + const auto u1z_dbl = static_cast(u1z); + const auto u2x_dbl = static_cast(u2x); + const auto u2y_dbl = static_cast(u2y); + const auto u2z_dbl = static_cast(u2z); + + const double m1_sq = m1_dbl*m1_dbl; + const double m2_sq = m2_dbl*m2_dbl; // Compute Lorentz factor gamma in the lab frame - const double g1 = std::sqrt( 1.0 + static_cast(u1x*u1x+u1y*u1y+u1z*u1z)*inv_csq ); - const double g2 = std::sqrt( 1.0 + static_cast(u2x*u2x+u2y*u2y+u2z*u2z)*inv_csq ); + const double g1 = std::sqrt( 1.0 + (u1x_dbl*u1x_dbl + u1y_dbl*u1y_dbl + u1z_dbl*u1z_dbl)*inv_csq ); + const double g2 = std::sqrt( 1.0 + (u2x_dbl*u2x_dbl + u2y_dbl*u2y_dbl + u2z_dbl*u2z_dbl)*inv_csq ); // Compute momenta - const amrex::ParticleReal p1x = u1x * m1; - const amrex::ParticleReal p1y = u1y * m1; - const amrex::ParticleReal p1z = u1z * m1; - const amrex::ParticleReal p2x = u2x * m2; - const amrex::ParticleReal p2y = u2y * m2; - const amrex::ParticleReal p2z = u2z * m2; + const double p1x = u1x_dbl * m1_dbl; + const double p1y = u1y_dbl * m1_dbl; + const double p1z = u1z_dbl * m1_dbl; + const double p2x = u2x_dbl * m2_dbl; + const double p2y = u2y_dbl * m2_dbl; + const double p2z = u2z_dbl * m2_dbl; + // Square norm of the total (sum between the two particles) momenta in the lab frame - const auto p_total_sq = static_cast( - powi<2>(p1x + p2x) + powi<2>(p1y + p2y) + powi<2>(p1z + p2z) - ); + const double p_total_sq = powi<2>(p1x + p2x) + powi<2>(p1y + p2y) + powi<2>(p1z + p2z); // Total energy in the lab frame // Note the use of `double` for energy since this calculation is // prone to error with single precision. - const auto m1_dbl = static_cast(m1); - const auto m2_dbl = static_cast(m2); - const double E_lab = (m1_dbl * g1 + m2_dbl * g2) * c_sq; + const double E_lab = (m1_dbl*g1 + m2_dbl*g2) * c_sq; // Total energy squared in the center of mass frame, calculated using the Lorentz invariance // of the four-momentum norm const double E_star_sq = E_lab*E_lab - c_sq*p_total_sq; // Kinetic energy in the center of mass frame const double E_star = std::sqrt(E_star_sq); + + // Cast back to chosen precision for output E_kin_COM = static_cast(E_star - (m1_dbl + m2_dbl)*c_sq); // Square of the norm of the momentum of one of the particles in the center of mass frame // Formula obtained by inverting E^2 = p^2*c^2 + m^2*c^4 in the COM frame for each particle // The expression below is specifically written in a form that avoids returning // small negative numbers due to machine precision errors, for low-energy particles - const auto E_ratio = static_cast(E_star/((m1 + m2)*c_sq)); - const auto p_star_sq = static_cast( - m1*m2*c_sq * ( powi<2>(E_ratio) - one_pr ) - + powi<2>(m1 - m2)*c_sq*inv_four_pr * powi<2>( E_ratio - 1._prt/E_ratio) - ); + const double E_ratio = E_star/((m1_dbl + m2_dbl)*c_sq); + const double p_star_sq = m1_dbl*m2_dbl*c_sq * ( powi<2>(E_ratio) - 1.0 ) + + powi<2>(m1_dbl - m2_dbl)*c_sq/4.0 * powi<2>( E_ratio - 1.0/E_ratio); // Lorentz factors in the center of mass frame - const auto g1_star = std::sqrt(one_pr + p_star_sq / static_cast(m1_sq*c_sq)); - const auto g2_star = std::sqrt(one_pr + p_star_sq / static_cast(m2_sq*c_sq)); + const double g1_star = std::sqrt(1.0 + p_star_sq / (m1_sq*c_sq)); + const double g2_star = std::sqrt(1.0 + p_star_sq / (m2_sq*c_sq)); - // relative velocity in the center of mass frame - v_rel_COM = std::sqrt(p_star_sq) * (one_pr/(m1*g1_star) + one_pr/(m2*g2_star)); + // relative velocity in the center of mass frame, cast back to chosen precision + v_rel_COM = static_cast(std::sqrt(p_star_sq) * (1.0/(m1_dbl*g1_star) + 1.0/(m2_dbl*g2_star))); // Cross sections and relative velocity are computed in the center of mass frame. // On the other hand, the particle densities (weight over volume) in the lab frame are used. @@ -124,7 +130,7 @@ namespace BinaryCollisionUtils{ // COM frame and the Lorentz factors in the lab frame (see // Perez et al., Phys.Plasmas.19.083104 (2012)). The correction factor // is calculated here. - lab_to_COM_lorentz_factor = g1_star*g2_star/static_cast(g1*g2); + lab_to_COM_lorentz_factor = static_cast(g1_star*g2_star/(g1*g2)); } /** diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp index ca0818a6041..9ee676bf002 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.cpp @@ -23,9 +23,6 @@ DSMCFunc::DSMCFunc ( { using namespace amrex::literals; - WARPX_ALWAYS_ASSERT_WITH_MESSAGE( (std::is_same::value), - "Particle precision must be double for DSMC collisions."); - const amrex::ParmParse pp_collision_name(collision_name); // query for a list of collision processes From afba801654016e29f327c7cac985ed607adcfd7e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 29 May 2024 06:16:26 -0700 Subject: [PATCH 104/190] Make particle IDs valid in BoundaryScraping (#4955) * Make particle IDs valid in BoundaryScraping * Add CI test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix automated test * Remove test in scraping with filtering This is because the scraping diagnostic does not output all particles in that case (particle filtering is on). * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- Examples/Tests/scraping/analysis_rz.py | 8 ++++++++ Source/Particles/ParticleBoundaryBuffer.cpp | 6 ++++++ 2 files changed, 14 insertions(+) diff --git a/Examples/Tests/scraping/analysis_rz.py b/Examples/Tests/scraping/analysis_rz.py index 193b2575992..11d0194707f 100755 --- a/Examples/Tests/scraping/analysis_rz.py +++ b/Examples/Tests/scraping/analysis_rz.py @@ -60,6 +60,14 @@ def n_scraped_particles( iteration ): n_total = n_remaining[0] assert np.all( n_scraped+n_remaining == n_total) +# Check that the particle IDs match between the initial iteration +# (all particles in the simulation domain) and the finall iteration (particles are either scraped or still in simulation box) +id_initial, = ts_full.get_particle(['id'], iteration=0) +id_final_scrape, = ts_scraping.get_particle(['id'], iteration=ts_scraping.iterations[0]) +id_final_box, = ts_full.get_particle(['id'], iteration=ts_full.iterations[-1]) +id_final = np.concatenate( (id_final_scrape, id_final_box)) +assert np.all( np.sort(id_initial) == np.sort(id_final) ) # Sort because particles may not be in the same order + # Checksum test test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 345a0bfa592..784c27816da 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -160,6 +160,9 @@ struct FindEmbeddedBoundaryIntersection { #else amrex::ignore_unused(x_temp, y_temp, z_temp,normal); #endif + + // flip id to positive in destination + amrex::ParticleIDWrapper{dst.m_idcpu[dst_i]}.make_valid(); } }; #endif @@ -199,6 +202,9 @@ struct CopyAndTimestamp { dst.m_runtime_rdata[m_normal_index][dst_i]= n[0]; dst.m_runtime_rdata[m_normal_index+1][dst_i]= n[1]; dst.m_runtime_rdata[m_normal_index+2][dst_i]= n[2]; + + // flip id to positive in destination + amrex::ParticleIDWrapper{dst.m_idcpu[dst_i]}.make_valid(); } }; From 0e41d93892dd5925b02bb22f57dda8d226246484 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 29 May 2024 18:17:41 +0200 Subject: [PATCH 105/190] Profiler: use amrex logic for device synchronization instead of custom solution (#4889) * Use amrex logic for device synchronization in tiny profiler instead of custom solution * improve comment * remove useless comment * fix missing include * fix bug * fix bug --- Source/Diagnostics/ParticleIO.cpp | 1 + .../HybridPICModel/HybridPICModel.H | 2 + .../HybridPICModel/HybridPICModel.cpp | 1 + .../SpectralSolver/SpectralSolver.cpp | 1 + Source/Filter/Filter.cpp | 1 + Source/Initialization/WarpXAMReXInit.cpp | 19 +++++++ Source/Initialization/WarpXInitData.cpp | 1 + Source/Particles/LaserParticleContainer.cpp | 1 + Source/Particles/WarpXParticleContainer.cpp | 4 +- Source/Utils/WarpXProfilerWrapper.H | 18 +++---- Source/WarpX.cpp | 8 --- Source/ablastr/fields/PoissonSolver.H | 2 +- Source/ablastr/math/fft/WrapCuFFT.cpp | 6 +-- Source/ablastr/particles/DepositCharge.H | 16 +++--- Source/ablastr/profiler/ProfilerWrapper.H | 54 +++---------------- 15 files changed, 52 insertions(+), 83 deletions(-) diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index f1f4d426a4a..bfb1867e741 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -18,6 +18,7 @@ #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" #include diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 5c0c2bcc96a..15208727559 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -22,6 +22,8 @@ #include #include +#include + /** * \brief This class contains the parameters needed to evaluate hybrid field * solutions (kinetic ions with fluid electrons). diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 8979036fbea..6a72bb3569c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -10,6 +10,7 @@ #include "HybridPICModel.H" #include "FieldSolver/Fields.H" +#include "WarpX.H" using namespace amrex; using namespace warpx::fields; diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index 80fb2c39545..460afee999f 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -14,6 +14,7 @@ #include "SpectralKSpace.H" #include "SpectralSolver.H" #include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXProfilerWrapper.H" #include diff --git a/Source/Filter/Filter.cpp b/Source/Filter/Filter.cpp index 5952ffde59c..6243ce4ebbf 100644 --- a/Source/Filter/Filter.cpp +++ b/Source/Filter/Filter.cpp @@ -9,6 +9,7 @@ #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" #include #include diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index 4036a49235c..900dbb53e06 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -7,9 +7,12 @@ #include "Initialization/WarpXAMReXInit.H" +#include "Utils/TextMsg.H" + #include #include #include +#include #include @@ -46,6 +49,22 @@ namespace { pp_amr.add("blocking_factor", 1); } + //See https://github.com/AMReX-Codes/amrex/pull/3763 +#ifdef AMREX_USE_GPU + bool warpx_do_device_synchronize = true; +#else + bool warpx_do_device_synchronize = false; +#endif + pp_warpx.query("do_device_synchronize", warpx_do_device_synchronize); + bool do_device_synchronize = warpx_do_device_synchronize; + amrex::ParmParse pp_tiny_profiler("tiny_profiler"); + if (pp_tiny_profiler.queryAdd("device_synchronize_around_region", do_device_synchronize) ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + do_device_synchronize == warpx_do_device_synchronize, + "tiny_profiler.device_synchronize_around_region overrides warpx.do_device_synchronize."); + } + // Here we override the default tiling option for particles, which is always // "false" in AMReX, to "false" if compiling for GPU execution and "true" // if compiling for CPU. diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index ff2ca01deec..0d5ab70c657 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -16,6 +16,7 @@ #endif #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "FieldSolver/Fields.H" #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Filter/BilinearFilter.H" diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index bf846c674f0..7b735053f0b 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -19,6 +19,7 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" #include diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 8692dff3302..4800d9e209f 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -1161,9 +1161,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, ng_rho, depos_lev, ref_ratio, offset, np_to_deposit, - icomp, nc, - WarpX::do_device_synchronize - ); + icomp, nc); } } diff --git a/Source/Utils/WarpXProfilerWrapper.H b/Source/Utils/WarpXProfilerWrapper.H index 856fc9d0e02..8b13b0df2ab 100644 --- a/Source/Utils/WarpXProfilerWrapper.H +++ b/Source/Utils/WarpXProfilerWrapper.H @@ -8,17 +8,13 @@ #ifndef WARPX_PROFILERWRAPPER_H_ #define WARPX_PROFILERWRAPPER_H_ -#include "WarpX.H" -#include "ablastr/profiler/ProfilerWrapper.H" +#include - -// `BL_PROFILE_PASTE(SYNC_SCOPE_, __COUNTER__)` and `SYNC_V_##vname` used to make unique names for -// synchronizeOnDestruct objects, like `SYNC_SCOPE_0` and `SYNC_V_pmain` -#define WARPX_PROFILE(fname) ABLASTR_PROFILE(fname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR(fname, vname) ABLASTR_PROFILE_VAR(fname, vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR_NS(fname, vname) ABLASTR_PROFILE_VAR_NS(fname, vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR_START(vname) ABLASTR_PROFILE_VAR_START(vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR_STOP(vname) ABLASTR_PROFILE_VAR_STOP(vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_REGION(rname) ABLASTR_PROFILE_REGION(rname, WarpX::do_device_synchronize) +#define WARPX_PROFILE(fname) BL_PROFILE(fname) +#define WARPX_PROFILE_VAR(fname, vname) BL_PROFILE_VAR(fname, vname) +#define WARPX_PROFILE_VAR_NS(fname, vname) BL_PROFILE_VAR_NS(fname, vname) +#define WARPX_PROFILE_VAR_START(vname) BL_PROFILE_VAR_START(vname) +#define WARPX_PROFILE_VAR_STOP(vname) BL_PROFILE_VAR_STOP(vname) +#define WARPX_PROFILE_REGION(rname) BL_PROFILE_REGION(rname) #endif // WARPX_PROFILERWRAPPER_H_ diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 8c3a6f15b1c..b4c088383ef 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -211,12 +211,6 @@ int WarpX::n_current_deposition_buffer = -1; short WarpX::grid_type; amrex::IntVect m_rho_nodal_flag; -#ifdef AMREX_USE_GPU -bool WarpX::do_device_synchronize = true; -#else -bool WarpX::do_device_synchronize = false; -#endif - WarpX* WarpX::m_instance = nullptr; void WarpX::MakeWarpX () @@ -683,8 +677,6 @@ WarpX::ReadParameters () ReadBoostedFrameParameters(gamma_boost, beta_boost, boost_direction); - pp_warpx.query("do_device_synchronize", do_device_synchronize); - // queryWithParser returns 1 if argument zmax_plasma_to_compute_max_step is // specified by the user, 0 otherwise. do_compute_max_step_from_zmax = utils::parser::queryWithParser( diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index da0078f8b5a..106c5475dd7 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -116,7 +116,7 @@ computePhi (amrex::Vector const & rho, { using namespace amrex::literals; - ABLASTR_PROFILE("computePhi", false); + ABLASTR_PROFILE("computePhi"); if (!rel_ref_ratio.has_value()) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(rho.size() == 1u, diff --git a/Source/ablastr/math/fft/WrapCuFFT.cpp b/Source/ablastr/math/fft/WrapCuFFT.cpp index b6b1706f807..9ceb91457c9 100644 --- a/Source/ablastr/math/fft/WrapCuFFT.cpp +++ b/Source/ablastr/math/fft/WrapCuFFT.cpp @@ -31,7 +31,7 @@ namespace ablastr::math::anyfft Complex * const complex_array, const direction dir, const int dim) { FFTplan fft_plan; - ABLASTR_PROFILE("ablastr::math::anyfft::CreatePlan", false); + ABLASTR_PROFILE("ablastr::math::anyfft::CreatePlan"); // Initialize fft_plan.m_plan with the vendor fft plan. cufftResult result; @@ -71,12 +71,12 @@ namespace ablastr::math::anyfft void DestroyPlan(FFTplan& fft_plan) { - ABLASTR_PROFILE("ablastr::math::anyfft::DestroyPlan", false); + ABLASTR_PROFILE("ablastr::math::anyfft::DestroyPlan"); cufftDestroy( fft_plan.m_plan ); } void Execute(FFTplan& fft_plan){ - ABLASTR_PROFILE("ablastr::math::anyfft::Execute", false); + ABLASTR_PROFILE("ablastr::math::anyfft::Execute"); // make sure that this is done on the same GPU stream as the above copy cudaStream_t stream = amrex::Gpu::Device::cudaStream(); cufftSetStream ( fft_plan.m_plan, stream); diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index b2dedccd03e..ff3741a7a43 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -44,7 +44,6 @@ namespace ablastr::particles * \param np_to_deposit number of particles to deposit (default: pti.numParticles()) * \param icomp component in MultiFab to start depositing to * \param nc number of components to deposit - * \param do_device_synchronize call amrex::Gpu::synchronize() for tiny profiler regions (default: true) */ template< typename T_PC > static void @@ -63,8 +62,7 @@ deposit_charge (typename T_PC::ParIterType& pti, std::optional rel_ref_ratio = std::nullopt, long const offset = 0, std::optional np_to_deposit = std::nullopt, - int const icomp = 0, int const nc = 1, - bool const do_device_synchronize = true) + int const icomp = 0, int const nc = 1) { // deposition guards amrex::IntVect ng_rho = rho->nGrowVect(); @@ -131,8 +129,8 @@ deposit_charge (typename T_PC::ParIterType& pti, amrex::numParticlesOutOfRange(pti, range) == 0, "Particles shape does not fit within tile (CPU) or guard cells (GPU) used for charge deposition"); - ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::ChargeDeposition", blp_ppc_chd, do_device_synchronize); - ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::Accumulate", blp_accumulate, do_device_synchronize); + ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::ChargeDeposition", blp_ppc_chd); + ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::Accumulate", blp_accumulate); // Get tile box where charge is deposited. // The tile box is different when depositing in the buffers (depos_lev(GetPosition, wp.dataPtr()+offset, ion_lev, @@ -192,13 +190,13 @@ deposit_charge (typename T_PC::ParIterType& pti, rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, n_rz_azimuthal_modes); } - ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd, do_device_synchronize); + ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd); #ifndef AMREX_USE_GPU // CPU, tiling: atomicAdd local_rho into rho - ABLASTR_PROFILE_VAR_START(blp_accumulate, do_device_synchronize); + ABLASTR_PROFILE_VAR_START(blp_accumulate); (*rho)[pti].lockAdd(local_rho, tb, tb, 0, icomp*nc, nc); - ABLASTR_PROFILE_VAR_STOP(blp_accumulate, do_device_synchronize); + ABLASTR_PROFILE_VAR_STOP(blp_accumulate); #endif } diff --git a/Source/ablastr/profiler/ProfilerWrapper.H b/Source/ablastr/profiler/ProfilerWrapper.H index 6b476d78114..268c1abd971 100644 --- a/Source/ablastr/profiler/ProfilerWrapper.H +++ b/Source/ablastr/profiler/ProfilerWrapper.H @@ -9,54 +9,12 @@ #define ABLASTR_PROFILERWRAPPER_H_ #include -#include - -namespace ablastr::profiler -{ - /** Conditionally synchronizes active GPU operations - * - * @param do_device_synchronize perform amrex::Gpu::synchronize() if true - */ - AMREX_FORCE_INLINE - void - device_synchronize(bool const do_device_synchronize = false) { - if (do_device_synchronize) { - amrex::Gpu::synchronize(); - } - } - - /** An object that conditionally calls device_synchronize() on destruction - * - * Note that objects are destructed in the reverse order of declaration - */ - struct SynchronizeOnDestruct { - SynchronizeOnDestruct(bool const do_device_synchronize = false) - : m_do_device_synchronize(do_device_synchronize) {} - - AMREX_FORCE_INLINE - ~SynchronizeOnDestruct() { - device_synchronize(m_do_device_synchronize); - } - - // default move and copy operations - SynchronizeOnDestruct(const SynchronizeOnDestruct&) = default; - SynchronizeOnDestruct& operator=(const SynchronizeOnDestruct&) = default; - SynchronizeOnDestruct(SynchronizeOnDestruct&&) = default; - SynchronizeOnDestruct& operator=(SynchronizeOnDestruct&& field_data) = default; - - bool m_do_device_synchronize = false; - }; - -} // namespace ablastr::profiler - -// `BL_PROFILE_PASTE(SYNC_SCOPE_, __COUNTER__)` and `SYNC_V_##vname` used to make unique names for -// synchronizeOnDestruct objects, like `SYNC_SCOPE_0` and `SYNC_V_pmain` -#define ABLASTR_PROFILE(fname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE(fname); const ablastr::profiler::SynchronizeOnDestruct BL_PROFILE_PASTE(SYNC_SCOPE_, __COUNTER__){sync} -#define ABLASTR_PROFILE_VAR(fname, vname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_VAR(fname, vname); const ablastr::profiler::SynchronizeOnDestruct SYNC_V_##vname{sync} -#define ABLASTR_PROFILE_VAR_NS(fname, vname, sync) BL_PROFILE_VAR_NS(fname, vname); const ablastr::profiler::SynchronizeOnDestruct SYNC_V_##vname{sync} -#define ABLASTR_PROFILE_VAR_START(vname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_VAR_START(vname) -#define ABLASTR_PROFILE_VAR_STOP(vname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_VAR_STOP(vname) -#define ABLASTR_PROFILE_REGION(rname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_REGION(rname); const ablastr::profiler::SynchronizeOnDestruct BL_PROFILE_PASTE(SYNC_R_, __COUNTER__){sync} +#define ABLASTR_PROFILE(fname) BL_PROFILE(fname) +#define ABLASTR_PROFILE_VAR(fname, vname) BL_PROFILE_VAR(fname, vname) +#define ABLASTR_PROFILE_VAR_NS(fname, vname) BL_PROFILE_VAR_NS(fname, vname) +#define ABLASTR_PROFILE_VAR_START(vname) BL_PROFILE_VAR_START(vname) +#define ABLASTR_PROFILE_VAR_STOP(vname) BL_PROFILE_VAR_STOP(vname) +#define ABLASTR_PROFILE_REGION(rname) BL_PROFILE_REGION(rname) #endif // ABLASTR_PROFILERWRAPPER_H_ From 25a4c5dc2969e4053b46adccd09d923628906974 Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 29 May 2024 13:46:03 -0700 Subject: [PATCH 106/190] Add particle boundary none (#4920) * Add None particle boundary condition * Use None particle boundary in CI test langmuir/inputs_rz * Update BC for picmi input files --- Docs/source/usage/parameters.rst | 4 ++++ .../laser_acceleration/PICMI_inputs_rz.py | 2 +- .../spacecraft_charging/PICMI_inputs_rz.py | 2 +- Examples/Tests/langmuir/PICMI_inputs_rz.py | 2 +- Examples/Tests/langmuir/inputs_rz | 2 ++ Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py | 2 +- Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py | 2 +- .../particle_boundary_interaction/PICMI_inputs_rz.py | 2 +- Source/Utils/WarpXAlgorithmSelection.H | 3 ++- Source/Utils/WarpXAlgorithmSelection.cpp | 1 + Source/WarpX.cpp | 8 ++++++++ 11 files changed, 23 insertions(+), 7 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 40c16a06df5..4f011025213 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -437,6 +437,10 @@ Domain Boundary Conditions The standard deviation for these distributions should be provided for each species using ``boundary..u_th``. The same standard deviation is used to sample all components. + * ``None``: No boundary conditions are applied to the particles. + When using RZ, this option must be used for the lower radial boundary, the first value of ``boundary.particle_lo``. + This should not be used in any other cases. + * ``boundary.reflect_all_velocities`` (`bool`) optional (default `false`) For a reflecting boundary condition, this flags whether the sign of only the normal velocity is changed or all velocities. diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py index 9990cca01cf..7f09db8d6b3 100755 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py @@ -31,7 +31,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'absorbing'], + lower_boundary_conditions_particles = ['none', 'absorbing'], upper_boundary_conditions_particles = ['absorbing', 'absorbing'], moving_window_velocity = [0., c], warpx_max_grid_size = max_grid_size, diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index d541c2393ca..5b57b59fbe3 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -220,7 +220,7 @@ def compute_actual_charge_on_spacecraft(): upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'reflecting'], + lower_boundary_conditions_particles = ['none', 'reflecting'], upper_boundary_conditions_particles = ['absorbing', 'reflecting'] ) diff --git a/Examples/Tests/langmuir/PICMI_inputs_rz.py b/Examples/Tests/langmuir/PICMI_inputs_rz.py index 8328aba5185..8da03b00469 100755 --- a/Examples/Tests/langmuir/PICMI_inputs_rz.py +++ b/Examples/Tests/langmuir/PICMI_inputs_rz.py @@ -90,7 +90,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'periodic'], upper_boundary_conditions = ['none', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], + lower_boundary_conditions_particles = ['none', 'periodic'], upper_boundary_conditions_particles = ['absorbing', 'periodic'], moving_window_velocity = [0.,0.], warpx_max_grid_size=64) diff --git a/Examples/Tests/langmuir/inputs_rz b/Examples/Tests/langmuir/inputs_rz index 534296759a2..3b70a91f207 100644 --- a/Examples/Tests/langmuir/inputs_rz +++ b/Examples/Tests/langmuir/inputs_rz @@ -28,6 +28,8 @@ geometry.prob_lo = 0.e-6 -20.e-6 # physical domain geometry.prob_hi = 20.e-6 20.e-6 boundary.field_lo = none periodic boundary.field_hi = none periodic +boundary.particle_lo = none periodic +boundary.particle_hi = absorbing periodic warpx.serialize_initial_conditions = 1 diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py index 1268f7a02b0..e46f561f538 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py @@ -69,7 +69,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['neumann', 'neumann'], - lower_boundary_conditions_particles = ['reflecting', 'absorbing'], + lower_boundary_conditions_particles = ['none', 'absorbing'], upper_boundary_conditions_particles = ['absorbing', 'absorbing'], warpx_potential_lo_z = V_domain_boundary, warpx_blocking_factor=8, diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py index 89c5230a6ad..5bd1e3518f9 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py @@ -156,7 +156,7 @@ def setup_run(self): upper_bound=[self.Lr, self.Lz/2.0], lower_boundary_conditions = ['none', 'periodic'], upper_boundary_conditions = ['dirichlet', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], + lower_boundary_conditions_particles = ['none', 'periodic'], upper_boundary_conditions_particles = ['reflecting', 'periodic'] ) simulation.time_step_size = self.dt diff --git a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py index 4017a05d413..df4a4579e2f 100644 --- a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py +++ b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py @@ -39,7 +39,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'reflecting'], + lower_boundary_conditions_particles = ['none', 'reflecting'], upper_boundary_conditions_particles = ['absorbing', 'reflecting'] ) diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index c34516aedc5..e067ed03a80 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -175,7 +175,8 @@ enum struct ParticleBoundaryType { Open = 1, //!< particles cross domain boundary leave with damped j Reflecting = 2, //!< particles are reflected Periodic = 3, //!< particles are introduced from the periodic boundary - Thermal = 4 + Thermal = 4, + None = 5 //!< For r=0 boundary with RZ simulations }; /** MPI reductions diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index b1f93c2f6c8..2449ea2f8dd 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -142,6 +142,7 @@ const std::map ParticleBCType_algo_to_enum = {"reflecting", ParticleBoundaryType::Reflecting}, {"periodic", ParticleBoundaryType::Periodic}, {"thermal", ParticleBoundaryType::Thermal}, + {"none", ParticleBoundaryType::None}, {"default", ParticleBoundaryType::Absorbing} }; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index b4c088383ef..de632211bad 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1169,6 +1169,14 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( WarpX::field_boundary_lo[0] == FieldBoundaryType::None, "Error : Field boundary at r=0 must be ``none``. \n"); + + const ParmParse pp_boundary("boundary"); + if (pp_boundary.contains("particle_lo")) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + WarpX::particle_boundary_lo[0] == ParticleBoundaryType::None, + "Error : Particle boundary at r=0 must be ``none``. \n"); + } + } if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { From d7b2ed28bc40e1c6d19cc60c2900f28006bb846f Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 30 May 2024 00:05:37 +0200 Subject: [PATCH 107/190] make RemakeMultiFab a lambda function inside WarpX::RemakeLevel (#4947) --- Source/Parallelization/WarpXRegrid.cpp | 105 ++++++++++++------------- 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 22979912a84..2846e79c361 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -164,22 +164,19 @@ WarpX::LoadBalance () #endif } - -template void -RemakeMultiFab (std::unique_ptr& mf, const DistributionMapping& dm, - const bool redistribute, const int lev) -{ - if (mf == nullptr) { return; } - const IntVect& ng = mf->nGrowVect(); - std::unique_ptr pmf; - WarpX::AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); - if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } - mf = std::move(pmf); -} - void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { + + const auto RemakeMultiFab = [&](auto& mf, const bool redistribute){ + if (mf == nullptr) { return; } + const IntVect& ng = mf->nGrowVect(); + auto pmf = std::remove_reference_t{}; + AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); + if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } + mf = std::move(pmf); + }; + if (ba == boxArray(lev)) { if (ParallelDescriptor::NProcs() == 1) { return; } @@ -187,60 +184,60 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi // Fine patch for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_fp[lev][idim], dm, true ,lev); - RemakeMultiFab(Efield_fp[lev][idim], dm, true ,lev); + RemakeMultiFab(Bfield_fp[lev][idim], true); + RemakeMultiFab(Efield_fp[lev][idim], true); if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Bfield_fp_external[lev][idim], dm, true ,lev); + RemakeMultiFab(Bfield_fp_external[lev][idim], true); } if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Efield_fp_external[lev][idim], dm, true ,lev); + RemakeMultiFab(Efield_fp_external[lev][idim], true); } - RemakeMultiFab(current_fp[lev][idim], dm, false ,lev); - RemakeMultiFab(current_store[lev][idim], dm, false ,lev); + RemakeMultiFab(current_fp[lev][idim], false); + RemakeMultiFab(current_store[lev][idim], false); if (current_deposition_algo == CurrentDepositionAlgo::Vay) { - RemakeMultiFab(current_fp_vay[lev][idim], dm, false ,lev); + RemakeMultiFab(current_fp_vay[lev][idim], false); } if (do_current_centering) { - RemakeMultiFab(current_fp_nodal[lev][idim], dm, false ,lev); + RemakeMultiFab(current_fp_nodal[lev][idim], false); } if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_fp[lev][idim], dm, true ,lev); - RemakeMultiFab(Bfield_avg_fp[lev][idim], dm, true ,lev); + RemakeMultiFab(Efield_avg_fp[lev][idim], true); + RemakeMultiFab(Bfield_avg_fp[lev][idim], true); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], dm, true, lev); - RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], dm, false, lev); - RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim], dm, true, lev); + RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], true); + RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], false); + RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim],true); } #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - RemakeMultiFab(m_edge_lengths[lev][idim], dm, false ,lev); - RemakeMultiFab(m_face_areas[lev][idim], dm, false ,lev); + RemakeMultiFab(m_edge_lengths[lev][idim], false); + RemakeMultiFab(m_face_areas[lev][idim], false); if(WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT){ - RemakeMultiFab(Venl[lev][idim], dm, false ,lev); - RemakeMultiFab(m_flag_info_face[lev][idim], dm, false ,lev); - RemakeMultiFab(m_flag_ext_face[lev][idim], dm, false ,lev); - RemakeMultiFab(m_area_mod[lev][idim], dm, false ,lev); - RemakeMultiFab(ECTRhofield[lev][idim], dm, false ,lev); + RemakeMultiFab(Venl[lev][idim], false); + RemakeMultiFab(m_flag_info_face[lev][idim], false); + RemakeMultiFab(m_flag_ext_face[lev][idim], false); + RemakeMultiFab(m_area_mod[lev][idim], false); + RemakeMultiFab(ECTRhofield[lev][idim], false); m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); } } #endif } - RemakeMultiFab(F_fp[lev], dm, true ,lev); - RemakeMultiFab(rho_fp[lev], dm, false ,lev); + RemakeMultiFab(F_fp[lev], true); + RemakeMultiFab(rho_fp[lev], false); // phi_fp should be redistributed since we use the solution from // the last step as the initial guess for the next solve - RemakeMultiFab(phi_fp[lev], dm, true ,lev); + RemakeMultiFab(phi_fp[lev], true); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], dm, true, lev); - RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], dm, false, lev); + RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], true); + RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], false); } #ifdef AMREX_USE_EB - RemakeMultiFab(m_distance_to_eb[lev], dm, false ,lev); + RemakeMultiFab(m_distance_to_eb[lev], false); int max_guard = guard_cells.ng_FieldSolver.max(); m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, @@ -296,8 +293,8 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi } else { for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_aux[lev][idim], dm, false ,lev); - RemakeMultiFab(Efield_aux[lev][idim], dm, false ,lev); + RemakeMultiFab(Bfield_aux[lev][idim], false); + RemakeMultiFab(Efield_aux[lev][idim], false); } } @@ -305,16 +302,16 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi if (lev > 0) { for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_cp[lev][idim], dm, true ,lev); - RemakeMultiFab(Efield_cp[lev][idim], dm, true ,lev); - RemakeMultiFab(current_cp[lev][idim], dm, false ,lev); + RemakeMultiFab(Bfield_cp[lev][idim], true); + RemakeMultiFab(Efield_cp[lev][idim], true); + RemakeMultiFab(current_cp[lev][idim], false); if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_cp[lev][idim], dm, true ,lev); - RemakeMultiFab(Bfield_avg_cp[lev][idim], dm, true ,lev); + RemakeMultiFab(Efield_avg_cp[lev][idim], true); + RemakeMultiFab(Bfield_avg_cp[lev][idim], true); } } - RemakeMultiFab(F_cp[lev], dm, true ,lev); - RemakeMultiFab(rho_cp[lev], dm, false ,lev); + RemakeMultiFab(F_cp[lev], true); + RemakeMultiFab(rho_cp[lev], false); #ifdef WARPX_USE_PSATD if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -354,14 +351,14 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi if (lev > 0 && (n_field_gather_buffer > 0 || n_current_deposition_buffer > 0)) { for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_cax[lev][idim], dm, false ,lev); - RemakeMultiFab(Efield_cax[lev][idim], dm, false ,lev); - RemakeMultiFab(current_buf[lev][idim], dm, false ,lev); + RemakeMultiFab(Bfield_cax[lev][idim], false); + RemakeMultiFab(Efield_cax[lev][idim], false); + RemakeMultiFab(current_buf[lev][idim], false); } - RemakeMultiFab(charge_buf[lev], dm, false ,lev); + RemakeMultiFab(charge_buf[lev], false); // we can avoid redistributing these since we immediately re-build the values via BuildBufferMasks() - RemakeMultiFab(current_buffer_masks[lev], dm, false ,lev); - RemakeMultiFab(gather_buffer_masks[lev], dm, false ,lev); + RemakeMultiFab(current_buffer_masks[lev], false); + RemakeMultiFab(gather_buffer_masks[lev], false); if (current_buffer_masks[lev] || gather_buffer_masks[lev]) { BuildBufferMasks(); From ca5978ad55f530a13afcc081b1d52c5bfaaba0b8 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 29 May 2024 16:44:14 -0700 Subject: [PATCH 108/190] Fix: `MPIInitHelpers.cpp` Macro (#4957) This failed if `AMREX_USE_OMP` is undefined. --- Source/ablastr/parallelization/MPIInitHelpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/ablastr/parallelization/MPIInitHelpers.cpp b/Source/ablastr/parallelization/MPIInitHelpers.cpp index 513035c483c..5ba15a9e64d 100644 --- a/Source/ablastr/parallelization/MPIInitHelpers.cpp +++ b/Source/ablastr/parallelization/MPIInitHelpers.cpp @@ -35,7 +35,7 @@ namespace ablastr::parallelization #ifdef AMREX_USE_MPI # ifdef AMREX_MPI_THREAD_MULTIPLE // i.e. for async_io return MPI_THREAD_MULTIPLE; -# elif AMREX_USE_OMP +# elif defined(AMREX_USE_OMP) return MPI_THREAD_FUNNELED; # else return MPI_THREAD_SINGLE; // equiv. to MPI_Init From 54ed5ccfe955c1b1e2995e64c181f2a6b3946033 Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 29 May 2024 18:59:07 -0700 Subject: [PATCH 109/190] Picmi: allow different `random_fraction` for different species (#4951) * picmi: allow different `random_fraction` or `uniform_stride` for different species * fix codeQL issue * use dictionary to specify particle diagnostic stride or random fraction * use 1 as stride and random fraction default for unspecified species * use species rather than species.name as key for random fraction dict * update docstring --- Python/pywarpx/picmi.py | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index e7575f83b26..34c21c2a34a 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2497,11 +2497,17 @@ class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnostic warpx_file_min_digits: integer, optional Minimum number of digits for the time step number in the file name - warpx_random_fraction: float, optional - Random fraction of particles to include in the diagnostic - - warpx_uniform_stride: integer, optional - Stride to down select to the particles to include in the diagnostic + warpx_random_fraction: float or dict, optional + Random fraction of particles to include in the diagnostic. If a float + is given the same fraction will be used for all species, if a dictionary + is given the keys should be species with the value specifying the random + fraction for that species. + + warpx_uniform_stride: integer or dict, optional + Stride to down select to the particles to include in the diagnostic. + If an integer is given the same stride will be used for all species, if + a dictionary is given the keys should be species with the value + specifying the stride for that species. warpx_plot_filter_function: string, optional Analytic expression to down select the particles to in the diagnostic @@ -2599,6 +2605,22 @@ def diagnostic_initialize_inputs(self): else: species_names = [self.species.name] + # check if random fraction is specified and whether a value is given per species + random_fraction = {} + random_fraction_default = self.random_fraction + if isinstance(self.random_fraction, dict): + random_fraction_default = 1.0 + for key, val in self.random_fraction.items(): + random_fraction[key.name] = val + + # check if uniform stride is specified and whether a value is given per species + uniform_stride = {} + uniform_stride_default = self.uniform_stride + if isinstance(self.uniform_stride, dict): + uniform_stride_default = 1 + for key, val in self.uniform_stride.items(): + uniform_stride[key.name] = val + if self.mangle_dict is None: # Only do this once so that the same variables are used in this distribution # is used multiple times @@ -2607,8 +2629,8 @@ def diagnostic_initialize_inputs(self): for name in species_names: diag = pywarpx.Bucket.Bucket(self.name + '.' + name, variables = variables, - random_fraction = self.random_fraction, - uniform_stride = self.uniform_stride) + random_fraction = random_fraction.get(name, random_fraction_default), + uniform_stride = uniform_stride.get(name, uniform_stride_default)) expression = pywarpx.my_constants.mangle_expression(self.plot_filter_function, self.mangle_dict) diag.__setattr__('plot_filter_function(t,x,y,z,ux,uy,uz)', expression) self.diagnostic._species_dict[name] = diag From a40d0063f275c2bb801cdf24ee9afebf132b8da0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 29 May 2024 20:29:42 -0700 Subject: [PATCH 110/190] AMReX/pyAMReX/PICSAR: Weekly Update (#4956) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index de1c8237021..c58a35eace8 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - + cd ../amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 855dcff58fd..12b0a0fdfcb 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 +branch = 28b010126a1b39297d8a496ba81f171d8563953b [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index ac321b8b694..58d97e4547c 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 +branch = 28b010126a1b39297d8a496ba81f171d8563953b [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 349a4fd3f63..f80ec0d5af7 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "7ca419ebb90da60fefc01d8c1816846fff8638a5" +set(WarpX_amrex_branch "28b010126a1b39297d8a496ba81f171d8563953b" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 79feddf3184..cdef8f277f6 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "1b795c2c9741c8a63a362ceef8bd5e70cb242e92" +set(WarpX_pyamrex_branch "d4d409bd21bc4c48487883ac2331efdb1a6b3d61" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 099a59702a2..e9dbb0f2533 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - +cd amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From b9549372d8f85071bcbffff906c38477e5909c85 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 30 May 2024 00:41:30 -0700 Subject: [PATCH 111/190] CMake: `WarpX_PSATD` -> `WarpX_FFT` (#4912) * CMake: `WarpX_PSATD` -> `WarpX_FFT` Rename `WarpX_PSATD` to `WarpX_FFT` in the build configuration options. We now use FFTs for more than the PSATD solver since the introduction of the IGF solver for electrostatic simulations. * Change Define: `WARPX_USE_PSATD` * GNUMake: `USE_PSATD` -> `USE_FFT` * Temporary backwards compatibility We keep the flag `WarpX_PSATD` as an alias around for a few releases before we drop it. --- .github/workflows/clang_tidy.yml | 2 +- .github/workflows/cuda.yml | 6 +- .github/workflows/hip.yml | 4 +- .github/workflows/ubuntu.yml | 8 +- CMakeLists.txt | 28 ++- Docs/Doxyfile | 2 +- Docs/source/developers/gnumake.rst | 2 +- Docs/source/developers/gnumake/rzgeometry.rst | 6 +- Docs/source/developers/gnumake/spectral.rst | 4 +- Docs/source/install/cmake.rst | 8 +- Docs/source/install/hpc/adastra.rst | 4 +- Docs/source/install/hpc/crusher.rst | 4 +- Docs/source/install/hpc/frontier.rst | 4 +- Docs/source/install/hpc/greatlakes.rst | 4 +- Docs/source/install/hpc/hpc3.rst | 4 +- Docs/source/install/hpc/juwels.rst | 2 +- Docs/source/install/hpc/karolina.rst | 4 +- Docs/source/install/hpc/lassen.rst | 4 +- Docs/source/install/hpc/lawrencium.rst | 6 +- Docs/source/install/hpc/leonardo.rst | 4 +- Docs/source/install/hpc/lumi.rst | 4 +- Docs/source/install/hpc/perlmutter.rst | 8 +- Docs/source/install/hpc/polaris.rst | 4 +- Docs/source/install/hpc/quartz.rst | 4 +- Docs/source/install/hpc/spock.rst | 2 +- Docs/source/install/hpc/summit.rst | 4 +- Docs/source/usage/parameters.rst | 4 +- GNUmakefile | 3 +- Regression/WarpX-GPU-tests.ini | 10 +- Regression/WarpX-tests.ini | 220 +++++++++--------- Regression/prepare_file_ci.py | 4 +- Source/BoundaryConditions/PML.H | 8 +- Source/BoundaryConditions/PML.cpp | 8 +- Source/BoundaryConditions/PML_RZ.H | 6 +- Source/BoundaryConditions/PML_RZ.cpp | 4 +- Source/BoundaryConditions/WarpXEvolvePML.cpp | 4 +- .../ComputeDiagFunctors/RhoFunctor.cpp | 4 +- .../FlushFormats/FlushFormatCheckpoint.cpp | 4 +- Source/Diagnostics/WarpXIO.cpp | 4 +- Source/Evolve/WarpXEvolve.cpp | 6 +- Source/Evolve/WarpXOneStepImplicitPicard.cpp | 2 +- Source/FieldSolver/CMakeLists.txt | 2 +- Source/FieldSolver/Make.package | 2 +- .../PsatdAlgorithmComoving.H | 4 +- .../PsatdAlgorithmComoving.cpp | 4 +- .../PsatdAlgorithmFirstOrder.H | 4 +- .../PsatdAlgorithmFirstOrder.cpp | 4 +- .../PsatdAlgorithmJConstantInTime.H | 4 +- .../PsatdAlgorithmJConstantInTime.cpp | 4 +- .../PsatdAlgorithmJLinearInTime.H | 4 +- .../PsatdAlgorithmJLinearInTime.cpp | 4 +- .../SpectralAlgorithms/PsatdAlgorithmPml.H | 4 +- .../SpectralAlgorithms/PsatdAlgorithmPml.cpp | 4 +- .../SpectralBaseAlgorithm.H | 4 +- .../SpectralSolver/SpectralFieldData.cpp | 4 +- .../SpectralSolver/SpectralSolver.H | 4 +- .../SpectralSolver/SpectralSolver.cpp | 4 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 8 +- Source/Initialization/WarpXInitData.cpp | 12 +- Source/Make.WarpX | 4 +- Source/Parallelization/WarpXComm.cpp | 8 +- Source/Parallelization/WarpXRegrid.cpp | 4 +- Source/Python/WarpX.cpp | 2 +- Source/Utils/WarpXMovingWindow.cpp | 6 +- Source/Utils/WarpX_Complex.H | 4 +- Source/WarpX.H | 14 +- Source/WarpX.cpp | 24 +- Source/ablastr/fields/CMakeLists.txt | 2 +- Source/ablastr/fields/Make.package | 2 +- Source/ablastr/fields/PoissonSolver.H | 4 +- Source/ablastr/math/fft/CMakeLists.txt | 2 +- Source/ablastr/math/fft/Make.package | 2 +- Tools/Linter/runClangTidy.sh | 2 +- WarpXConfig.cmake | 13 +- cmake/WarpXFunctions.cmake | 4 +- setup.py | 4 +- 76 files changed, 311 insertions(+), 297 deletions(-) diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index 5abb42352f4..28b46b52530 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -43,7 +43,7 @@ jobs: -DWarpX_DIMS="${{ matrix.dim }}" \ -DWarpX_MPI=ON \ -DWarpX_COMPUTE=OMP \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index c58a35eace8..07422c36e15 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -70,7 +70,7 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build_sp -j 4 @@ -116,7 +116,7 @@ jobs: git clone https://github.com/AMReX-Codes/amrex.git ../amrex cd ../amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd - - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 + make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s du -hs ~/.cache/ccache @@ -168,7 +168,7 @@ jobs: -DWarpX_PYTHON=OFF \ -DAMReX_CUDA_ARCH=8.0 \ -DWarpX_OPENPMD=ON \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build -j 4 diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 0e311f061ef..0f6710b1405 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -55,7 +55,7 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_PSATD=ON + -DWarpX_FFT=ON cmake --build build_sp -j 4 export WARPX_MPI=OFF @@ -115,7 +115,7 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ - -DWarpX_PSATD=ON + -DWarpX_FFT=ON cmake --build build_2d -j 4 export WARPX_MPI=OFF diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 2997e9cdd16..9f4953b7a98 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -77,7 +77,7 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="1;2" \ -DWarpX_EB=OFF \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_QED_TOOLS=ON @@ -127,7 +127,7 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;3" \ -DWarpX_EB=OFF \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_PRECISION=SINGLE \ -DWarpX_PARTICLE_PRECISION=SINGLE \ -DWarpX_QED_TABLE_GEN=ON @@ -211,7 +211,7 @@ jobs: cmake -S . -B build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_APP=OFF \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_PYTHON=ON \ -DWarpX_QED_TABLE_GEN=ON cmake --build build -j 4 --target pip_install @@ -260,7 +260,7 @@ jobs: -GNinja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;1;2;3" \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ diff --git a/CMakeLists.txt b/CMakeLists.txt index 31b410fc55e..85b4a42b040 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,6 +16,12 @@ endif() # CMake policies ############################################################## # +# Setting a cmake_policy to OLD is deprecated by definition and will raise a +# verbose warning +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "" FORCE) +endif() + # AMReX 21.06+ supports CUDA_ARCHITECTURES with CMake 3.20+ # CMake 3.18+: CMAKE_CUDA_ARCHITECTURES # https://cmake.org/cmake/help/latest/policy/CMP0104.html @@ -72,7 +78,7 @@ option(WarpX_EB "Embedded boundary support" OFF) option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) -option(WarpX_PSATD "spectral solver support" OFF) +option(WarpX_FFT "FFT-based solvers" OFF) option(WarpX_PYTHON "Python bindings" OFF) option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF) option(WarpX_QED "QED support (requires PICSAR)" ON) @@ -149,6 +155,14 @@ if(WarpX_APP OR WarpX_PYTHON) set(WarpX_LIB ON CACHE STRING "Build WarpX as a library" FORCE) endif() +# deprecated options: transition phase + +if(DEFINED WarpX_PSATD) + message(WARNING "CMake option WarpX_PSATD is deprecated. Use WarpX_FFT instead.\n" + "Overwriting WarpX_FFT with '${WarpX_PSATD}' because WarpX_PSATD was set.") + set(WarpX_FFT ${WarpX_PSATD} CACHE STRING "FFT-based solvers" FORCE) +endif() + # note: we could skip this if we solely build WarpX_APP, but if we build a # shared WarpX library or a third party, like ImpactX, uses ablastr in a # shared library (e.g., for Python bindings), then we need relocatable code. @@ -156,9 +170,9 @@ option(ABLASTR_POSITION_INDEPENDENT_CODE "Build ABLASTR with position independent code" ON) mark_as_advanced(ABLASTR_POSITION_INDEPENDENT_CODE) -option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_PSATD}) -if(WarpX_PSATD) - set(ABLASTR_FFT ON CACHE STRING "compile AnyFFT wrappers" FORCE) +option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT}) +if(WarpX_FFT) + set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE) endif() # this defined the variable BUILD_TESTING which is ON by default @@ -192,7 +206,7 @@ include(${WarpX_SOURCE_DIR}/cmake/dependencies/openPMD.cmake) # PSATD include(${WarpX_SOURCE_DIR}/cmake/dependencies/FFT.cmake) -if(WarpX_PSATD) +if(WarpX_FFT) # BLASPP and LAPACKPP if(RZ IN_LIST WarpX_DIMS) find_package(blaspp CONFIG REQUIRED) @@ -517,8 +531,8 @@ foreach(D IN LISTS WarpX_DIMS) endif() endif() - if(WarpX_PSATD) - target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_PSATD) + if(WarpX_FFT) + target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_FFT) endif() if(ABLASTR_FFT) # We need to enable FFT support in ABLASTR for PSATD solver diff --git a/Docs/Doxyfile b/Docs/Doxyfile index 550ed9a7a01..5fbb7651b18 100644 --- a/Docs/Doxyfile +++ b/Docs/Doxyfile @@ -2301,7 +2301,7 @@ PREDEFINED = AMREX_Linux=1 \ WARPX_DIM_XZ=1 \ WARPX_USE_GPU=1 \ WARPX_USE_OPENPMD=1 \ - WARPX_USE_PSATD=1 \ + WARPX_USE_FFT=1 \ WARPX_QED=1 \ WARPX_QED_TABLE_GEN=1 diff --git a/Docs/source/developers/gnumake.rst b/Docs/source/developers/gnumake.rst index ca51dacfbd5..6324ceb4ac7 100644 --- a/Docs/source/developers/gnumake.rst +++ b/Docs/source/developers/gnumake.rst @@ -56,7 +56,7 @@ options are: * ``DIM=3`` or ``2``: Geometry of the simulation (note that running an executable compiled for 3D with a 2D input file will crash). * ``DEBUG=FALSE`` or ``TRUE``: Compiling in ``DEBUG`` mode can help tremendously during code development. - * ``USE_PSATD=FALSE`` or ``TRUE``: Compile the Pseudo-Spectral Analytical Time Domain Maxwell solver. Requires an FFT library. + * ``USE_FFT=FALSE`` or ``TRUE``: Compile the Pseudo-Spectral Analytical Time Domain Maxwell solver. Requires an FFT library. * ``USE_RZ=FALSE`` or ``TRUE``: Compile for 2D axisymmetric geometry. * ``COMP=gcc`` or ``intel``: Compiler. * ``USE_MPI=TRUE`` or ``FALSE``: Whether to compile with MPI support. diff --git a/Docs/source/developers/gnumake/rzgeometry.rst b/Docs/source/developers/gnumake/rzgeometry.rst index ac90fb754f2..a5c0993a805 100644 --- a/Docs/source/developers/gnumake/rzgeometry.rst +++ b/Docs/source/developers/gnumake/rzgeometry.rst @@ -17,7 +17,7 @@ RZ geometry with spectral solver Additional steps are needed to build the spectral solver. Some of the steps are the same as is done for the Cartesian spectral solver, setting up the FFTW -package and setting ``USE_PSATD=TRUE``. +package and setting ``USE_FFT=TRUE``. - Install (or load) an MPI-enabled version of FFTW. For instance, for Debian, this can be done with @@ -54,7 +54,7 @@ package and setting ``USE_PSATD=TRUE``. export BLAS_LIB=-lblas - - Set ``USE_PSATD=TRUE`` when compiling: + - Set ``USE_FFT=TRUE`` when compiling: :: - make -j 4 USE_RZ=TRUE USE_PSATD=TRUE + make -j 4 USE_RZ=TRUE USE_FFT=TRUE diff --git a/Docs/source/developers/gnumake/spectral.rst b/Docs/source/developers/gnumake/spectral.rst index a5a0e6448f5..bb0baf01c09 100644 --- a/Docs/source/developers/gnumake/spectral.rst +++ b/Docs/source/developers/gnumake/spectral.rst @@ -16,10 +16,10 @@ In order to run the code with a spectral solver, you need to: export FFTW_HOME=/usr/ - - Set ``USE_PSATD=TRUE`` when compiling: + - Set ``USE_FFT=TRUE`` when compiling: :: - make -j 4 USE_PSATD=TRUE + make -j 4 USE_FFT=TRUE See :doc:`rzgeometry` for using the spectral solver with USE_RZ. Additional steps are needed. PSATD is compatible with single precision, but please note that, on CPU, FFTW needs to be compiled with option ``--enable-float``. diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 006cd843b31..b9fd0b2be45 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -96,7 +96,7 @@ CMake Option Default & Values Descr ``WarpX_OPENPMD`` **ON**/OFF openPMD I/O (HDF5, ADIOS) ``WarpX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WarpX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set -``WarpX_PSATD`` ON/**OFF** Spectral solver +``WarpX_FFT`` ON/**OFF** FFT-based solvers ``WarpX_PYTHON`` ON/**OFF** Python bindings ``WarpX_QED`` **ON**/OFF QED support (requires PICSAR) ``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost) @@ -270,7 +270,7 @@ Environment Variable Default & Values Descr ``WARPX_OPENPMD`` **ON**/OFF openPMD I/O (HDF5, ADIOS) ``WARPX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WARPX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set -``WARPX_PSATD`` ON/**OFF** Spectral solver +``WARPX_FFT`` ON/**OFF** FFT-based solvers ``WARPX_QED`` **ON**/OFF PICSAR QED (requires PICSAR) ``WARPX_QED_TABLE_GEN`` ON/**OFF** QED table generation (requires PICSAR and Boost) ``BUILD_PARALLEL`` ``2`` Number of threads to use for parallel builds @@ -310,11 +310,11 @@ Some Developers like to code directly against a local copy of AMReX, changing bo WARPX_AMREX_SRC=$PWD/../amrex python3 -m pip install --force-reinstall --no-deps -v . Additional environment control as common for CMake (:ref:`see above `) can be set as well, e.g. ``CC``, `CXX``, and ``CMAKE_PREFIX_PATH`` hints. -So another sophisticated example might be: use Clang as the compiler, build with local source copies of PICSAR and AMReX, support the PSATD solver, MPI and openPMD, hint a parallel HDF5 installation in ``$HOME/sw/hdf5-parallel-1.10.4``, and only build 2D and 3D geometry: +So another sophisticated example might be: use Clang as the compiler, build with local source copies of PICSAR and AMReX, support the FFT-based solvers, MPI and openPMD, hint a parallel HDF5 installation in ``$HOME/sw/hdf5-parallel-1.10.4``, and only build 2D and 3D geometry: .. code-block:: bash - CC=$(which clang) CXX=$(which clang++) WARPX_AMREX_SRC=$PWD/../amrex WARPX_PICSAR_SRC=$PWD/../picsar WARPX_PSATD=ON WARPX_MPI=ON WARPX_DIMS="2;3" CMAKE_PREFIX_PATH=$HOME/sw/hdf5-parallel-1.10.4:$CMAKE_PREFIX_PATH python3 -m pip install --force-reinstall --no-deps -v . + CC=$(which clang) CXX=$(which clang++) WARPX_AMREX_SRC=$PWD/../amrex WARPX_PICSAR_SRC=$PWD/../picsar WARPX_FFT=ON WARPX_MPI=ON WARPX_DIMS="2;3" CMAKE_PREFIX_PATH=$HOME/sw/hdf5-parallel-1.10.4:$CMAKE_PREFIX_PATH python3 -m pip install --force-reinstall --no-deps -v . Here we wrote this all in one line, but one can also set all environment variables in a development environment and keep the pip call nice and short as in the beginning. Note that you need to use absolute paths for external source trees, because pip builds in a temporary directory, e.g. ``export WARPX_AMREX_SRC=$HOME/src/amrex``. diff --git a/Docs/source/install/hpc/adastra.rst b/Docs/source/install/hpc/adastra.rst index 0b984d5e2be..e47cb5f535e 100644 --- a/Docs/source/install/hpc/adastra.rst +++ b/Docs/source/install/hpc/adastra.rst @@ -100,7 +100,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $SHAREDHOMEDIR/src/warpx rm -rf build_adastra - cmake -S . -B build_adastra -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_adastra -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_adastra -j 16 The WarpX application executables are now in ``$SHAREDHOMEDIR/src/warpx/build_adastra/bin/``. @@ -110,7 +110,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_adastra_py - cmake -S . -B build_adastra_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_adastra_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_adastra_py -j 16 --target pip_install Now, you can :ref:`submit Adstra compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/crusher.rst b/Docs/source/install/hpc/crusher.rst index 1aa471ce68b..7a657e8a51c 100644 --- a/Docs/source/install/hpc/crusher.rst +++ b/Docs/source/install/hpc/crusher.rst @@ -99,7 +99,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_crusher - cmake -S . -B build_crusher -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_crusher -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_crusher -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_crusher/bin/``. @@ -109,7 +109,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_crusher_py - cmake -S . -B build_crusher_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_crusher_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_crusher_py -j 16 --target pip_install Now, you can :ref:`submit Crusher compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/frontier.rst b/Docs/source/install/hpc/frontier.rst index e9f8b687d6b..c33c2fc7401 100644 --- a/Docs/source/install/hpc/frontier.rst +++ b/Docs/source/install/hpc/frontier.rst @@ -99,7 +99,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_frontier - cmake -S . -B build_frontier -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_frontier -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_frontier -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_frontier/bin/``. @@ -109,7 +109,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_frontier_py - cmake -S . -B build_frontier_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_frontier_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_frontier_py -j 16 --target pip_install Now, you can :ref:`submit Frontier compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/greatlakes.rst b/Docs/source/install/hpc/greatlakes.rst index d238d498e5c..9691134f571 100644 --- a/Docs/source/install/hpc/greatlakes.rst +++ b/Docs/source/install/hpc/greatlakes.rst @@ -116,7 +116,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_v100 - cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_v100 -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build_v100/bin/``. @@ -127,7 +127,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_v100_py - cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_v100_py -j 8 --target pip_install diff --git a/Docs/source/install/hpc/hpc3.rst b/Docs/source/install/hpc/hpc3.rst index e2f900de273..5d398b1f4f5 100644 --- a/Docs/source/install/hpc/hpc3.rst +++ b/Docs/source/install/hpc/hpc3.rst @@ -95,7 +95,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build/bin/``. @@ -105,7 +105,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_py - cmake -S . -B build_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_py -j 8 --target pip_install Now, you can :ref:`submit HPC3 compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/juwels.rst b/Docs/source/install/hpc/juwels.rst index a8d820ccd46..5358e649825 100644 --- a/Docs/source/install/hpc/juwels.rst +++ b/Docs/source/install/hpc/juwels.rst @@ -57,7 +57,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_MPI_THREAD_MULTIPLE=OFF + cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_MPI_THREAD_MULTIPLE=OFF cmake --build build -j 16 The other :ref:`general compile-time options ` apply as usual. diff --git a/Docs/source/install/hpc/karolina.rst b/Docs/source/install/hpc/karolina.rst index fffb917df1c..127b81477da 100644 --- a/Docs/source/install/hpc/karolina.rst +++ b/Docs/source/install/hpc/karolina.rst @@ -80,7 +80,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $WORK/src/warpx rm -rf build_gpu - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu -j 48 The WarpX application executables are now in ``$WORK/src/warpx/build_gpu/bin/``. @@ -91,7 +91,7 @@ Additionally, the following commands will install WarpX as a Python module: cd $WORK/src/warpx rm -rf build_gpu_py - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu_py -j 48 --target pip_install Now, you can :ref:`submit Karolina compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index d8565f06983..0d885dff04d 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -200,7 +200,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd /usr/workspace/${USER}/lassen/src/warpx rm -rf build_lassen - cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lassen -j 8 The WarpX application executables are now in ``/usr/workspace/${USER}/lassen/src/warpx/build_lassen/bin/``. @@ -210,7 +210,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_lassen_py - cmake -S . -B build_lassen_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lassen_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lassen_py -j 8 --target pip_install Now, you can :ref:`submit lassen compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/lawrencium.rst b/Docs/source/install/hpc/lawrencium.rst index 23be9b75431..2217c5a31ce 100644 --- a/Docs/source/install/hpc/lawrencium.rst +++ b/Docs/source/install/hpc/lawrencium.rst @@ -102,7 +102,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_DIMS="1;2;RZ;3" -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON + cmake -S . -B build -DWarpX_DIMS="1;2;RZ;3" -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON cmake --build build -j 12 The general :ref:`cmake compile-time options ` apply as usual. @@ -122,14 +122,14 @@ For a *full PICMI install*, follow the :ref:`instructions for Python (PICMI) bin python3 -m pip install -r requirements.txt # compile parallel PICMI interfaces in 3D, 2D, 1D and RZ - WARPX_MPI=ON WARPX_COMPUTE=CUDA WARPX_PSATD=ON BUILD_PARALLEL=12 python3 -m pip install --force-reinstall --no-deps -v . + WARPX_MPI=ON WARPX_COMPUTE=CUDA WARPX_FFT=ON BUILD_PARALLEL=12 python3 -m pip install --force-reinstall --no-deps -v . Or, if you are *developing*, do a quick PICMI install of a *single geometry* (see: :ref:`WarpX_DIMS `) using: .. code-block:: bash # find dependencies & configure - cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS=RZ + cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS=RZ # build and then call "python3 -m pip install ..." cmake --build build --target pip_install -j 12 diff --git a/Docs/source/install/hpc/leonardo.rst b/Docs/source/install/hpc/leonardo.rst index 1f3c1e8fbfa..568a5612250 100644 --- a/Docs/source/install/hpc/leonardo.rst +++ b/Docs/source/install/hpc/leonardo.rst @@ -89,7 +89,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_gpu - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``. @@ -100,7 +100,7 @@ Additionally, the following commands will install WarpX as a Python module: cd $HOME/src/warpx rm -rf build_gpu_py - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu_py -j 16 --target pip_install Now, you can :ref:`submit Leonardo compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/lumi.rst b/Docs/source/install/hpc/lumi.rst index 2bae4d21599..2d155dab222 100644 --- a/Docs/source/install/hpc/lumi.rst +++ b/Docs/source/install/hpc/lumi.rst @@ -98,7 +98,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_lumi - cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_lumi/bin/``. @@ -108,7 +108,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_lumi_py - cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi_py -j 16 --target pip_install diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst index 14972566abc..9612b64476d 100644 --- a/Docs/source/install/hpc/perlmutter.rst +++ b/Docs/source/install/hpc/perlmutter.rst @@ -153,7 +153,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu - cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. @@ -164,7 +164,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu_py - cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu_py -j 16 --target pip_install .. tab-item:: CPU Nodes @@ -174,7 +174,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_cpu - cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_cpu/bin/``. @@ -184,7 +184,7 @@ Use the following :ref:`cmake commands ` to compile the applicat rm -rf build_pm_cpu_py - cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu_py -j 16 --target pip_install Now, you can :ref:`submit Perlmutter compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/polaris.rst b/Docs/source/install/hpc/polaris.rst index d20ecccee32..a2d95a144d6 100644 --- a/Docs/source/install/hpc/polaris.rst +++ b/Docs/source/install/hpc/polaris.rst @@ -101,7 +101,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu - cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. @@ -112,7 +112,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu_py - cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu_py -j 16 --target pip_install .. tab-item:: CPU Nodes diff --git a/Docs/source/install/hpc/quartz.rst b/Docs/source/install/hpc/quartz.rst index 03860687971..a49327e8613 100644 --- a/Docs/source/install/hpc/quartz.rst +++ b/Docs/source/install/hpc/quartz.rst @@ -93,7 +93,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_quartz - cmake -S . -B build_quartz -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_quartz -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_quartz -j 6 The WarpX application executables are now in ``$HOME/src/warpx/build_quartz/bin/``. @@ -103,7 +103,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_quartz_py - cmake -S . -B build_quartz_py -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_quartz_py -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_quartz_py -j 6 --target pip_install Now, you can :ref:`submit Quartz compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/spock.rst b/Docs/source/install/hpc/spock.rst index 64f307f9dae..144257289f1 100644 --- a/Docs/source/install/hpc/spock.rst +++ b/Docs/source/install/hpc/spock.rst @@ -51,7 +51,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DAMReX_AMD_ARCH=gfx908 -DMPI_CXX_COMPILER=$(which CC) -DMPI_C_COMPILER=$(which cc) -DMPI_COMPILER_FLAGS="--cray-print-opts=all" + cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DAMReX_AMD_ARCH=gfx908 -DMPI_CXX_COMPILER=$(which CC) -DMPI_C_COMPILER=$(which cc) -DMPI_COMPILER_FLAGS="--cray-print-opts=all" cmake --build build -j 10 The general :ref:`cmake compile-time options ` apply as usual. diff --git a/Docs/source/install/hpc/summit.rst b/Docs/source/install/hpc/summit.rst index 918da842165..bf226df69f2 100644 --- a/Docs/source/install/hpc/summit.rst +++ b/Docs/source/install/hpc/summit.rst @@ -124,7 +124,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_summit - cmake -S . -B build_summit -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_summit -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_summit -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build_summit/bin/``. @@ -134,7 +134,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_summit_py - cmake -S . -B build_summit_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_summit_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_summit_py -j 8 --target pip_install Now, you can :ref:`submit Summit compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 4f011025213..f3a74f969a0 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -173,7 +173,7 @@ Overall simulation parameters * ``fft``: Poisson's equation is solved using an Integrated Green Function method (which requires FFT calculations). See these references for more details :cite:t:`QiangPhysRevSTAB2006`, :cite:t:`QiangPhysRevSTAB2006err`. - It only works in 3D and it requires the compilation flag ``-DWarpX_PSATD=ON``. + It only works in 3D and it requires the compilation flag ``-DWarpX_FFT=ON``. If mesh refinement is enabled, this solver only works on the coarsest level. On the refined patches, the Poisson equation is solved with the multigrid solver. In electrostatic mode, this solver requires open field boundary conditions (``boundary.field_lo,hi = open``). @@ -2256,7 +2256,7 @@ Maxwell solver: PSATD method * ``psatd.v_comoving`` (3 floating-point values, in units of the speed of light; default ``0. 0. 0.``) Defines the comoving velocity in the comoving PSATD scheme. - A non-zero comoving velocity selects the comoving PSATD algorithm, which suppresses the numerical Cherenkov instability (NCI) in boosted-frame simulations, under certain assumptions. This option requires that WarpX is compiled with ``USE_PSATD = TRUE``. It also requires the use of direct current deposition (``algo.current_deposition = direct``) and has not been neither implemented nor tested with other current deposition schemes. + A non-zero comoving velocity selects the comoving PSATD algorithm, which suppresses the numerical Cherenkov instability (NCI) in boosted-frame simulations, under certain assumptions. This option requires that WarpX is compiled with ``USE_FFT = TRUE``. It also requires the use of direct current deposition (``algo.current_deposition = direct``) and has not been neither implemented nor tested with other current deposition schemes. * ``psatd.do_time_averaging`` (`0` or `1`; default: 0) Whether to use an averaged Galilean PSATD algorithm or standard Galilean PSATD. diff --git a/GNUmakefile b/GNUmakefile index 863d37dd56f..86bdab2709f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -37,8 +37,7 @@ USE_OPENPMD = FALSE WarpxBinDir = Bin -USE_PSATD = FALSE -USE_PSATD_PICSAR = FALSE +USE_FFT = FALSE USE_RZ = FALSE USE_EB = FALSE diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 12b0a0fdfcb..8542e5f35d9 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -107,7 +107,7 @@ analysisRoutine = Examples/Tests/pml/analysis_pml_ckc.py #inputFile = Examples/Tests/pml/inputs_2d #runtime_params = algo.maxwell_solver=psatd warpx.do_dynamic_scheduling=0 #dim = 2 -#addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +#addToCompileString = USE_FFT=TRUE USE_GPU=TRUE #restartTest = 0 #useMPI = 1 #numprocs = 2 @@ -377,7 +377,7 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d_multi_rt runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +addToCompileString = USE_FFT=TRUE USE_GPU=TRUE restartTest = 0 useMPI = 1 numprocs = 2 @@ -395,7 +395,7 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d_multi_rt runtime_params = algo.maxwell_solver=psatd warpx.do_dynamic_scheduling=0 warpx.grid_type=collocated algo.current_deposition=direct warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +addToCompileString = USE_FFT=TRUE USE_GPU=TRUE restartTest = 0 useMPI = 1 numprocs = 2 @@ -431,7 +431,7 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d_multi_rt runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +addToCompileString = USE_FFT=TRUE USE_GPU=TRUE restartTest = 0 useMPI = 1 numprocs = 2 @@ -449,7 +449,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png # inputFile = Examples/Tests/langmuir/inputs_2d_multi_rt # runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell # dim = 2 -# addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +# addToCompileString = USE_FFT=TRUE USE_GPU=TRUE # restartTest = 0 # useMPI = 1 # numprocs = 4 diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 58d97e4547c..6ee7ad3929d 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -74,8 +74,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_2d runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -92,8 +92,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_2d runtime_params = amr.max_grid_size_x=128 amr.max_grid_size_y=64 warpx.grid_type=hybrid psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -110,8 +110,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_3d runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -128,8 +128,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_3d runtime_params = warpx.grid_type=hybrid psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -247,8 +247,8 @@ buildDir = . inputFile = Examples/Tests/collision/inputs_rz runtime_params = dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=FALSE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=OFF +addToCompileString = USE_RZ=TRUE USE_FFT=FALSE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=OFF restartTest = 0 useMPI = 1 numprocs = 1 @@ -301,8 +301,8 @@ buildDir = . inputFile = Examples/Tests/comoving/inputs_2d_hybrid runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -798,8 +798,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d runtime_params = warpx.grid_type=collocated algo.current_deposition=direct psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -816,8 +816,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d runtime_params = psatd.periodic_single_box_fft=0 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE amr.max_grid_size=64 amr.blocking_factor=64 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -834,8 +834,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d runtime_params = psatd.periodic_single_box_fft=1 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -852,8 +852,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d_hybrid runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -870,8 +870,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -888,8 +888,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 warpx.numprocs=1 1 2 psatd.periodic_single_box_fft=0 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -906,8 +906,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 warpx.numprocs=1 1 1 psatd.periodic_single_box_fft=1 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -924,8 +924,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_rz runtime_params = electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -942,8 +942,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_rz runtime_params = psatd.periodic_single_box_fft=1 psatd.current_correction=1 electrons.random_theta=0 ions.random_theta=0 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -960,8 +960,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_rz runtime_params = psatd.periodic_single_box_fft=0 psatd.current_correction=1 electrons.random_theta=0 ions.random_theta=0 amr.max_grid_size=32 amr.blocking_factor=32 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1284,8 +1284,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver = psatd warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1322,8 +1322,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1341,8 +1341,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1360,8 +1360,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1379,8 +1379,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1398,8 +1398,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.7071067811865475 psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1417,8 +1417,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.7071067811865475 psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium warpx.grid_type=collocated dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1436,8 +1436,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1455,8 +1455,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1474,8 +1474,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1493,8 +1493,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 warpx.grid_type=collocated algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1531,8 +1531,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1550,8 +1550,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1569,8 +1569,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1588,8 +1588,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 psatd.update_with_rho = 1 algo.current_deposition = direct warpx.do_dive_cleaning = 1 warpx.do_divb_cleaning = 1 diag1.intervals = 0, 38:40:1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE F warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1607,8 +1607,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1626,8 +1626,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.5773502691896258 algo.current_deposition=direct psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1645,8 +1645,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.5773502691896258 algo.current_deposition=direct psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium warpx.grid_type=collocated dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1664,8 +1664,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct warpx.cfl = 0.5773502691896258 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1683,8 +1683,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON -DWarpX_PRECISION=SINGLE +addToCompileString = USE_FFT=TRUE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_PRECISION=SINGLE restartTest = 0 useMPI = 1 numprocs = 2 @@ -1702,8 +1702,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.current_deposition=vay diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1721,8 +1721,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=vay diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1760,8 +1760,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -1780,8 +1780,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 amr.max_grid_size=128 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot=jr jz Er Ez Bt rho divE electrons.random_theta=0 ions.random_theta=0 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 @@ -1800,8 +1800,8 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz runtime_params = amr.max_grid_size=32 algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 warpx.n_rz_azimuthal_modes=2 electrons.random_theta=0 electrons.num_particles_per_cell_each_dim=2 4 2 ions.random_theta=0 ions.num_particles_per_cell_each_dim=2 4 2 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=4 warpx.use_filter=1 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2307,8 +2307,8 @@ buildDir = . inputFile = Examples/Tests/maxwell_hybrid_qed/inputs_2d runtime_params = warpx.cfl=0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2341,8 +2341,8 @@ buildDir = . inputFile = Examples/Tests/multi_j/inputs_rz runtime_params = warpx.abort_on_warning_threshold=medium psatd.J_in_time=linear dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2896,8 +2896,8 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_3d runtime_params = warpx.do_similar_dm_pml=0 warpx.abort_on_warning_threshold=medium ablastr.fillboundary_always_sync=1 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2912,8 +2912,8 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_rz runtime_params = warpx.cfl=0.7 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2944,8 +2944,8 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_2d runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz rho divE warpx.cfl=0.7071067811865475 warpx.do_pml_dive_cleaning=1 warpx.do_pml_divb_cleaning=1 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium psatd.v_galilean=0. 0. 0.99 warpx.grid_type=collocated dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -2960,8 +2960,8 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_2d runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE warpx.cfl = 0.7071067811865475 warpx.do_pml_dive_cleaning=0 warpx.do_pml_divb_cleaning=0 chk.file_prefix=pml_x_psatd_chk chk.file_min_digits=5 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 1 restartFileNum = 150 useMPI = 1 @@ -3865,8 +3865,8 @@ inputFile = Examples/Tests/python_wrappers/PICMI_inputs_2d.py runtime_params = customRunCmd = python3 PICMI_inputs_2d.py dim = 2 -addToCompileString = USE_PSATD=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON +addToCompileString = USE_FFT=TRUE USE_PYTHON_MAIN=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON target = pip_install restartTest = 0 useMPI = 1 @@ -4124,8 +4124,8 @@ buildDir = . inputFile = Examples/Tests/reduced_diags/inputs_loadbalancecosts runtime_params = algo.load_balance_costs_update=Timers dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -4264,8 +4264,8 @@ buildDir = . inputFile = Examples/Tests/restart/inputs runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_chk chk.file_min_digits=5 boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 1 restartFileNum = 5 useMPI = 1 @@ -4283,8 +4283,8 @@ buildDir = . inputFile = Examples/Tests/restart/inputs runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_time_avg_chk chk.file_min_digits=5 boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped psatd.do_time_averaging=1 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 1 restartFileNum = 5 useMPI = 1 @@ -4523,8 +4523,8 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.solution_type=first-order psatd.J_in_time=constant psatd.rho_in_time=constant warpx.do_dive_cleaning=1 warpx.do_divb_cleaning=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=1 diag1.fields_to_plot=Bx By Bz divE Ex Ey Ez F G jx jy jz rho warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -4541,8 +4541,8 @@ buildDir = . inputFile = Examples/Tests/vay_deposition/inputs_2d runtime_params = dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -4559,8 +4559,8 @@ buildDir = . inputFile = Examples/Tests/vay_deposition/inputs_3d runtime_params = dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 @@ -4794,8 +4794,8 @@ buildDir = . inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d runtime_params = warpx.abort_on_warning_threshold = high dim = 3 -addToCompileString = USE_OPENPMD=TRUE USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON -DWarpX_OPENPMD=ON +addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON restartTest = 0 useMPI = 1 numprocs = 2 diff --git a/Regression/prepare_file_ci.py b/Regression/prepare_file_ci.py index ce7a3398b95..f7b610dace9 100644 --- a/Regression/prepare_file_ci.py +++ b/Regression/prepare_file_ci.py @@ -59,8 +59,8 @@ # always build with PSATD support (runtime controlled if used) if ci_psatd: text = re.sub('addToCompileString =', - 'addToCompileString = USE_PSATD=TRUE ', text) - text = re.sub('USE_PSATD=FALSE', + 'addToCompileString = USE_FFT=TRUE ', text) + text = re.sub('USE_FFT=FALSE', '', text) # CCache diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 42a96f3628a..7ed1def8f72 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -13,7 +13,7 @@ #include "Utils/WarpXAlgorithmSelection.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralSolver.H" #endif @@ -179,7 +179,7 @@ public: return *sigba_cp; } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPSATD (int lev); #endif @@ -245,7 +245,7 @@ private: std::unique_ptr sigba_fp; std::unique_ptr sigba_cp; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT std::unique_ptr spectral_solver_fp; std::unique_ptr spectral_solver_cp; #endif @@ -289,7 +289,7 @@ private: static void CopyToPML (amrex::MultiFab& pml, amrex::MultiFab& reg, const amrex::Geometry& geom); }; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPMLPSATDSinglePatch( int lev, SpectralSolver& solver, std::array,3>& pml_E, diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 25b34818dd1..ed82cc07bb9 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -11,7 +11,7 @@ #include "BoundaryConditions/PML.H" #include "BoundaryConditions/PMLComponent.H" #include "FieldSolver/Fields.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralFieldData.H" #endif #include "Utils/TextMsg.H" @@ -743,7 +743,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, IntVect(ncell), IntVect(delta), single_domain_box, v_sigma_sb); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT amrex::ignore_unused(lev, dt, psatd_solution_type, J_in_time, rho_in_time); # if(AMREX_SPACEDIM!=3) amrex::ignore_unused(noy_fft); @@ -858,7 +858,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, cncells, cdelta, single_domain_box, v_sigma_sb); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT amrex::ignore_unused(dt); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, "PML: PSATD solver selected but not built."); @@ -1367,7 +1367,7 @@ PML::Restart (const std::string& dir) } } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PML::PushPSATD (const int lev) { diff --git a/Source/BoundaryConditions/PML_RZ.H b/Source/BoundaryConditions/PML_RZ.H index 74d3da2e19b..c908681d8e5 100644 --- a/Source/BoundaryConditions/PML_RZ.H +++ b/Source/BoundaryConditions/PML_RZ.H @@ -12,7 +12,7 @@ #include "Utils/WarpXAlgorithmSelection.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #endif @@ -40,7 +40,7 @@ public: std::array GetE_fp (); std::array GetB_fp (); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPSATD (int lev); #endif @@ -62,7 +62,7 @@ private: std::array,2> pml_E_fp; std::array,2> pml_B_fp; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPMLPSATDSinglePatchRZ ( int lev, SpectralSolverRZ& solver, std::array,2>& pml_E, diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index faa7609b3ab..78f3cf24987 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -9,7 +9,7 @@ #include "BoundaryConditions/PML_RZ.H" #include "FieldSolver/Fields.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" #endif #include "Utils/WarpXConst.H" @@ -187,7 +187,7 @@ PML_RZ::Restart (const std::string& dir) } } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PML_RZ::PushPSATD (const int lev) { diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index af721d70b6d..ec696689373 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -8,7 +8,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "PML_current.H" @@ -60,7 +60,7 @@ WarpX::DampPML (const int lev, PatchType patch_type) if (!do_pml) { return; } WARPX_PROFILE("WarpX::DampPML()"); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->ApplyDamping(Efield_fp[lev][1].get(), Efield_fp[lev][2].get(), Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get(), diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp index 340cc635fea..32e11903778 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp @@ -1,7 +1,7 @@ #include "RhoFunctor.H" #include "Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) #include "FieldSolver/SpectralSolver/SpectralFieldData.H" #include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -60,7 +60,7 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i // apply the filtering if requested. warpx.ApplyFilterandSumBoundaryRho(m_lev, m_lev, *rho, 0, rho->nComp()); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) // Apply k-space filtering when using the PSATD solver if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 27d0ad84b9c..1a3318ae0d8 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -1,7 +1,7 @@ #include "FlushFormatCheckpoint.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/ParticleDiag/ParticleDiag.H" @@ -152,7 +152,7 @@ FlushFormatCheckpoint::WriteToFile ( warpx.GetPML(lev)->CheckPoint( amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "pml")); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (warpx.GetPML_RZ(lev)) { warpx.GetPML_RZ(lev)->CheckPoint( amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "pml_rz")); diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 91409bc294d..a17abe04178 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -8,7 +8,7 @@ * License: BSD-3-Clause-LBNL */ #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "FieldIO.H" @@ -385,7 +385,7 @@ WarpX::InitFromCheckpoint () if (pml[lev]) { pml[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index de586895c2d..881967a0479 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -15,7 +15,7 @@ #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "Evolve/WarpXDtType.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" # else @@ -603,7 +603,7 @@ void WarpX::SyncCurrentAndRho () void WarpX::OneStep_multiJ (const amrex::Real cur_time) { -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, @@ -766,7 +766,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) amrex::ignore_unused(cur_time); WARPX_ABORT_WITH_MESSAGE( "multi-J algorithm not implemented for FDTD"); -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT } /* /brief Perform one PIC iteration, with subcycling diff --git a/Source/Evolve/WarpXOneStepImplicitPicard.cpp b/Source/Evolve/WarpXOneStepImplicitPicard.cpp index 22d42dfe89d..6df3cd0d4e6 100644 --- a/Source/Evolve/WarpXOneStepImplicitPicard.cpp +++ b/Source/Evolve/WarpXOneStepImplicitPicard.cpp @@ -11,7 +11,7 @@ #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "Evolve/WarpXDtType.H" #include "Evolve/WarpXPushType.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" # else diff --git a/Source/FieldSolver/CMakeLists.txt b/Source/FieldSolver/CMakeLists.txt index 58dd10f41cf..cf741aa6e01 100644 --- a/Source/FieldSolver/CMakeLists.txt +++ b/Source/FieldSolver/CMakeLists.txt @@ -11,6 +11,6 @@ endforeach() add_subdirectory(FiniteDifferenceSolver) add_subdirectory(MagnetostaticSolver) -if(WarpX_PSATD) +if(WarpX_FFT) add_subdirectory(SpectralSolver) endif() diff --git a/Source/FieldSolver/Make.package b/Source/FieldSolver/Make.package index c1535ae2f22..3b88f4627e5 100644 --- a/Source/FieldSolver/Make.package +++ b/Source/FieldSolver/Make.package @@ -2,7 +2,7 @@ CEXE_sources += WarpXPushFieldsEM.cpp CEXE_sources += WarpXPushFieldsHybridPIC.cpp CEXE_sources += ElectrostaticSolver.cpp CEXE_sources += WarpX_QED_Field_Pushers.cpp -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) include $(WARPX_HOME)/Source/FieldSolver/SpectralSolver/Make.package endif include $(WARPX_HOME)/Source/FieldSolver/FiniteDifferenceSolver/Make.package diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H index 87dfe6f2d9a..70468277749 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H @@ -14,7 +14,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* \brief Class that updates the field in spectral space and stores the coefficients * of the corresponding update equation, according to the comoving spectral scheme. @@ -86,5 +86,5 @@ class PsatdAlgorithmComoving : public SpectralBaseAlgorithm amrex::Real m_dt; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_COMOVING_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp index 640cf1b82af..6da6641c052 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp @@ -19,7 +19,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -505,4 +505,4 @@ PsatdAlgorithmComoving::VayDeposition (SpectralFieldData& /*field_data*/) "Vay deposition not implemented for comoving PSATD"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H index 8fde50d8172..23d4c2f82e3 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H @@ -20,7 +20,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* * \brief Class that updates the fields in spectral space according to the first-order PSATD equations. */ @@ -94,5 +94,5 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm int m_J_in_time; int m_rho_in_time; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_FIRST_ORDER_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp index 6d4d8613940..041b70b8edf 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp @@ -24,7 +24,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex::literals; @@ -371,4 +371,4 @@ PsatdAlgorithmFirstOrder::VayDeposition (SpectralFieldData& field_data) "Vay deposition not implemented for first-order PSATD equations"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H index 94f00d6d4eb..5d83b5519a6 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H @@ -20,7 +20,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -139,5 +139,5 @@ class PsatdAlgorithmJConstantInTime : public SpectralBaseAlgorithm bool m_divb_cleaning; bool m_is_galilean; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_J_CONSTANT_IN_TIME_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp index 167d7565bc0..59d56b99afc 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp @@ -25,7 +25,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -861,4 +861,4 @@ PsatdAlgorithmJConstantInTime::VayDeposition (SpectralFieldData& field_data) } } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H index aa3bcc08e77..970657978bb 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H @@ -20,7 +20,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* * \brief Class that updates the fields in spectral space according to the multi-J algorithm * and stores the coefficients of the corresponding update equations. J is assumed to be @@ -125,5 +125,5 @@ class PsatdAlgorithmJLinearInTime : public SpectralBaseAlgorithm bool m_dive_cleaning; bool m_divb_cleaning; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_J_LINEAR_IN_TIME_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp index 92e35c8c03b..a6ee55147c9 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp @@ -24,7 +24,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex::literals; @@ -444,4 +444,4 @@ PsatdAlgorithmJLinearInTime::VayDeposition (SpectralFieldData& field_data) "Vay deposition not implemented for multi-J PSATD algorithm"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H index 1c25ca91903..1c19827a0a7 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H @@ -19,7 +19,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* * \brief Class that updates the field in spectral space @@ -115,5 +115,5 @@ class PsatdAlgorithmPml : public SpectralBaseAlgorithm bool m_is_galilean; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_PML_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp index 792d5126ef7..ee776a71a20 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp @@ -26,7 +26,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -466,4 +466,4 @@ void PsatdAlgorithmPml::VayDeposition (SpectralFieldData& /*field_data*/) WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented for PML PSATD"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H index 4af24cfa559..e6787c9e52d 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H @@ -23,7 +23,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. @@ -101,5 +101,5 @@ class SpectralBaseAlgorithm KVectorComponent modified_kz_vec; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_SPECTRAL_BASE_ALGORITHM_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp index 35d37df6038..20c97f4b5d4 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp @@ -29,7 +29,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -475,4 +475,4 @@ SpectralFieldData::BackwardTransform (const int lev, } } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.H b/Source/FieldSolver/SpectralSolver/SpectralSolver.H index 9c74cdbea5a..3d751974a2b 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.H @@ -21,7 +21,7 @@ #include #include -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT /** * \brief Top-level class for the electromagnetic spectral solver * @@ -205,5 +205,5 @@ class SpectralSolver // to an instance of a sub-class defining a specific algorithm std::unique_ptr algorithm; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_SPECTRAL_SOLVER_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index 460afee999f..9fc8f8056ca 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -19,7 +19,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT SpectralSolver::SpectralSolver( const int lev, @@ -145,4 +145,4 @@ SpectralSolver::pushSpectralFields(){ algorithm->pushSpectralFields( field_data ); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index fe7642fd172..6631c1b88ee 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -11,7 +11,7 @@ #include "BoundaryConditions/PML.H" #include "Evolve/WarpXDtType.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" -#if defined(WARPX_USE_PSATD) +#if defined(WARPX_USE_FFT) # include "FieldSolver/SpectralSolver/SpectralFieldData.H" # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" @@ -54,7 +54,7 @@ using namespace amrex; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT namespace { void ForwardTransformVect ( @@ -645,12 +645,12 @@ WarpX::PSATDScaleAverageFields (const amrex::Real scale_factor) } } } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT void WarpX::PushPSATD () { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ABORT_WITH_MESSAGE( "PushFieldsEM: PSATD solver selected but not built"); #else diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 0d5ab70c657..bd40c5e0cd4 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -11,7 +11,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/MultiDiagnostics.H" @@ -279,7 +279,7 @@ WarpX::PrintMainPICparameters () else if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC){ amrex::Print() << "Maxwell Solver: | Hybrid-PIC (Ohm's law) \n"; } - #ifdef WARPX_USE_PSATD + #ifdef WARPX_USE_FFT // Print PSATD solver's configuration if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD){ amrex::Print() << "Maxwell Solver: | PSATD \n"; @@ -326,12 +326,12 @@ WarpX::PrintMainPICparameters () if (fft_do_time_averaging == 1){ amrex::Print()<<" | - time-averaged is ON \n"; } - #endif // WARPX_USE_PSATD + #endif // WARPX_USE_FFT if (grid_type == GridType::Collocated){ amrex::Print() << " | - collocated grid \n"; } - #ifdef WARPX_USE_PSATD + #ifdef WARPX_USE_FFT if ( (grid_type == GridType::Staggered) && (field_gathering_algo == GatheringAlgo::EnergyConserving) ){ amrex::Print()<<" | - staggered grid " << "\n"; } @@ -384,7 +384,7 @@ WarpX::PrintMainPICparameters () amrex::Print() << "Guard cells | - ng_alloc_EB = " << guard_cells.ng_alloc_EB << "\n"; amrex::Print() << " (allocated for E/B) | \n"; - #endif // WARPX_USE_PSATD + #endif // WARPX_USE_FFT amrex::Print() << "-------------------------------------------------------------------------------" << "\n"; //Print main boosted frame algorithm's parameters if (WarpX::gamma_boost!=1){ @@ -590,7 +590,7 @@ WarpX::InitPML () if (finest_level > 0) { do_pml = 1; } if (do_pml) { -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) do_pml_Lo[0][0] = 0; // no PML at r=0, in cylindrical geometry pml_rz[0] = std::make_unique(0, boxArray(0), DistributionMap(0), &Geom(0), pml_ncell, do_pml_in_domain); #else diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 4662dc18abd..29f047c14e6 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -171,9 +171,9 @@ ifeq ($(USE_OPENPMD), TRUE) endif -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) USERSuffix := $(USERSuffix).PSATD - DEFINES += -DWARPX_USE_PSATD -DABLASTR_USE_FFT + DEFINES += -DWARPX_USE_FFT -DABLASTR_USE_FFT ifeq ($(USE_CUDA),TRUE) # Use cuFFT libraries += -lcufft diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index f3bff69546b..dcf5fbe5621 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -9,7 +9,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Filter/BilinearFilter.H" @@ -64,7 +64,7 @@ WarpX::UpdateAuxilaryData () void WarpX::UpdateAuxilaryDataStagToNodal () { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( false, "WarpX::UpdateAuxilaryDataStagToNodal: PSATD solver requires " @@ -584,7 +584,7 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In pml[lev]->FillBoundaryE(patch_type, nodal_sync); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->FillBoundaryE(patch_type, nodal_sync); @@ -641,7 +641,7 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In pml[lev]->FillBoundaryB(patch_type, nodal_sync); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->FillBoundaryB(patch_type, nodal_sync); diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 2846e79c361..52586a5c88c 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -249,7 +249,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi m_field_factory[lev] = std::make_unique(); #endif -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { if (spectral_solver_fp[lev] != nullptr) { // Get the cell-centered box @@ -313,7 +313,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi RemakeMultiFab(F_cp[lev], true); RemakeMultiFab(rho_cp[lev], false); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { if (spectral_solver_cp[lev] != nullptr) { BoxArray cba = ba; diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 4b0ad56a0ce..b3bd5414303 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -14,7 +14,7 @@ #include #include #include -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include # ifdef WARPX_DIM_RZ # include diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index eb7eceaa2fd..5fcaccf761a 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -9,7 +9,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Initialization/ExternalField.H" @@ -252,7 +252,7 @@ WarpX::MoveWindow (const int step, bool move_j) shiftMF(*pml_B[dim], geom[lev], num_shift, dir, lev, dont_update_cost); shiftMF(*pml_E[dim], geom[lev], num_shift, dir, lev, dont_update_cost); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev] && dim < 2) { const std::array& pml_rz_B = pml_rz[lev]->GetB_fp(); const std::array& pml_rz_E = pml_rz[lev]->GetE_fp(); @@ -598,7 +598,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, } } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (WarpX::GetInstance().getPMLRZ()) { // This does the exchange of data in the corner guard cells, the cells that are in the // guard region both radially and longitudinally. These are the PML cells in the overlapping diff --git a/Source/Utils/WarpX_Complex.H b/Source/Utils/WarpX_Complex.H index 7a1f1669286..54209b1d5d8 100644 --- a/Source/Utils/WarpX_Complex.H +++ b/Source/Utils/WarpX_Complex.H @@ -8,7 +8,7 @@ #ifndef WARPX_COMPLEX_H_ #define WARPX_COMPLEX_H_ -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include #endif @@ -21,7 +21,7 @@ // Defines a complex type on GPU & CPU using Complex = amrex::GpuComplex; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT static_assert(sizeof(Complex) == sizeof(ablastr::math::anyfft::Complex), "The complex type in WarpX and the FFT library do not match."); #endif diff --git a/Source/WarpX.H b/Source/WarpX.H index 6c6538325e6..df06a1723ca 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -27,7 +27,7 @@ #include "Fluids/MultiFluidContainer_fwd.H" #include "Fluids/WarpXFluidContainer_fwd.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ_fwd.H" # include "BoundaryConditions/PML_RZ_fwd.H" @@ -509,7 +509,7 @@ public: [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) const PML_RZ* getPMLRZ() {return pml_rz[0].get();} #endif @@ -747,7 +747,7 @@ public: static bool isAnyParticleBoundaryThermal(); PML* GetPML (int lev); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) PML_RZ* GetPML_RZ (int lev); #endif @@ -1167,7 +1167,7 @@ public: */ void PSATDSubtractCurrentPartialSumsAvg (); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ SpectralSolverRZ& @@ -1422,7 +1422,7 @@ private: const amrex::IntVect& ngRho, const amrex::IntVect& ngF, const amrex::IntVect& ngG, bool aux_is_nodal); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ void AllocLevelSpectralSolverRZ (amrex::Vector>& spectral_solver, int lev, @@ -1591,7 +1591,7 @@ private: amrex::Vector do_pml_Lo; amrex::Vector do_pml_Hi; amrex::Vector > pml; -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) amrex::Vector > pml_rz; #endif amrex::Real v_particle_pml; @@ -1766,7 +1766,7 @@ private: void PushPSATD (); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT /** * \brief Forward FFT of E,B on all mesh refinement levels diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index de632211bad..512ed7fe9d0 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -18,7 +18,7 @@ #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralKSpace.H" # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" @@ -393,7 +393,7 @@ WarpX::WarpX () charge_buf.resize(nlevs_max); pml.resize(nlevs_max); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) pml_rz.resize(nlevs_max); #endif @@ -467,7 +467,7 @@ WarpX::WarpX () } // Allocate field solver objects -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { spectral_solver_fp.resize(nlevs_max); spectral_solver_cp.resize(nlevs_max); @@ -760,10 +760,10 @@ WarpX::ReadParameters () poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, "The FFT Poisson solver only works in 3D."); #endif -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( poisson_solver_id!=PoissonSolverAlgo::IntegratedGreenFunction, - "To use the FFT Poisson solver, compile with WARPX_USE_PSATD=ON."); + "To use the FFT Poisson solver, compile with WARPX_USE_FFT=ON."); #endif WARPX_ALWAYS_ASSERT_WITH_MESSAGE( @@ -1155,7 +1155,7 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id != ElectromagneticSolverAlgo::CKC, "algo.maxwell_solver = ckc is not (yet) available for RZ geometry"); #endif -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD, "algo.maxwell_solver = psatd is not supported because WarpX was built without spectral solvers"); #endif @@ -2091,7 +2091,7 @@ WarpX::ClearLevel (int lev) G_cp [lev].reset(); rho_cp[lev].reset(); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { spectral_solver_fp[lev].reset(); spectral_solver_cp[lev].reset(); @@ -2477,7 +2477,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { // Allocate and initialize the spectral solver -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( false, "WarpX::AllocLevelMFs: PSATD solver requires WarpX build with spectral solver support."); #else @@ -2635,7 +2635,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { // Allocate and initialize the spectral solver -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( false, "WarpX::AllocLevelMFs: PSATD solver requires WarpX build with spectral solver support."); #else @@ -2729,7 +2729,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ /* \brief Allocate spectral Maxwell solver (RZ dimensions) at a level * @@ -2986,7 +2986,7 @@ void WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) { if ( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD ) { -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT spectral_solver_fp[lev]->ComputeSpectralDivE( lev, Efield_aux[lev], divE ); #else WARPX_ABORT_WITH_MESSAGE( @@ -2997,7 +2997,7 @@ WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) } } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) PML_RZ* WarpX::GetPML_RZ (int lev) { diff --git a/Source/ablastr/fields/CMakeLists.txt b/Source/ablastr/fields/CMakeLists.txt index f7dfb6763ec..158e623ebaf 100644 --- a/Source/ablastr/fields/CMakeLists.txt +++ b/Source/ablastr/fields/CMakeLists.txt @@ -1,6 +1,6 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) - if(WarpX_PSATD AND D EQUAL 3) + if(WarpX_FFT AND D EQUAL 3) target_sources(ablastr_${SD} PRIVATE IntegratedGreenFunctionSolver.cpp diff --git a/Source/ablastr/fields/Make.package b/Source/ablastr/fields/Make.package index 4fb7bd88e5f..01392991559 100644 --- a/Source/ablastr/fields/Make.package +++ b/Source/ablastr/fields/Make.package @@ -1,4 +1,4 @@ -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) ifeq ($(DIM),3) CEXE_sources += IntegratedGreenFunctionSolver.cpp endif diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 106c5475dd7..6b1cce415b4 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -15,7 +15,7 @@ #include #include -#if defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D) +#if defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D) #include #endif @@ -160,7 +160,7 @@ computePhi (amrex::Vector const & rho, {{ beta[0], beta[1], beta[2] }}; #endif -#if (defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D)) +#if (defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D)) // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected if(!is_solver_multigrid && lev==0){ amrex::Array const dx_igf diff --git a/Source/ablastr/math/fft/CMakeLists.txt b/Source/ablastr/math/fft/CMakeLists.txt index f7d689c98e9..2b9fc3d37f9 100644 --- a/Source/ablastr/math/fft/CMakeLists.txt +++ b/Source/ablastr/math/fft/CMakeLists.txt @@ -1,6 +1,6 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) - if(WarpX_PSATD STREQUAL ON) + if(WarpX_FFT STREQUAL ON) if(WarpX_COMPUTE STREQUAL CUDA) target_sources(ablastr_${SD} PRIVATE WrapCuFFT.cpp) elseif(WarpX_COMPUTE STREQUAL HIP) diff --git a/Source/ablastr/math/fft/Make.package b/Source/ablastr/math/fft/Make.package index b04062bd9d5..63786cdc006 100644 --- a/Source/ablastr/math/fft/Make.package +++ b/Source/ablastr/math/fft/Make.package @@ -1,4 +1,4 @@ -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) ifeq ($(USE_CUDA),TRUE) CEXE_sources += WrapCuFFT.cpp else ifeq ($(USE_HIP),TRUE) diff --git a/Tools/Linter/runClangTidy.sh b/Tools/Linter/runClangTidy.sh index 39a96cc97d4..046c72d7b27 100755 --- a/Tools/Linter/runClangTidy.sh +++ b/Tools/Linter/runClangTidy.sh @@ -104,7 +104,7 @@ cmake -S ${REPO_DIR} -B ${REPO_DIR}/build_clang_tidy \ -DWarpX_DIMS="1;2;3;RZ" \ -DWarpX_MPI=ON \ -DWarpX_COMPUTE=OMP \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ diff --git a/WarpXConfig.cmake b/WarpXConfig.cmake index 5a2c0916432..8536d08f67a 100644 --- a/WarpXConfig.cmake +++ b/WarpXConfig.cmake @@ -35,11 +35,12 @@ if(WarpX_OPENPMD) endif() set(WarpX_OPENPMD_FOUND ${WarpX_OPENPMD}) -WarpX_ASCENT "Ascent in situ diagnostics" OFF) -WarpX_PSATD "spectral solver support" OFF) -WarpX_SENSEI "SENSEI in situ diagnostics" OFF) -WarpX_QED "QED support (requires PICSAR)" ON) -WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" OFF) +# TODO +#WarpX_ASCENT "Ascent in situ diagnostics" OFF) +#WarpX_FFT "FFT-based solvers" OFF) +#WarpX_SENSEI "SENSEI in situ diagnostics" OFF) +#WarpX_QED "QED support (requires PICSAR)" ON) +#WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" OFF) @@ -50,7 +51,7 @@ endif() set(WarpX_ADIOS2_FOUND ${WarpX_HAVE_ADIOS2}) # define central WarpX::app WarpX::shared ... targets -ABLASTR here, too? or separate Config.cmake? +#ABLASTR here, too? or separate Config.cmake? include("${CMAKE_CURRENT_LIST_DIR}/WarpXTargets.cmake") # check if components are fulfilled and set WarpX__FOUND vars diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index 9b9feb64437..a68ebca6c60 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -305,7 +305,7 @@ function(set_warpx_binary_name D) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".OPMD") endif() - if(WarpX_PSATD) + if(WarpX_FFT) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".PSATD") endif() @@ -452,7 +452,7 @@ function(warpx_print_summary) endif() message(" PARTICLE PRECISION: ${WarpX_PARTICLE_PRECISION}") message(" PRECISION: ${WarpX_PRECISION}") - message(" PSATD: ${WarpX_PSATD}") + message(" FFT Solvers: ${WarpX_FFT}") message(" PYTHON: ${WarpX_PYTHON}") if(WarpX_PYTHON) message(" PYTHON IPO: ${WarpX_PYTHON_IPO}") diff --git a/setup.py b/setup.py index 164f4f1395f..3d61cc5be36 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ def build_extension(self, ext): '-DWarpX_OPENPMD:BOOL=' + WARPX_OPENPMD, '-DWarpX_PRECISION=' + WARPX_PRECISION, '-DWarpX_PARTICLE_PRECISION=' + WARPX_PARTICLE_PRECISION, - '-DWarpX_PSATD:BOOL=' + WARPX_PSATD, + '-DWarpX_FFT:BOOL=' + WARPX_FFT, '-DWarpX_PYTHON:BOOL=ON', '-DWarpX_PYTHON_IPO:BOOL=' + WARPX_PYTHON_IPO, '-DWarpX_QED:BOOL=' + WARPX_QED, @@ -205,7 +205,7 @@ def build_extension(self, ext): WARPX_OPENPMD = env.pop('WARPX_OPENPMD', 'ON') WARPX_PRECISION = env.pop('WARPX_PRECISION', 'DOUBLE') WARPX_PARTICLE_PRECISION = env.pop('WARPX_PARTICLE_PRECISION', WARPX_PRECISION) -WARPX_PSATD = env.pop('WARPX_PSATD', 'OFF') +WARPX_FFT = env.pop('WARPX_FFT', 'OFF') WARPX_QED = env.pop('WARPX_QED', 'ON') WARPX_QED_TABLE_GEN = env.pop('WARPX_QED_TABLE_GEN', 'OFF') WARPX_DIMS = env.pop('WARPX_DIMS', '1;2;RZ;3') From bfffedf5862fb180619567559b48d4e8aa7a0c6d Mon Sep 17 00:00:00 2001 From: Arianna Formenti Date: Thu, 30 May 2024 20:09:46 -0700 Subject: [PATCH 112/190] FFT-based Poisson solver follow-up (#4945) * changed 2 * add assert to check fft compilation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Source/ablastr/fields/PoissonSolver.H Co-authored-by: Axel Huebl * Update Source/ablastr/fields/PoissonSolver.H Co-authored-by: Axel Huebl --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- Source/FieldSolver/ElectrostaticSolver.cpp | 6 +++--- Source/ablastr/fields/PoissonSolver.H | 16 +++++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index a74beae3c91..3860bd55325 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -382,8 +382,8 @@ WarpX::computePhi (const amrex::Vector >& rho, const std::optional > eb_farray_box_factory; #endif - bool const is_solver_multigrid = - WarpX::poisson_solver_id != PoissonSolverAlgo::IntegratedGreenFunction; + bool const is_solver_igf_on_lev0 = + WarpX::poisson_solver_id == PoissonSolverAlgo::IntegratedGreenFunction; ablastr::fields::computePhi( sorted_rho, @@ -397,7 +397,7 @@ WarpX::computePhi (const amrex::Vector >& rho, this->dmap, this->grids, this->m_poisson_boundary_handler, - is_solver_multigrid, + is_solver_igf_on_lev0, WarpX::do_single_precision_comms, this->ref_ratio, post_phi_calculation, diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 6b1cce415b4..e5340e825c9 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -82,7 +82,7 @@ namespace ablastr::fields { * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler - * \param[in] is_solver_multigrid boolean to select the Poisson solver: 1 for Multigrid, 0 for FFT + * \param[in] is_solver_igf_on_lev0 boolean to select the Poisson solver: 1 for FFT on level 0 & Multigrid on other levels, 0 for Multigrid on all levels * \param[in] do_single_precision_comms perform communications in single precision * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) * \param[in] post_phi_calculation perform a calculation per level directly after phi was calculated; required for embedded boundaries (default: none) @@ -106,7 +106,7 @@ computePhi (amrex::Vector const & rho, amrex::Vector const& dmap, amrex::Vector const& grids, T_BoundaryHandler const boundary_handler, - [[maybe_unused]] bool is_solver_multigrid, + bool is_solver_igf_on_lev0, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, @@ -160,9 +160,19 @@ computePhi (amrex::Vector const & rho, {{ beta[0], beta[1], beta[2] }}; #endif +#if !defined(WARPX_USE_FFT) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "Must compile with -DWarpX_FFT=ON to use the FFT solver!"); +#endif + +#if !defined(WARPX_DIM_3D) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "The FFT Poisson solver is currently only implemented for 3D!"); +#endif + #if (defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D)) // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected - if(!is_solver_multigrid && lev==0){ + if(is_solver_igf_on_lev0 && lev==0){ amrex::Array const dx_igf {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), From f9daed91c0a72062aaf0a87420f83185998f025c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Fri, 31 May 2024 13:05:09 -0700 Subject: [PATCH 113/190] Document PICMI function `LoadInitialField` (#4962) --- Docs/source/usage/python.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 3aba5f165de..4d53b311764 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -77,6 +77,8 @@ Instances of the classes below need to be passed to the method `add_applied_fiel .. autoclass:: pywarpx.picmi.AnalyticAppliedField +.. autoclass:: pywarpx.picmi.LoadInitialField + .. autoclass:: pywarpx.picmi.PlasmaLens .. autoclass:: pywarpx.picmi.Mirror From 7f4b086eefb1d0b3b36a9d72faacf7d472cc740e Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Sun, 2 Jun 2024 07:02:26 -0700 Subject: [PATCH 114/190] New Implicit Solver interface with options to select Picard or Newton (JFNK) for the nonlinear solver. (#4736) * created Source/FieldSolver/WarXImplicitFieldsEM.cpp file to store functions used with the implicit solvers. Only has ComputeRHSE() and ComputeRHSB() functions thus far. * OneStep_ImplicitPicard routine now uses ComputeRHSE and ComputeRHSB functions in place of EvolveE and EvolveB functions. * created Source/Evolve/WarpXImplicitOps.cpp file to store common functions used by implicit solvers. Moved several functions from WarpXOneStepImplicitPicard.cpp to here. * added NonlinearSolver:: enum. Options are picard and newton. Set from input file using algo.nonlinear_solver = picard, for example. Default is picard. It is not used yet. * changed EvolveScheme::ImplicitPicard and SemiImplicitPicard to ThetaImplicit and SemiImplicit, respectively. This affects parsing from input file. algo.evolve_scheme = implicit_picard ==> theta_implicit. Have not updated Docs yet. * NonlinearSolver ==> NonlinearSolverType * intermediate commit. Added an ImplicitSolversEM class. WarpX owns an instance of this class. OneStepImplicitEM is now done here. This class owns all things pertaining to the implicit integrator. Also added a NonlinerSolvers class, but it is not used yet. * cleaning up ImplicitSolverEM::OneStep(). * more refactoring of ImplicitSovlerEM class. * WarpXFieldVec ==> WarpXSolverVec * removed depricated functions. WarpXSolverVec has zero ghost. * ImplicitSolverEM::OneStep now looks exactly like Picard solver. Next step is to move it there. * ImplicitSolverEM::OneStep() now uses the Picard solver object to solve the nonlinear equation. * changed where implicit solver parameters are printed. * refactoring of WarpXImplicitOps.cpp * added NewtonSolver.H file. Doesn't work yet. * adding more functionality to WarpXSovlerVec class. * added JacobianFunctionJFNK.H file in NonlinarSolvers/ folder. It contains all of the necessary functions required by the linear operator template parameter for AMReX_GMRES. * dotMask object used for dot product from Linear Function class now lives in the implicit solver class. This ensures that 1 and only 1 instance of this object will be defined. dotProduct and norm can now be called through the implicit sovler class. * moved temporary linear_function and GMRES testing lines out of Picard::Define() and into Newton::Define() * intermediate commit. JFNK almost ready. * small refactoring of PreRHSOp() and PostUpdateState() functions. * cleaning things up. * Newton solver runs. GMRES runs. Next step is to do Particle-suppressed (PS) part. * minor clean up. * fixed typo in convergence message for Newton and Picard solvers. * changed how PostUpdateState() is used in implicit solver. Now parsing Picard particle solver parameters in ImplicitSolverEM class. Using a new formula for the epsilon used in the finite-difference Jacobian action calculation that is suitable for large absolute norms of the solution vector. * Picard method for particle is now being used. PS-JFNK works. * moved WarpXImplicitOps.cpp from Source/Evolve/ to Source/FieldSolvers/ImplicitSolvers/ * minor cleanup. PostUpdateState() ==> UpdateWarpXState(). * Moved the particle convergence check into its own function. * added increment function to WarpXSolverVec class. * removed some commented out lines in JacobianFunctionJFNK.H * removed a_tol condition for print message when maximum iterations reached in nonlinear solvers. Newton solver iter is base zero now. * cleaned up picard method for self-consistent particle update. added ablastr warning message for particles that don't converge after the maximum number of iterations. * fixed small bug in PicardSolver related to absolute tolerance check for convegence. Default absolute tolerance values are set to zero. * the mask used to compute the dot product of a WarpXSolverVec is now owned by the WarpXSolverVec class rather then the time solver class. It is a static member to avoid multiple definitions. * defined accessors for Efield_fp and Bfield_fp Vectors owned by WarpX. ImplicitSolver is no longer a friend class of WarpX. Small tidy for PicardSolver.H. * SemiImplicitEM and ThetaImplicitEM are now their own independent derived classes from the base ImplicitSolver class. * added algorithm descriptions to the top of the SemiImplicitEM.cpp and ThetaImplicitEM.cpp files. * updated appropriate files in Examples and Regression folders to reflect new changes. * updating docs. * JacobianFunctionJFNK.H ==> JacobianFunctionMF.H * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * need to call clear on the static vector m_dotMask owned by the WarpXSovlerVec class to avoid malloc_consolidate(): unaligned fastbin chunk detected error messages at simulation finish. * moved WarpXSovlerVec class from ../ImplicitSolvers/Utils/ to ../ImplicitSolvers/. Utils directory is deleted. * ImplicitPushXP: GPU Support for Convergence Test * cleaning up. * Atomic: Fix type of `1` * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * bug fix. * Update Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H Co-authored-by: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> * Remove Zero * adding Copyright lines to top of files. Removed commented code. * Prevent calling the implicit solver if not initialized * More robust if conditions * set implicit verbose to warpx verbose * Update Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp * Simplify call to updates of E * changed benchmarks_json file names as needed. * using warpx.verbose * clang-tidying * changed header names. * clang-tyding * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * more clang-tyding * clang tidy again * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * UpdateElectricFiled ==> SetElectricFieldAndApplyBCs * clang tidy * more clang tidy * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * prohibiting copy and move constructors for solver classes. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed bug in move constructor in WarpXSovlerVec. * slight refactoring of ThetaImplicitEM class. * reducing divisions in Picard method for particles. * small cosmetic changes to implicit solvers. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * removed commented out code. * updating Docs and adding briefs. * Fix HIP compilation * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix const * fixed indent. updated comments. * New Python test: Particle-Boundary interaction (#4729) * enable the diagnostic of ParticleScraping in Python * Update picmi.py * Update picmi.py * new test * python update * modification of the script * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update PICMI_inputs_rz.py * json * update * Update PICMI_inputs_rz.py * Update particle_boundary_interaction.json * Update PICMI_inputs_rz.py * Update PICMI_inputs_rz.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update PICMI_inputs_rz.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix hanging script in parallel * Make the test executable * Update analysis script * Update particle_containers.py * Update PICMI_inputs_rz.py * Update analysis.py * Update analysis.py * Update particle_containers.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Remi Lehe Co-authored-by: Axel Huebl * Adding normal components to regular boundary buffer (#4742) * first draft * adding normal only * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update ParticleBoundaryBuffer.cpp * Update ParticleBoundaryBuffer.cpp --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * Add function to set domain boundary potentials from Python (#4740) * add function to set domain boundary potentials from Python * switch function arguments to form `potential_[lo/hi]_[x/y/z]` and add to docs * clean up `ablastr/fields` (#4753) * move PoissonInterpCPtoFP to Interpolate.H * concatenate nested namespaces * Split clang-tidy CI test into 4 to improve performances (#4747) * split clang-tidy checks to improve performances * rename folders and tests * fix concurrency * Simplify --------- Co-authored-by: Axel Huebl * Replace links to learn git (#4758) * Replace links to learn git * Bugfix in `fields.py` for GPU run without `cupy` (#4750) * Bugfix in `fields.py` for GPU run without `cupy` * apply suggestion from code review * Release 24.03 (#4759) * AMReX: 24.03 * pyAMReX: 24.03 * WarpX: 24.03 * Implement stair-case Yee solver with EB in RZ geometry (#2707) * Allow compilation with RZ EB * Do not push cells for RZ Yee solver, when covered with EB * Fix compilation errors * Fix additional compilation errors * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix additional compilation errors * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add automated test * Add automated test * Fix path in tests * Enable parser in RZ * Update example script * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Clean-up PR * Initialize EB quantities * Modified EM field initialization in 2D with EB * Typo fix * Typo fix * Ignoring unused variables correctly * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Correct condition for updating E * Correct update of B * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update B push * Update input script * Revert "Update input script" This reverts commit 5087485db606f77806c08849c51af9be635ca622. * Update initialization * Updated test * Move test to a different folder * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add test for WarpX-test.ini * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix path for tests * Update test description * Update test metadata * Add checksum file * Revert changes * Revert changes * Change lx to lr * Revert "Change lx to lr" This reverts commit be3039af229d5dee509c8de0a712addf099a2b82. * Change lx to lr --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: lgiacome * AMReX/pyAMReX/PICSAR: Weekly Update (#4763) * AMReX: Weekly Update * pyAMReX: Weekly Update * clean up (#4761) * Fix compilation * updating some function names to contain Implicit. * fixed bug that caused segfault on GMRES restart. * parsing GMRES restart length from input file. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update field accessor * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * removed +, -, and * operators from WarpXSolverVec class. These operators encourage inefficient vector arithmetic. * fix/workaround to field accessor issue. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * changing implicit-scheme related header files in WarpX.H and WarpX.cpp * WarpX::max_particle_iterations ==> WarpX::max_particle_its_in_implicit_scheme and WarpX::particle_tolerance ==> WarpX:particle_tol_in_implicit_scheme * updating docs. * updating docs again. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * adding comments about the template parameters Vec and Ops used in the NonlinearSolver class. * adding comments. addressing comments from PR. * Ensure that laser particles can converge in the implicit solver * Add braces to make clang-tidy happy * moving nonlinear solver parameters to base ImplicitSolver class * mirrors not to be used with implicit schemes. * moved more to base implicit solver class. adding comments. * removed some WarpXSolverVec functions. Updated comments. * clang tidy complains when removing default copy constructor in WarpXSolverVec.H * amrex::ExecOnFinalize(WarpXSolverVec::clearDotMask) * WarpXSolverVec (const WarpXSolverVec&) = delete * updating briefs for nonlinear solvers. * adding loop over levels. * static cast amrex::Vector.size() to int * updating docs for nonlinear solvers. * adding gmres.restart_length to docs. * fixed typos in docs. * Removed PreRHSOp() call from nonlinear solvers. * clang tidy. * Prohibit = operator for WarpXSolverVec. Using Copy() instead. * Document PICMI function `LoadInitialField` * updating comments in WarpXImplicitOps.cpp * moved static member m_dotMask definition to the header file with inline added to the declaration. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fixed indent. --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl Co-authored-by: Debojyoti Ghosh Co-authored-by: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Co-authored-by: Remi Lehe Co-authored-by: Weiqun Zhang Co-authored-by: Eya D <81635404+EyaDammak@users.noreply.github.com> Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Co-authored-by: Arianna Formenti Co-authored-by: Luca Fedeli Co-authored-by: lgiacome --- Docs/source/usage/parameters.rst | 145 ++++-- Examples/Tests/Implicit/analysis_1d.py | 2 +- ..._vandb_2d.py => analysis_vandb_jfnk_2d.py} | 0 Examples/Tests/Implicit/inputs_1d | 14 +- .../Tests/Implicit/inputs_1d_semiimplicit | 14 +- .../{inputs_vandb_2d => inputs_vandb_jfnk_2d} | 30 +- ...d.json => ThetaImplicitJFNK_VandB_2d.json} | 0 ...rd_1d.json => ThetaImplicitPicard_1d.json} | 0 Regression/WarpX-tests.ini | 8 +- Source/Evolve/CMakeLists.txt | 1 - Source/Evolve/Make.package | 1 - Source/Evolve/WarpXEvolve.cpp | 6 +- Source/Evolve/WarpXOneStepImplicitPicard.cpp | 423 ------------------ Source/FieldSolver/CMakeLists.txt | 1 + .../ImplicitSolvers/CMakeLists.txt | 10 + .../ImplicitSolvers/ImplicitSolver.H | 145 ++++++ .../ImplicitSolvers/ImplicitSolverLibrary.H | 13 + .../FieldSolver/ImplicitSolvers/Make.package | 6 + .../ImplicitSolvers/SemiImplicitEM.H | 80 ++++ .../ImplicitSolvers/SemiImplicitEM.cpp | 117 +++++ .../ImplicitSolvers/ThetaImplicitEM.H | 124 +++++ .../ImplicitSolvers/ThetaImplicitEM.cpp | 171 +++++++ .../ImplicitSolvers/WarpXImplicitOps.cpp | 352 +++++++++++++++ .../ImplicitSolvers/WarpXSolverVec.H | 233 ++++++++++ .../ImplicitSolvers/WarpXSolverVec.cpp | 51 +++ Source/FieldSolver/Make.package | 1 + Source/Initialization/WarpXInitData.cpp | 26 ++ Source/Make.WarpX | 3 +- Source/NonlinearSolvers/CMakeLists.txt | 6 + Source/NonlinearSolvers/JacobianFunctionMF.H | 234 ++++++++++ Source/NonlinearSolvers/Make.package | 1 + Source/NonlinearSolvers/NewtonSolver.H | 345 ++++++++++++++ Source/NonlinearSolvers/NonlinearSolver.H | 87 ++++ .../NonlinearSolvers/NonlinearSolverLibrary.H | 15 + Source/NonlinearSolvers/PicardSolver.H | 217 +++++++++ Source/Particles/LaserParticleContainer.H | 2 +- Source/Particles/LaserParticleContainer.cpp | 46 +- .../Particles/PhysicalParticleContainer.cpp | 185 +++++--- Source/Particles/Pusher/UpdatePosition.H | 42 ++ Source/Utils/WarpXAlgorithmSelection.H | 4 +- Source/Utils/WarpXAlgorithmSelection.cpp | 8 +- Source/WarpX.H | 63 ++- Source/WarpX.cpp | 67 ++- 43 files changed, 2717 insertions(+), 582 deletions(-) rename Examples/Tests/Implicit/{analysis_vandb_2d.py => analysis_vandb_jfnk_2d.py} (100%) rename Examples/Tests/Implicit/{inputs_vandb_2d => inputs_vandb_jfnk_2d} (78%) rename Regression/Checksum/benchmarks_json/{ImplicitPicard_VandB_2d.json => ThetaImplicitJFNK_VandB_2d.json} (100%) rename Regression/Checksum/benchmarks_json/{ImplicitPicard_1d.json => ThetaImplicitPicard_1d.json} (100%) delete mode 100644 Source/Evolve/WarpXOneStepImplicitPicard.cpp create mode 100644 Source/FieldSolver/ImplicitSolvers/CMakeLists.txt create mode 100644 Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H create mode 100644 Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H create mode 100644 Source/FieldSolver/ImplicitSolvers/Make.package create mode 100644 Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H create mode 100644 Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp create mode 100644 Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H create mode 100644 Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp create mode 100644 Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp create mode 100644 Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H create mode 100644 Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp create mode 100644 Source/NonlinearSolvers/CMakeLists.txt create mode 100644 Source/NonlinearSolvers/JacobianFunctionMF.H create mode 100644 Source/NonlinearSolvers/Make.package create mode 100644 Source/NonlinearSolvers/NewtonSolver.H create mode 100644 Source/NonlinearSolvers/NonlinearSolver.H create mode 100644 Source/NonlinearSolvers/NonlinearSolverLibrary.H create mode 100644 Source/NonlinearSolvers/PicardSolver.H diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index f3a74f969a0..fb53e81012f 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -86,37 +86,122 @@ Overall simulation parameters * ``explicit``: Use an explicit solver, such as the standard FDTD or PSATD - * ``implicit_picard``: Use an implicit solver with exact energy conservation that uses a Picard iteration to solve the system. - Note that this method is for demonstration only. It is inefficient and does not work well when - :math:`\omega_{pe} \Delta t` is close to or greater than one. - The method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. - The version implemented is an updated version that is relativistically correct, including the relativistic gamma factor for the particles. - For exact energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, - and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in - which case charge will also be conserved). - - * ``semi_implicit_picard``: Use an energy conserving semi-implicit solver that uses a Picard iteration to solve the system. - Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. It is inefficient and does not work well or at all when :math:`\omega_{pe} \Delta t` is close to or greater than one. + * ``theta_implicit_em``: Use a fully implicit electromagnetic solver with a time-biasing parameter theta bound between 0.5 and 1.0. Exact energy conservation is achieved using theta = 0.5. Maximal damping of high-k modes is obtained using theta = 1.0. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed (PS) JNFK. + The algorithm itself is numerical stable for large time steps. That is, it does not require time steps that resolve the plasma period or the CFL condition for light waves. However, the practicality of using a large time step depends on the nonlinear solver. Note that the Picard solver is for demonstration only. It is inefficient and will most like not converge when + :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relavent physics. + The Picard method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. + The PS-JFNK method is described in `Angus et al., An implicit particle code with exact energy and charge conservation for electromagnetic studies of dense plasmas `__ . (The version implemented in WarpX is an updated version that includes the relativistic gamma factor for the particles.) Also see `Chen et al., An energy- and charge-conserving, implicit, electrostatic particle-in-cell algorithm. `__ . + Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. + + * ``semi_implicit_em``: Use an approximately energy conserving semi-implicit electromagnetic solver. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed JFNK. + Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. The Picard solver for this method can only be expected to work well when :math:`\omega_{pe} \Delta t` is less than one. The method is described in `Chen et al., A semi-implicit, energy- and charge-conserving particle-in-cell algorithm for the relativistic Vlasov-Maxwell equations `__. - For energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, - and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in - which case charge will also be conserved). - -* ``algo.max_picard_iterations`` (`integer`, default: 10) - When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the maximum number of Picard - itearations that are done each time step. - -* ``algo.picard_iteration_tolerance`` (`float`, default: 1.e-7) - When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the convergence tolerance of - the iterations, the maximum of the relative change of the L2 norm of the field from one iteration to the next. - If this is set to zero, the maximum number of iterations will always be done with the change only calculated on the last - iteration (for a slight optimization). - -* ``algo.require_picard_convergence`` (`bool`, default: 1) - When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets whether the iteration each step - is required to converge. - If it is required, an abort is raised if it does not converge and the code then exits. - If not, then a warning is issued and the calculation continues. + Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. + +* ``implicit_evolve.theta`` (`float`, default: 0.5) + When `algo.evolve_scheme = theta_implicit_em`, the fields used on the RHS of the equations for the implicit advance + are computed as (1-theta)*E_{n} + theta*E_{n+1}. theta is bound between 0.5 and 1. The default value of theta = 0.5 + is needed for exact energy conservation. For theta > 0.5, high-k modes will be damped and the method will not be + exactly energy conserving, but the solver may perform better. + +* ``implicit_evolve.nonlinear_solver`` (`string`, default: None) + When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em`, this sets the nonlinear solver used + to advance the field-particle system in time. Options are `picard` or `newton`. + +* ``implicit_evolve.max_particle_iterations`` (`integer`, default: 21) + When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + , this sets the maximum number of iterations for the method used to obtain a self-consistent update of the particles at + each iteration in the JFNK process. + +* ``implicit_evolve.particle_tolerance`` (`float`, default: 1.e-10) + When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + , this sets the relative tolerance for the iterative method used to obtain a self-consistent update of the particles at + each iteration in the JFNK process. + +* ``picard.verbose`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = picard`, this sets the verbosity of the Picard solver. If true, then information + on the nonlinear error are printed to screen at each nonlinear iteration. + +* ``picard.require_convergence`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = picard`, this sets whether the Picard method is required to converge at each + time step. If it is required, an abort is raised if it does not converge and the code then exits. If not, then a warning + is issued and the calculation continues. + +* ``picard.maximum_iterations`` (`int`, default: 100) + When `implicit_evolve.nonlinear_solver = picard`, this sets the maximum iterations used by the Picard method. If + `picard.require_convergence = false`, the solution is considered converged if the iteration count reaches this value, + but a warning is issued. If `picard.require_convergence = true`, then an abort is raised if the iteration count reaches + this value. + +* ``picard.relative_tolerance`` (`float`, default: 1.0e-6) + When `implicit_evolve.nonlinear_solver = picard`, this sets the relative tolerance used by the Picard method for determining + convergence. The absolute error for the Picard method is the L2 norm of the difference of the solution vector between + two successive iterations. The relative error is the absolute error after iteration k > 1 divided by the absolute error + after the first iteration. The Picard method is considered converged when the relative error is below the relative tolerance. + This is the preferred means of determining convergence. + +* ``picard.absolute_tolerance`` (`float`, default: 0.0) + When `implicit_evolve.nonlinear_solver = picard`, this sets the absolute tolerance used by the Picard method for determining + convergence. The default value is 0.0, which means that the absolute tolerance is not used to determine converence. The + solution vector in the nonlinear solvers are in physical units rather than normalized ones. Thus, the absolute scale + of the problem can vary over many orders and magnitude depending on the problem. The relative tolerance is the preferred + means of determining convergence. + +* ``newton.verbose`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = newton`, this sets the verbosity of the Newton solver. If true, then information + on the nonlinear error are printed to screen at each nonlinear iteration. + +* ``newton.require_convergence`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = newton`, this sets whether the Newton method is required to converge at each + time step. If it is required, an abort is raised if it does not converge and the code then exits. If not, then a warning + is issued and the calculation continues. + +* ``newton.maximum_iterations`` (`int`, default: 100) + When `implicit_evolve.nonlinear_solver = newton`, this sets the maximum iterations used by the Newton method. If + `newton.require_convergence = false`, the solution is considered converged if the iteration count reaches this value, + but a warning is issued. If `newton.require_convergence = true`, then an abort is raised if the iteration count reaches + this value. + +* ``newton.relative_tolerance`` (`float`, default: 1.0e-6) + When `implicit_evolve.nonlinear_solver = newton`, this sets the relative tolerance used by the Newton method for determining + convergence. The absolute error for the Newton method is the L2 norm of the residual vector. The relative error is the + absolute error divided by the L2 norm of the initial residual associated with the initial guess. The Newton method is + considered converged when the relative error is below the relative tolerance. This is the preferred means of determining + convergence. + +* ``newton.absolute_tolerance`` (`float`, default: 0.0) + When `implicit_evolve.nonlinear_solver = newton`, this sets the absolute tolerance used by the Newton method for determining + convergence. The default value is 0.0, which means that the absolute tolerance is not used to determine converence. The + residual vector in the nonlinear solvers are in physical units rather than normalized ones. Thus, the absolute scale + of the problem can vary over many orders and magnitude depending on the problem. The relative tolerance is the preferred + means of determining convergence. + +* ``gmres.verbose_int`` (`int`, default: 2) + When `implicit_evolve.nonlinear_solver = newton`, this sets the verbosity of the AMReX::GMRES linear solver. The default + value of 2 gives maximumal verbosity and information about the residual are printed to the screen at each GMRES iteration. + +* ``gmres.restart_length`` (`int`, default: 30) + When `implicit_evolve.nonlinear_solver = newton`, this sets the iteration number at which to do a restart in AMReX::GMRES. + This parameter is used to save memory on building the Krylov subspace basis vectors for linear systems that are ill-conditioned + and require many iterations to converge. + +* ``gmres.relative_tolerance`` (`float`, default: 1.0e-4) + When `implicit_evolve.nonlinear_solver = newton`, this sets the relative tolerance used to determine convergence of the + AMReX::GMRES linear solver used to compute the Newton step in the JNFK process. The absolute error is the L2 norm of the + residual vector. The relative error is the absolute error divided by the L2 norm of the initial residual (typically equal + to the norm of the nonlinear residual from the end of the previous Newton iteration). The linear solver is considered + converged when the relative error is below the relative tolerance. This is the preferred means of determining convergence. + +* ``gmres.absolute_tolerance`` (`float`, default: 0.0) + When `implicit_evolve.nonlinear_solver = newton`, this sets the absolute tolerance used to determine converence of the + GMRES linear solver used to compute the Newton step in the JNFK process. The default value is 0.0, which means that the + absolute tolerance is not used to determine converence. The residual vector in the nonlinear/linear solvers are in physical + units rather than normalized ones. Thus, the absolute scale of the problem can vary over many orders and magnitude depending + on the problem. The relative tolerance is the preferred means of determining convergence. + +* ``gmres.maximum_iterations`` (`int`, default: 1000) + When `implicit_evolve.nonlinear_solver = newton`, this sets the maximum iterations used by the GMRES linear solver. The + solution to the linear system is considered converged if the iteration count reaches this value. * ``warpx.do_electrostatic`` (`string`) optional (default `none`) Specifies the electrostatic mode. When turned on, instead of updating diff --git a/Examples/Tests/Implicit/analysis_1d.py b/Examples/Tests/Implicit/analysis_1d.py index 0f00010a505..0e20b925df5 100755 --- a/Examples/Tests/Implicit/analysis_1d.py +++ b/Examples/Tests/Implicit/analysis_1d.py @@ -31,7 +31,7 @@ if re.match('SemiImplicitPicard_1d', fn): tolerance_rel = 2.5e-5 -elif re.match('ImplicitPicard_1d', fn): +elif re.match('ThetaImplicitPicard_1d', fn): # This case should have near machine precision conservation of energy tolerance_rel = 1.e-14 diff --git a/Examples/Tests/Implicit/analysis_vandb_2d.py b/Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py similarity index 100% rename from Examples/Tests/Implicit/analysis_vandb_2d.py rename to Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d index 465d9dd6965..3e57689b723 100644 --- a/Examples/Tests/Implicit/inputs_1d +++ b/Examples/Tests/Implicit/inputs_1d @@ -31,10 +31,18 @@ boundary.particle_hi = periodic ############ NUMERICS ########### ################################# +warpx.verbose = 1 warpx.const_dt = dt -algo.evolve_scheme = implicit_picard -algo.max_picard_iterations = 31 -algo.picard_iteration_tolerance = 0. +algo.evolve_scheme = theta_implicit_em + +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 31 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + algo.current_deposition = esirkepov algo.field_gathering = energy-conserving algo.particle_shape = 2 diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit index 4008a559588..07460e08be8 100644 --- a/Examples/Tests/Implicit/inputs_1d_semiimplicit +++ b/Examples/Tests/Implicit/inputs_1d_semiimplicit @@ -31,10 +31,18 @@ boundary.particle_hi = periodic ############ NUMERICS ########### ################################# +warpx.verbose = 1 warpx.const_dt = dt -algo.evolve_scheme = semi_implicit_picard -algo.max_picard_iterations = 5 -algo.picard_iteration_tolerance = 0. +algo.evolve_scheme = semi_implicit_em + +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 5 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + algo.current_deposition = esirkepov algo.field_gathering = energy-conserving algo.particle_shape = 2 diff --git a/Examples/Tests/Implicit/inputs_vandb_2d b/Examples/Tests/Implicit/inputs_vandb_jfnk_2d similarity index 78% rename from Examples/Tests/Implicit/inputs_vandb_2d rename to Examples/Tests/Implicit/inputs_vandb_jfnk_2d index 33ce964710a..393a9d90330 100644 --- a/Examples/Tests/Implicit/inputs_vandb_2d +++ b/Examples/Tests/Implicit/inputs_vandb_jfnk_2d @@ -38,10 +38,32 @@ warpx.const_dt = dt warpx.use_filter = 0 algo.maxwell_solver = Yee -algo.evolve_scheme = "implicit_picard" -algo.require_picard_convergence = 0 -algo.max_picard_iterations = 25 -algo.picard_iteration_tolerance = 0.0 #1.0e-12 +algo.evolve_scheme = "theta_implicit_em" +#algo.evolve_scheme = "semi_implicit_em" + +implicit_evolve.theta = 0.5 +implicit_evolve.max_particle_iterations = 21 +implicit_evolve.particle_tolerance = 1.0e-12 + +#implicit_evolve.nonlinear_solver = "picard" +#picard.verbose = true +#picard.max_iterations = 25 +#picard.relative_tolerance = 0.0 #1.0e-12 +#picard.absolute_tolerance = 0.0 #1.0e-24 +#picard.require_convergence = false + +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 20 +newton.relative_tolerance = 1.0e-12 +newton.absolute_tolerance = 0.0 +newton.require_convergence = false + +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-8 +gmres.absolute_tolerance = 0.0 + algo.particle_pusher = "boris" #algo.particle_pusher = "higuera" diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json b/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json rename to Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d.json diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/ThetaImplicitPicard_1d.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json rename to Regression/Checksum/benchmarks_json/ThetaImplicitPicard_1d.json diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 6ee7ad3929d..e2d69be9c60 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -4666,7 +4666,7 @@ particleTypes = electrons outputFile = Point_of_contact_EB_rz_plt analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py -[ImplicitPicard_1d] +[ThetaImplicitPicard_1d] buildDir = . inputFile = Examples/Tests/Implicit/inputs_1d runtime_params = warpx.abort_on_warning_threshold=high @@ -4683,9 +4683,9 @@ doVis = 0 compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py -[ImplicitPicard_VandB_2d] +[ThetaImplicitJFNK_VandB_2d] buildDir = . -inputFile = Examples/Tests/Implicit/inputs_vandb_2d +inputFile = Examples/Tests/Implicit/inputs_vandb_jfnk_2d runtime_params = warpx.abort_on_warning_threshold=high dim = 2 addToCompileString = @@ -4698,7 +4698,7 @@ numthreads = 1 compileTest = 0 doVis = 0 compareParticles = 1 -analysisRoutine = Examples/Tests/Implicit/analysis_vandb_2d.py +analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py [SemiImplicitPicard_1d] buildDir = . diff --git a/Source/Evolve/CMakeLists.txt b/Source/Evolve/CMakeLists.txt index a84d6e1e42b..3ef0dae5e8e 100644 --- a/Source/Evolve/CMakeLists.txt +++ b/Source/Evolve/CMakeLists.txt @@ -4,6 +4,5 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE WarpXEvolve.cpp WarpXComputeDt.cpp - WarpXOneStepImplicitPicard.cpp ) endforeach() diff --git a/Source/Evolve/Make.package b/Source/Evolve/Make.package index a9a877475b9..275b8bfde5a 100644 --- a/Source/Evolve/Make.package +++ b/Source/Evolve/Make.package @@ -1,5 +1,4 @@ CEXE_sources += WarpXEvolve.cpp CEXE_sources += WarpXComputeDt.cpp -CEXE_sources += WarpXOneStepImplicitPicard.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Evolve diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 881967a0479..4ac50483ad3 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -125,10 +125,8 @@ WarpX::Evolve (int numsteps) ExecutePythonCallback("particleinjection"); - // TODO: move out - if (evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard) { - OneStep_ImplicitPicard(cur_time); + if (m_implicit_solver) { + m_implicit_solver->OneStep(cur_time, dt[0], step); } else if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) diff --git a/Source/Evolve/WarpXOneStepImplicitPicard.cpp b/Source/Evolve/WarpXOneStepImplicitPicard.cpp deleted file mode 100644 index 6df3cd0d4e6..00000000000 --- a/Source/Evolve/WarpXOneStepImplicitPicard.cpp +++ /dev/null @@ -1,423 +0,0 @@ -/* Copyright 2022 David Grote - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#include "WarpX.H" - -#include "BoundaryConditions/PML.H" -#include "Diagnostics/MultiDiagnostics.H" -#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" -#include "Evolve/WarpXDtType.H" -#include "Evolve/WarpXPushType.H" -#ifdef WARPX_USE_FFT -# ifdef WARPX_DIM_RZ -# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" -# else -# include "FieldSolver/SpectralSolver/SpectralSolver.H" -# endif -#endif -#include "Parallelization/GuardCellManager.H" -#include "Particles/MultiParticleContainer.H" -#include "Particles/ParticleBoundaryBuffer.H" -#include "Python/callbacks.H" -#include "Utils/TextMsg.H" -#include "Utils/WarpXAlgorithmSelection.H" -#include "Utils/WarpXUtil.H" -#include "Utils/WarpXConst.H" -#include "Utils/WarpXProfilerWrapper.H" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -void -WarpX::EvolveImplicitPicardInit (int lev) -{ - - if (lev == 0) { - // Add space to save the positions and velocities at the start of the time steps - for (auto const& pc : *mypc) { -#if (AMREX_SPACEDIM >= 2) - pc->AddRealComp("x_n"); -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - pc->AddRealComp("y_n"); -#endif - pc->AddRealComp("z_n"); - pc->AddRealComp("ux_n"); - pc->AddRealComp("uy_n"); - pc->AddRealComp("uz_n"); - } - } - - // Initialize MultiFabs to hold the E and B fields at the start of the time steps - // Only one refinement level is supported - const int nlevs_max = maxLevel() + 1; - Efield_n.resize(nlevs_max); - Efield_save.resize(nlevs_max); - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - Bfield_n.resize(nlevs_max); - Bfield_save.resize(nlevs_max); - } - - // The Efield_n and Bfield_n will hold the fields at the start of the time step. - // This is needed since in each iteration the fields are advanced from the values - // at the start of the step. - // The Efield_save and Bfield_save will hold the fields from the previous iteration, - // to check the change in the fields after the iterations to check for convergence. - // The Efiel_fp and Bfield_fp will hole the n+theta during the iterations and then - // advance to the n+1 time level after the iterations complete. - AllocInitMultiFabFromModel(Efield_n[lev][0], *Efield_fp[0][0], lev, "Efield_n[0]"); - AllocInitMultiFabFromModel(Efield_n[lev][1], *Efield_fp[0][1], lev, "Efield_n[1]"); - AllocInitMultiFabFromModel(Efield_n[lev][2], *Efield_fp[0][2], lev, "Efield_n[2]"); - AllocInitMultiFabFromModel(Efield_save[lev][0], *Efield_fp[0][0], lev, "Efield_save[0]"); - AllocInitMultiFabFromModel(Efield_save[lev][1], *Efield_fp[0][1], lev, "Efield_save[1]"); - AllocInitMultiFabFromModel(Efield_save[lev][2], *Efield_fp[0][2], lev, "Efield_save[2]"); - - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - AllocInitMultiFabFromModel(Bfield_n[lev][0], *Bfield_fp[0][0], lev, "Bfield_n[0]"); - AllocInitMultiFabFromModel(Bfield_n[lev][1], *Bfield_fp[0][1], lev, "Bfield_n[1]"); - AllocInitMultiFabFromModel(Bfield_n[lev][2], *Bfield_fp[0][2], lev, "Bfield_n[2]"); - AllocInitMultiFabFromModel(Bfield_save[lev][0], *Bfield_fp[0][0], lev, "Bfield_save[0]"); - AllocInitMultiFabFromModel(Bfield_save[lev][1], *Bfield_fp[0][1], lev, "Bfield_save[1]"); - AllocInitMultiFabFromModel(Bfield_save[lev][2], *Bfield_fp[0][2], lev, "Bfield_save[2]"); - } - -} - -void -WarpX::OneStep_ImplicitPicard(amrex::Real cur_time) -{ - using namespace amrex::literals; - - // We have E^{n}. - // Particles have p^{n} and x^{n}. - // With full implicit, B^{n} - // With semi-implicit, B^{n-1/2} - - // Save the values at the start of the time step, - // copying particle data to x_n etc. - for (auto const& pc : *mypc) { - SaveParticlesAtImplicitStepStart (*pc, 0); - } - - // Save the fields at the start of the step - amrex::MultiFab::Copy(*Efield_n[0][0], *Efield_fp[0][0], 0, 0, ncomps, Efield_fp[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_n[0][1], *Efield_fp[0][1], 0, 0, ncomps, Efield_fp[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_n[0][2], *Efield_fp[0][2], 0, 0, ncomps, Efield_fp[0][2]->nGrowVect()); - - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - amrex::MultiFab::Copy(*Bfield_n[0][0], *Bfield_fp[0][0], 0, 0, ncomps, Bfield_fp[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_n[0][1], *Bfield_fp[0][1], 0, 0, ncomps, Bfield_fp[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_n[0][2], *Bfield_fp[0][2], 0, 0, ncomps, Bfield_fp[0][2]->nGrowVect()); - } else if (evolve_scheme == EvolveScheme::SemiImplicitPicard) { - // This updates Bfield_fp so it holds the new B at n+1/2 - EvolveB(dt[0], DtType::Full); - // WarpX::sync_nodal_points is used to avoid instability - FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); - } - - // Start the iterations - amrex::Real deltaE = 1._rt; - amrex::Real deltaB = 1._rt; - int iteration_count = 0; - while (iteration_count < max_picard_iterations && - (deltaE > picard_iteration_tolerance || deltaB > picard_iteration_tolerance)) { - iteration_count++; - - // Advance the particle positions by 1/2 dt, - // particle velocities by dt, then take average of old and new v, - // deposit currents, giving J at n+1/2 - // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration. - const bool skip_current = false; - const PushType push_type = PushType::Implicit; - PushParticlesandDeposit(cur_time, skip_current, push_type); - - SyncCurrentAndRho(); - - if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { - // Save the E at n+1/2 from the previous iteration so that the change - // in this iteration can be calculated - amrex::MultiFab::Copy(*Efield_save[0][0], *Efield_fp[0][0], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Efield_save[0][1], *Efield_fp[0][1], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Efield_save[0][2], *Efield_fp[0][2], 0, 0, ncomps, 0); - } - - // Copy Efield_n into Efield_fp since EvolveE updates Efield_fp in place - amrex::MultiFab::Copy(*Efield_fp[0][0], *Efield_n[0][0], 0, 0, ncomps, Efield_n[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_fp[0][1], *Efield_n[0][1], 0, 0, ncomps, Efield_n[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_fp[0][2], *Efield_n[0][2], 0, 0, ncomps, Efield_n[0][2]->nGrowVect()); - - // Updates Efield_fp so it holds the new E at n+1/2 - EvolveE(0.5_rt*dt[0]); - // WarpX::sync_nodal_points is used to avoid instability - FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyEfieldBoundary(0, PatchType::fine); - - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { - // Save the B at n+1/2 from the previous iteration so that the change - // in this iteration can be calculated - amrex::MultiFab::Copy(*Bfield_save[0][0], *Bfield_fp[0][0], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Bfield_save[0][1], *Bfield_fp[0][1], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Bfield_save[0][2], *Bfield_fp[0][2], 0, 0, ncomps, 0); - } - - // Copy Bfield_n into Bfield_fp since EvolveB updates Bfield_fp in place - amrex::MultiFab::Copy(*Bfield_fp[0][0], *Bfield_n[0][0], 0, 0, ncomps, Bfield_n[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_fp[0][1], *Bfield_n[0][1], 0, 0, ncomps, Bfield_n[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_fp[0][2], *Bfield_n[0][2], 0, 0, ncomps, Bfield_n[0][2]->nGrowVect()); - - // This updates Bfield_fp so it holds the new B at n+1/2 - EvolveB(0.5_rt*dt[0], DtType::Full); - // WarpX::sync_nodal_points is used to avoid instability - FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); - } - - // The B field update needs - if (num_mirrors>0){ - applyMirrors(cur_time); - // E : guard cells are NOT up-to-date from the mirrors - // B : guard cells are NOT up-to-date from the mirrors - } - - if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { - // Calculate the change in E and B from this iteration - // deltaE = abs(Enew - Eold)/max(abs(Enew)) - Efield_save[0][0]->minus(*Efield_fp[0][0], 0, ncomps, 0); - Efield_save[0][1]->minus(*Efield_fp[0][1], 0, ncomps, 0); - Efield_save[0][2]->minus(*Efield_fp[0][2], 0, ncomps, 0); - const amrex::Real maxE0 = std::max(1._rt, Efield_fp[0][0]->norm0(0, 0)); - const amrex::Real maxE1 = std::max(1._rt, Efield_fp[0][1]->norm0(0, 0)); - const amrex::Real maxE2 = std::max(1._rt, Efield_fp[0][2]->norm0(0, 0)); - const amrex::Real deltaE0 = Efield_save[0][0]->norm0(0, 0)/maxE0; - const amrex::Real deltaE1 = Efield_save[0][1]->norm0(0, 0)/maxE1; - const amrex::Real deltaE2 = Efield_save[0][2]->norm0(0, 0)/maxE2; - deltaE = std::max(std::max(deltaE0, deltaE1), deltaE2); - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - Bfield_save[0][0]->minus(*Bfield_fp[0][0], 0, ncomps, 0); - Bfield_save[0][1]->minus(*Bfield_fp[0][1], 0, ncomps, 0); - Bfield_save[0][2]->minus(*Bfield_fp[0][2], 0, ncomps, 0); - const amrex::Real maxB0 = std::max(1._rt, Bfield_fp[0][0]->norm0(0, 0)); - const amrex::Real maxB1 = std::max(1._rt, Bfield_fp[0][1]->norm0(0, 0)); - const amrex::Real maxB2 = std::max(1._rt, Bfield_fp[0][2]->norm0(0, 0)); - const amrex::Real deltaB0 = Bfield_save[0][0]->norm0(0, 0)/maxB0; - const amrex::Real deltaB1 = Bfield_save[0][1]->norm0(0, 0)/maxB1; - const amrex::Real deltaB2 = Bfield_save[0][2]->norm0(0, 0)/maxB2; - deltaB = std::max(std::max(deltaB0, deltaB1), deltaB2); - } else { - deltaB = 0.; - } - amrex::Print() << "Max delta " << iteration_count << " " << deltaE << " " << deltaB << "\n"; - } - - // Now, the particle positions and velocities and the Efield_fp and Bfield_fp hold - // the new values at n+1/2 - } - - amrex::Print() << "Picard iterations = " << iteration_count << ", Eerror = " << deltaE << ", Berror = " << deltaB << "\n"; - if (picard_iteration_tolerance > 0. && iteration_count == max_picard_iterations) { - std::stringstream convergenceMsg; - convergenceMsg << "The Picard implicit solver failed to converge after " << iteration_count << " iterations, with Eerror = " << deltaE << ", Berror = " << deltaB << " with a tolerance of " << picard_iteration_tolerance; - if (require_picard_convergence) { - WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); - } else { - ablastr::warn_manager::WMRecordWarning("PicardSolver", convergenceMsg.str()); - } - } - - // Advance particles to step n+1 - for (auto const& pc : *mypc) { - FinishImplicitParticleUpdate(*pc, 0); - } - - // Advance fields to step n+1 - // WarpX::sync_nodal_points is used to avoid instability - FinishImplicitFieldUpdate(Efield_fp, Efield_n); - FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - FinishImplicitFieldUpdate(Bfield_fp, Bfield_n); - FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - } - -} - -void -WarpX::SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev) -{ - -#ifdef AMREX_USE_OMP -#pragma omp parallel -#endif - { - - auto particle_comps = pc.getParticleComps(); - - for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { - - const auto getPosition = GetParticlePosition(pti); - - auto& attribs = pti.GetAttribs(); - amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - -#if (AMREX_SPACEDIM >= 2) - amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); -#endif - amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); - amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); - amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); - amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); - - const long np = pti.numParticles(); - - amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) - { - amrex::ParticleReal xp, yp, zp; - getPosition(ip, xp, yp, zp); - -#if (AMREX_SPACEDIM >= 2) - x_n[ip] = xp; -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - y_n[ip] = yp; -#endif - z_n[ip] = zp; - - ux_n[ip] = ux[ip]; - uy_n[ip] = uy[ip]; - uz_n[ip] = uz[ip]; - - }); - - } - } -} - -void -WarpX::FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev) -{ - using namespace amrex::literals; - -#ifdef AMREX_USE_OMP -#pragma omp parallel -#endif - { - - auto particle_comps = pc.getParticleComps(); - - for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { - - const auto getPosition = GetParticlePosition(pti); - const auto setPosition = SetParticlePosition(pti); - - auto& attribs = pti.GetAttribs(); - amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - -#if (AMREX_SPACEDIM >= 2) - amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); -#endif - amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); - amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); - amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); - amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); - - const long np = pti.numParticles(); - - amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) - { - amrex::ParticleReal xp, yp, zp; - getPosition(ip, xp, yp, zp); - -#if (AMREX_SPACEDIM >= 2) - xp = 2._rt*xp - x_n[ip]; -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - yp = 2._rt*yp - y_n[ip]; -#endif - zp = 2._rt*zp - z_n[ip]; - - ux[ip] = 2._rt*ux[ip] - ux_n[ip]; - uy[ip] = 2._rt*uy[ip] - uy_n[ip]; - uz[ip] = 2._rt*uz[ip] - uz_n[ip]; - - setPosition(ip, xp, yp, zp); - }); - - } - } -} - -void -WarpX::FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Field_fp, - amrex::Vector, 3 > >& Field_n) -{ - using namespace amrex::literals; - - for (int lev = 0; lev <= finest_level; ++lev) { - -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for ( amrex::MFIter mfi(*Field_fp[lev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { - - amrex::Array4 const& Fx = Field_fp[lev][0]->array(mfi); - amrex::Array4 const& Fy = Field_fp[lev][1]->array(mfi); - amrex::Array4 const& Fz = Field_fp[lev][2]->array(mfi); - - amrex::Array4 const& Fx_n = Field_n[lev][0]->array(mfi); - amrex::Array4 const& Fy_n = Field_n[lev][1]->array(mfi); - amrex::Array4 const& Fz_n = Field_n[lev][2]->array(mfi); - - amrex::Box const tbx = mfi.tilebox(Field_fp[lev][0]->ixType().toIntVect()); - amrex::Box const tby = mfi.tilebox(Field_fp[lev][1]->ixType().toIntVect()); - amrex::Box const tbz = mfi.tilebox(Field_fp[lev][2]->ixType().toIntVect()); - - amrex::ParallelFor( - tbx, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) - { - Fx(i,j,k,n) = 2._rt*Fx(i,j,k,n) - Fx_n(i,j,k,n); - }, - tby, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) - { - Fy(i,j,k,n) = 2._rt*Fy(i,j,k,n) - Fy_n(i,j,k,n); - }, - tbz, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) - { - Fz(i,j,k,n) = 2._rt*Fz(i,j,k,n) - Fz_n(i,j,k,n); - }); - } - } -} diff --git a/Source/FieldSolver/CMakeLists.txt b/Source/FieldSolver/CMakeLists.txt index cf741aa6e01..859117eb214 100644 --- a/Source/FieldSolver/CMakeLists.txt +++ b/Source/FieldSolver/CMakeLists.txt @@ -11,6 +11,7 @@ endforeach() add_subdirectory(FiniteDifferenceSolver) add_subdirectory(MagnetostaticSolver) +add_subdirectory(ImplicitSolvers) if(WarpX_FFT) add_subdirectory(SpectralSolver) endif() diff --git a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt new file mode 100644 index 00000000000..6e16f19084c --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt @@ -0,0 +1,10 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + SemiImplicitEM.cpp + ThetaImplicitEM.cpp + WarpXImplicitOps.cpp + WarpXSolverVec.cpp + ) +endforeach() diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H new file mode 100644 index 00000000000..88ad6a058fd --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H @@ -0,0 +1,145 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef Implicit_Solver_H_ +#define Implicit_Solver_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" +#include "NonlinearSolvers/NonlinearSolverLibrary.H" + +#include +#include + +/** + * \brief Base class for implicit time solvers. The base functions are those + * needed by an implicit solver to be used through WarpX and those needed + * to use the nonlinear solvers, such as Picard or Newton (i.e., JFNK). + */ + +class WarpX; +class ImplicitSolver +{ +public: + + ImplicitSolver() = default; + + virtual ~ImplicitSolver() = default; + + // Prohibit Move and Copy operations + ImplicitSolver(const ImplicitSolver&) = delete; + ImplicitSolver& operator=(const ImplicitSolver&) = delete; + ImplicitSolver(ImplicitSolver&&) = delete; + ImplicitSolver& operator=(ImplicitSolver&&) = delete; + + // + // the following routines are called by WarpX + // + + /** + * \brief Read user-provided parameters that control the implicit solver. + * Allocate internal arrays for intermediate field values needed by the solver. + */ + virtual void Define ( WarpX* a_WarpX ) = 0; + + [[nodiscard]] bool IsDefined () const { return m_is_defined; } + + virtual void PrintParameters () const = 0; + + void GetParticleSolverParams (int& a_max_particle_iter, + amrex::ParticleReal& a_particle_tol ) const + { + a_max_particle_iter = m_max_particle_iterations; + a_particle_tol = m_particle_tolerance; + } + + /** + * \brief Advance fields and particles by one time step using the specified implicit algorithm + */ + virtual void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) = 0; + + // + // the following routines are called by the linear and nonlinear solvers + // + + /** + * \brief Computes the RHS of the equation corresponding to the specified implicit algorithm. + * The discrete equations corresponding to numerical integration of ODEs are often + * written in the form U = b + RHS(U), where U is the variable to be solved for (e.g., + * the solution at the next time step), b is a constant (i.e., the solution from the + * previous time step), and RHS(U) is the right-hand-side of the equation. Iterative + * solvers, such as Picard and Newton, and higher-order Runge-Kutta methods, need to + * compute RHS(U) multiple times for different values of U. Thus, a routine that + * returns this value is needed. + * e.g., Ebar - E^n = cvac^2*0.5*dt*(curl(Bbar) - mu0*Jbar(Ebar,Bbar)) + * Here, U = Ebar, b = E^n, and the expression on the right is RHS(U). + */ + virtual void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) = 0; + +protected: + + /** + * \brief Pointer back to main WarpX class + */ + WarpX* m_WarpX; + + bool m_is_defined = false; + + /** + * \brief Nonlinear solver type and object + */ + NonlinearSolverType m_nlsolver_type; + std::unique_ptr> m_nlsolver; + + /** + * \brief tolerance used by the iterative method used to obtain a self-consistent + * update of the particle positions and velocities for given E and B on the grid + */ + amrex::ParticleReal m_particle_tolerance = 1.0e-10; + + /** + * \brief maximum iterations for the iterative method used to obtain a self-consistent + * update of the particle positions and velocities for given E and B on the grid + */ + int m_max_particle_iterations = 21; + + /** + * \brief parse nonlinear solver parameters (if one is used) + */ + void parseNonlinearSolverParams( const amrex::ParmParse& pp ) + { + + std::string nlsolver_type_str; + pp.get("nonlinear_solver", nlsolver_type_str); + + if (nlsolver_type_str=="picard") { + m_nlsolver_type = NonlinearSolverType::Picard; + m_nlsolver = std::make_unique>(); + m_max_particle_iterations = 1; + m_particle_tolerance = 0.0; + } + else if (nlsolver_type_str=="newton") { + m_nlsolver_type = NonlinearSolverType::Newton; + m_nlsolver = std::make_unique>(); + pp.query("max_particle_iterations", m_max_particle_iterations); + pp.query("particle_tolerance", m_particle_tolerance); + } + else { + WARPX_ABORT_WITH_MESSAGE( + "invalid nonlinear_solver specified. Valid options are picard and newton."); + } + + } + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H new file mode 100644 index 00000000000..423957ef061 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H @@ -0,0 +1,13 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef IMPLICIT_SOLVER_LIBRARY_H_ +#define IMPLICIT_SOLVER_LIBRARY_H_ + +#include "SemiImplicitEM.H" // IWYU pragma: export +#include "ThetaImplicitEM.H" // IWYU pragma: export + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/Make.package b/Source/FieldSolver/ImplicitSolvers/Make.package new file mode 100644 index 00000000000..a4543f94dd3 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/Make.package @@ -0,0 +1,6 @@ +CEXE_sources += SemiImplicitEM.cpp +CEXE_sources += ThetaImplicitEM.cpp +CEXE_sources += WarpXImplicitOps.cpp +CEXE_sources += WarpXSolverVec.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver/ImplicitSolvers diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H new file mode 100644 index 00000000000..6e3e5db2c74 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H @@ -0,0 +1,80 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef SEMI_IMPLICIT_EM_H_ +#define SEMI_IMPLICIT_EM_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +#include +#include +#include + +#include "ImplicitSolver.H" + +/** @file + * Semi-implicit electromagnetic time solver class. The electric field and the + * particles are implicitly coupled in this algorithm, but the magnetic field + * is advanced in the standard explicit leap-frog manner (hence semi-implicit). + * + * The time stencil is + * Eg^{n+1} = Eg^n + c^2*dt*( curlBg^{n+1/2} - mu0*Jg^{n+1/2} ) + * Bg^{n+3/2} = Bg^{n+1/2} - dt*curlEg^{n+1} + * xp^{n+1} = xp^n + dt*up^{n+1/2}/(0.5(gammap^n + gammap^{n+1})) + * up^{n+1} = up^n + dt*qp/mp*(Ep^{n+1/2} + up^{n+1/2}/gammap^{n+1/2} x Bp^{n+1/2}) + * where f^{n+1/2} = (f^{n+1} + f^n)/2.0, for all but Bg, which lives at half steps + * + * This algorithm is approximately energy conserving. It is exactly energy conserving + * using a non-standard definition for the magnetic field energy. The advantage of + * this method over the exactly energy-conserving theta-implicit EM method is that + * light wave dispersion is captured much better. However, the CFL condition for light + * waves has to be satisifed for numerical stability (and for the modified definition + * of the magnetic field energy to be well-posed). + * + * See G. Chen, L. Chacon, L. Yin, B.J. Albright, D.J. Stark, R.F. Bird, + * "A semi-implicit energy- and charge-conserving particle-in-cell algorithm for the + * relativistic Vlasov-Maxwell equations.", JCP 407 (2020). + */ + +class SemiImplicitEM : public ImplicitSolver +{ +public: + + SemiImplicitEM() = default; + + ~SemiImplicitEM() override = default; + + // Prohibit Move and Copy operations + SemiImplicitEM(const SemiImplicitEM&) = delete; + SemiImplicitEM& operator=(const SemiImplicitEM&) = delete; + SemiImplicitEM(SemiImplicitEM&&) = delete; + SemiImplicitEM& operator=(SemiImplicitEM&&) = delete; + + void Define ( WarpX* a_WarpX ) override; + + void PrintParameters () const override; + + void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) override; + + void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) override; + +private: + + /** + * \brief Solver vectors for E and Eold + */ + WarpXSolverVec m_E, m_Eold; + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp new file mode 100644 index 00000000000..897bd5c07f3 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp @@ -0,0 +1,117 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "SemiImplicitEM.H" +#include "WarpX.H" + +using namespace warpx::fields; +using namespace amrex::literals; + +void SemiImplicitEM::Define ( WarpX* a_WarpX ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !m_is_defined, + "SemiImplicitEM object is already defined!"); + + // Retain a pointer back to main WarpX class + m_WarpX = a_WarpX; + + // Define E and Eold vectors + m_E.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_Eold.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + + // Need to define the WarpXSolverVec owned dot_mask to do dot + // product correctly for linear and nonlinear solvers + const amrex::Vector& Geom = m_WarpX->Geom(); + m_E.SetDotMask(Geom); + + // Parse implicit solver parameters + const amrex::ParmParse pp("implicit_evolve"); + parseNonlinearSolverParams( pp ); + + // Define the nonlinear solver + m_nlsolver->Define(m_E, this); + m_is_defined = true; + +} + +void SemiImplicitEM::PrintParameters () const +{ + if (!m_WarpX->Verbose()) { return; } + amrex::Print() << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "----------- SEMI IMPLICIT EM SOLVER PARAMETERS ------------" << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << std::endl; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << std::endl; + if (m_nlsolver_type==NonlinearSolverType::Picard) { + amrex::Print() << "Nonlinear solver type: Picard" << std::endl; + } + else if (m_nlsolver_type==NonlinearSolverType::Newton) { + amrex::Print() << "Nonlinear solver type: Newton" << std::endl; + } + m_nlsolver->PrintParams(); + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << std::endl; +} + +void SemiImplicitEM::OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) +{ + amrex::ignore_unused(a_step); + + // Fields have E^{n}, B^{n-1/2} + // Particles have p^{n} and x^{n}. + + // Save the values at the start of the time step, + m_WarpX->SaveParticlesAtImplicitStepStart ( ); + + // Save the fields at the start of the step + m_Eold.Copy( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_E.Copy(m_Eold); // initial guess for E + + // Compute Bfield at time n+1/2 + m_WarpX->EvolveB(a_dt, DtType::Full); + m_WarpX->ApplyMagneticFieldBCs(); + + const amrex::Real half_time = a_time + 0.5_rt*a_dt; + + // Solve nonlinear system for E at t_{n+1/2} + // Particles will be advanced to t_{n+1/2} + m_nlsolver->Solve( m_E, m_Eold, half_time, a_dt ); + + // Update WarpX owned Efield_fp to t_{n+1/2} + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + + // Advance particles from time n+1/2 to time n+1 + m_WarpX->FinishImplicitParticleUpdate(); + + // Advance E fields from time n+1/2 to time n+1 + // Eg^{n+1} = 2.0*E_g^{n+1/2} - E_g^n + m_E.linComb( 2._rt, m_E, -1._rt, m_Eold ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + +} + +void SemiImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + // update WarpX-owned Efield_fp using current state of E from + // the nonlinear solver at time n+theta + m_WarpX->SetElectricFieldAndApplyBCs( a_E ); + + // Self consistently update particle positions and velocities using the + // current state of the fields E and B. Deposit current density at time n+1/2. + m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + + // RHS = cvac^2*0.5*dt*( curl(B^{n+1/2}) - mu0*J^{n+1/2} ) + m_WarpX->ImplicitComputeRHSE(0.5_rt*a_dt, a_RHS); +} diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H new file mode 100644 index 00000000000..009c2c7e546 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H @@ -0,0 +1,124 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef THETA_IMPLICIT_EM_H_ +#define THETA_IMPLICIT_EM_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +#include +#include +#include + +#include "ImplicitSolver.H" + +/** @file + * Theta-implicit electromagnetic time solver class. This is a fully implicit + * algorithm where both the fields and particles are treated implicitly. + * + * The time stencil is + * Eg^{n+1} = Eg^n + c^2*dt*( curlBg^{n+theta} - mu0*Jg^{n+1/2} ) + * Bg^{n+1} = Bg^n - dt*curlEg^{n+theta} + * xp^{n+1} = xp^n + dt*up^{n+1/2}/(0.5*(gammap^n + gammap^{n+1})) + * up^{n+1} = up^n + dt*qp/mp*(Ep^{n+theta} + up^{n+1/2}/gammap^{n+1/2} x Bp^{n+theta}) + * where f^{n+theta} = (1.0-theta)*f^{n} + theta*f^{n+1} with 0.5 <= theta <= 1.0 + * + * The user-specified time-biasing parameter theta used for the fields on the RHS is bound + * between 0.5 and 1.0. The algorithm is exactly energy conserving for theta = 0.5. + * Signifcant damping of high-k modes will occur as theta approaches 1.0. The algorithm is + * numerially stable for any time step. I.e., the CFL condition for light waves does not + * have to be satisifed and the time step is not limited by the plasma period. However, how + * efficiently the algorithm can use large time steps depends strongly on the nonlinear solver. + * Furthermore, the time step should always be such that particles do not travel outside the + * ghost region of the box they live in, which is an MPI-related limitation. The time step + * is always limited by the need to resolve the appropriate physics. + * + * See S. Markidis, G. Lapenta, "The energy conserving particle-in-cell method." JCP 230 (2011). + * + * See G. Chen, L. Chacon, D.C. Barnes, "An energy- and charge-conserving, implicit, + * elctrostatic particle-in-cell algorithm." JCP 230 (2011). + * + * See J.R. Angus, A. Link, A. Friedman, D. Ghosh, J. D. Johnson, "On numerical energy + * conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo + * algorithm for Coulomb collisions.", JCP 456 (2022). + * + * See J.R. Angus, W. Farmer, A. Friedman, D. Ghosh, D. Grote, D. Larson, A. Link, "An + * implicit particle code with exact energy and charge conservation for electromagnetic studies + * of dense plasmas.", JCP 491 (2023). + */ + +class ThetaImplicitEM : public ImplicitSolver +{ +public: + + ThetaImplicitEM() = default; + + ~ThetaImplicitEM() override = default; + + // Prohibit Move and Copy operations + ThetaImplicitEM(const ThetaImplicitEM&) = delete; + ThetaImplicitEM& operator=(const ThetaImplicitEM&) = delete; + ThetaImplicitEM(ThetaImplicitEM&&) = delete; + ThetaImplicitEM& operator=(ThetaImplicitEM&&) = delete; + + void Define ( WarpX* a_WarpX ) override; + + void PrintParameters () const override; + + void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) override; + + void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) override; + + [[nodiscard]] amrex::Real theta () const { return m_theta; } + +private: + + /** + * \brief Time-biasing parameter for fields used on RHS to advance system + */ + amrex::Real m_theta = 0.5; + + /** + * \brief Solver vectors to be used in the nonlinear solver to solve for the + * electric field E. The main logic for determining which variables should be + * WarpXSolverVec type is that it must have the same size and have the same + * centering of the data as the variable being solved for, which is E here. + * For example, if using a Yee grid then a container for curlB could be a + * WarpXSovlerVec, but magnetic field B should not be. + */ + WarpXSolverVec m_E, m_Eold; + + /** + * \brief B is a derived variable from E. Need to save Bold to update B during + * the iterative nonlinear solve for E. Bold is owned here, but only used by WarpX. + * It is not used directly by the nonlinear solver, nor is it the same size as the + * solver vector (size E), and so it should not be WarpXSolverVec type. + */ + amrex::Vector, 3 > > m_Bold; + + /** + * \brief Update the E and B fields owned by WarpX + */ + void UpdateWarpXFields ( const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt ); + + /** + * \brief Nonlinear solver is for the time-centered values of E. After + * the solver, need to use m_E and m_Eold to compute E^{n+1} + */ + void FinishFieldUpdate ( amrex::Real a_new_time ); + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp new file mode 100644 index 00000000000..026c509c3ba --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -0,0 +1,171 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "FieldSolver/Fields.H" +#include "ThetaImplicitEM.H" +#include "WarpX.H" + +using namespace warpx::fields; +using namespace amrex::literals; + +void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !m_is_defined, + "ThetaImplicitEM object is already defined!"); + + // Retain a pointer back to main WarpX class + m_WarpX = a_WarpX; + + // Define E and Eold vectors + m_E.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_Eold.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + + // Need to define the WarpXSolverVec owned dot_mask to do dot + // product correctly for linear and nonlinear solvers + const amrex::Vector& Geom = m_WarpX->Geom(); + m_E.SetDotMask(Geom); + + // Define Bold MultiFab + const int num_levels = 1; + m_Bold.resize(num_levels); // size is number of levels + for (int lev = 0; lev < num_levels; ++lev) { + for (int n=0; n<3; n++) { + const amrex::MultiFab& Bfp = m_WarpX->getField( FieldType::Bfield_fp,lev,n); + m_Bold[lev][n] = std::make_unique( Bfp.boxArray(), + Bfp.DistributionMap(), + Bfp.nComp(), + Bfp.nGrowVect() ); + } + } + + // Parse theta-implicit solver specific parameters + const amrex::ParmParse pp("implicit_evolve"); + pp.query("theta", m_theta); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_theta>=0.5 && m_theta<=1.0, + "theta parameter for theta implicit time solver must be between 0.5 and 1.0"); + + // Parse nonlinear solver parameters + parseNonlinearSolverParams( pp ); + + // Define the nonlinear solver + m_nlsolver->Define(m_E, this); + m_is_defined = true; + +} + +void ThetaImplicitEM::PrintParameters () const +{ + if (!m_WarpX->Verbose()) { return; } + amrex::Print() << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "----------- THETA IMPLICIT EM SOLVER PARAMETERS -----------" << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "Time-bias parameter theta: " << m_theta << std::endl; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << std::endl; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << std::endl; + if (m_nlsolver_type==NonlinearSolverType::Picard) { + amrex::Print() << "Nonlinear solver type: Picard" << std::endl; + } + else if (m_nlsolver_type==NonlinearSolverType::Newton) { + amrex::Print() << "Nonlinear solver type: Newton" << std::endl; + } + m_nlsolver->PrintParams(); + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << std::endl; +} + +void ThetaImplicitEM::OneStep ( const amrex::Real a_time, + const amrex::Real a_dt, + const int a_step ) +{ + amrex::ignore_unused(a_step); + + // Fields have E^{n} and B^{n} + // Particles have p^{n} and x^{n}. + + // Save the values at the start of the time step, + m_WarpX->SaveParticlesAtImplicitStepStart ( ); + + // Save the fields at the start of the step + m_Eold.Copy( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_E.Copy(m_Eold); // initial guess for E + + const int num_levels = static_cast(m_Bold.size()); + for (int lev = 0; lev < num_levels; ++lev) { + for (int n=0; n<3; n++) { + const amrex::MultiFab& Bfp = m_WarpX->getField(FieldType::Bfield_fp,lev,n); + amrex::MultiFab& Bold = *m_Bold[lev][n]; + amrex::MultiFab::Copy(Bold, Bfp, 0, 0, 1, Bold.nGrowVect()); + } + } + + const amrex::Real theta_time = a_time + m_theta*a_dt; + + // Solve nonlinear system for E at t_{n+theta} + // Particles will be advanced to t_{n+1/2} + m_nlsolver->Solve( m_E, m_Eold, theta_time, a_dt ); + + // Update WarpX owned Efield_fp and Bfield_fp to t_{n+theta} + UpdateWarpXFields( m_E, theta_time, a_dt ); + + // Advance particles from time n+1/2 to time n+1 + m_WarpX->FinishImplicitParticleUpdate(); + + // Advance E and B fields from time n+theta to time n+1 + const amrex::Real new_time = a_time + a_dt; + FinishFieldUpdate( new_time ); + +} + +void ThetaImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + // update WarpX-owned Efield_fp and Bfield_fp using current state of E from + // the nonlinear solver at time n+theta + UpdateWarpXFields( a_E, a_time, a_dt ); + + // Self consistently update particle positions and velocities using the + // current state of the fields E and B. Deposit current density at time n+1/2. + m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + + // RHS = cvac^2*m_theta*dt*( curl(B^{n+theta}) - mu0*J^{n+1/2} ) + m_WarpX->ImplicitComputeRHSE(m_theta*a_dt, a_RHS); +} + +void ThetaImplicitEM::UpdateWarpXFields ( const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt ) +{ + amrex::ignore_unused(a_time); + + // Update Efield_fp owned by WarpX + m_WarpX->SetElectricFieldAndApplyBCs( a_E ); + + // Update Bfield_fp owned by WarpX + m_WarpX->UpdateMagneticFieldAndApplyBCs( m_Bold, m_theta*a_dt ); + +} + +void ThetaImplicitEM::FinishFieldUpdate ( amrex::Real a_new_time ) +{ + amrex::ignore_unused(a_new_time); + + // Eg^{n+1} = (1/theta)*E_g^{n+theta} + (1-1/theta)*E_g^n + // Bg^{n+1} = (1/theta)*B_g^{n+theta} + (1-1/theta)*B_g^n + + const amrex::Real c0 = 1._rt/m_theta; + const amrex::Real c1 = 1._rt - c0; + m_E.linComb( c0, m_E, c1, m_Eold ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + m_WarpX->FinishMagneticFieldAndApplyBCs( m_Bold, m_theta ); + +} diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp new file mode 100644 index 00000000000..7b95abade3e --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -0,0 +1,352 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "WarpX.H" + +#include "BoundaryConditions/PML.H" +#include "Diagnostics/MultiDiagnostics.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" +#include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" +#ifdef WARPX_USE_PSATD +# ifdef WARPX_DIM_RZ +# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" +# else +# include "FieldSolver/SpectralSolver/SpectralSolver.H" +# endif +#endif +#include "Parallelization/GuardCellManager.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleBoundaryBuffer.H" +#include "Python/callbacks.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" +#include "Utils/WarpXUtil.H" +#include "Utils/WarpXConst.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +void +WarpX::ImplicitPreRHSOp ( amrex::Real a_cur_time, + amrex::Real a_full_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + using namespace amrex::literals; + amrex::ignore_unused( a_full_dt, a_nl_iter, a_from_jacobian ); + + // Advance the particle positions by 1/2 dt, + // particle velocities by dt, then take average of old and new v, + // deposit currents, giving J at n+1/2 + // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration. + const bool skip_current = false; + const PushType push_type = PushType::Implicit; + PushParticlesandDeposit(a_cur_time, skip_current, push_type); + + SyncCurrentAndRho(); + +} + +void +WarpX::SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ) +{ + const amrex::Vector, 3 > >& Evec = a_E.getVec(); + amrex::MultiFab::Copy(*Efield_fp[0][0], *Evec[0][0], 0, 0, ncomps, Evec[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][1], *Evec[0][1], 0, 0, ncomps, Evec[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][2], *Evec[0][2], 0, 0, ncomps, Evec[0][2]->nGrowVect()); + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyEfieldBoundary(0, PatchType::fine); +} + +void +WarpX::UpdateMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_thetadt ) +{ + amrex::MultiFab::Copy(*Bfield_fp[0][0], *a_Bn[0][0], 0, 0, ncomps, a_Bn[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][1], *a_Bn[0][1], 0, 0, ncomps, a_Bn[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][2], *a_Bn[0][2], 0, 0, ncomps, a_Bn[0][2]->nGrowVect()); + EvolveB(a_thetadt, DtType::Full); + ApplyMagneticFieldBCs(); +} + +void +WarpX::FinishMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_theta ) +{ + FinishImplicitField(Bfield_fp, a_Bn, a_theta); + ApplyMagneticFieldBCs(); +} + +void +WarpX::ApplyMagneticFieldBCs() +{ + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); +} + +void +WarpX::SaveParticlesAtImplicitStepStart ( ) +{ + // The implicit advance routines require the particle velocity + // and position values at the beginning of the step to compute the + // time-centered position and velocity needed for the implicit stencil. + // Thus, we need to save this information. + + for (auto const& pc : *mypc) { + + for (int lev = 0; lev <= finest_level; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc->getParticleComps(); + + for (WarpXParIter pti(*pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + x_n[ip] = xp; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + y_n[ip] = yp; +#endif + z_n[ip] = zp; + + ux_n[ip] = ux[ip]; + uy_n[ip] = uy[ip]; + uz_n[ip] = uz[ip]; + + }); + + } + } + + } + + } + +} + +void +WarpX::FinishImplicitParticleUpdate () +{ + using namespace amrex::literals; + + // The implicit advance routines use the time-centered position and + // momentum to advance the system in time. Thus, at the end of the + // step we need to transform the particle postion and momentum from + // time n+1/2 to time n+1. This is done here. + + for (auto const& pc : *mypc) { + + for (int lev = 0; lev <= finest_level; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc->getParticleComps(); + + for (WarpXParIter pti(*pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + const auto setPosition = SetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + xp = 2._rt*xp - x_n[ip]; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + yp = 2._rt*yp - y_n[ip]; +#endif + zp = 2._rt*zp - z_n[ip]; + + ux[ip] = 2._rt*ux[ip] - ux_n[ip]; + uy[ip] = 2._rt*uy[ip] - uy_n[ip]; + uz[ip] = 2._rt*uz[ip] - uz_n[ip]; + + setPosition(ip, xp, yp, zp); + }); + + } + } + + } + + } + +} + +void +WarpX::FinishImplicitField( amrex::Vector, 3 > >& Field_fp, + const amrex::Vector, 3 > >& Field_n, + amrex::Real theta ) +{ + using namespace amrex::literals; + + // The implicit field advance routines use the fields at time n+theta + // with 0.5 <= theta <= 1.0. Thus, at the end of the step we need to + // transform the fields from time n+theta to time n+1. This is done here. + + for (int lev = 0; lev <= finest_level; ++lev) { + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*Field_fp[lev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + + amrex::Array4 const& Fx = Field_fp[lev][0]->array(mfi); + amrex::Array4 const& Fy = Field_fp[lev][1]->array(mfi); + amrex::Array4 const& Fz = Field_fp[lev][2]->array(mfi); + + amrex::Array4 const& Fx_n = Field_n[lev][0]->array(mfi); + amrex::Array4 const& Fy_n = Field_n[lev][1]->array(mfi); + amrex::Array4 const& Fz_n = Field_n[lev][2]->array(mfi); + + amrex::Box const& tbx = mfi.tilebox(Field_n[lev][0]->ixType().toIntVect()); + amrex::Box const& tby = mfi.tilebox(Field_n[lev][1]->ixType().toIntVect()); + amrex::Box const& tbz = mfi.tilebox(Field_n[lev][2]->ixType().toIntVect()); + + const amrex::Real c0 = 1._rt/theta; + const amrex::Real c1 = 1._rt - c0; + + amrex::ParallelFor( + tbx, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fx(i,j,k,n) = c0*Fx(i,j,k,n) + c1*Fx_n(i,j,k,n); + }, + tby, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fy(i,j,k,n) = c0*Fy(i,j,k,n) + c1*Fy_n(i,j,k,n); + }, + tbz, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fz(i,j,k,n) = c0*Fz(i,j,k,n) + c1*Fz_n(i,j,k,n); + }); + } + } +} + +void +WarpX::ImplicitComputeRHSE (amrex::Real a_dt, WarpXSolverVec& a_Erhs_vec) +{ + for (int lev = 0; lev <= finest_level; ++lev) + { + ImplicitComputeRHSE(lev, a_dt, a_Erhs_vec); + } +} + +void +WarpX::ImplicitComputeRHSE (int lev, amrex::Real a_dt, WarpXSolverVec& a_Erhs_vec) +{ + WARPX_PROFILE("WarpX::ImplicitComputeRHSE()"); + ImplicitComputeRHSE(lev, PatchType::fine, a_dt, a_Erhs_vec); + if (lev > 0) + { + ImplicitComputeRHSE(lev, PatchType::coarse, a_dt, a_Erhs_vec); + } +} + +void +WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, WarpXSolverVec& a_Erhs_vec) +{ + // set RHS to zero value + a_Erhs_vec.getVec()[lev][0]->setVal(0.0); + a_Erhs_vec.getVec()[lev][1]->setVal(0.0); + a_Erhs_vec.getVec()[lev][2]->setVal(0.0); + + // Compute Efield_rhs in regular cells by calling EvolveE. Because + // a_Erhs_vec is set to zero above, calling EvolveE below results in + // a_Erhs_vec storing only the RHS of the update equation. I.e., + // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) + if (patch_type == PatchType::fine) { + m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getVec()[lev], Bfield_fp[lev], + current_fp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], + F_fp[lev], lev, a_dt ); + } else { + m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getVec()[lev], Bfield_cp[lev], + current_cp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], + F_cp[lev], lev, a_dt ); + } + + // Compute Efield_rhs in PML cells by calling EvolveEPML + if (do_pml && pml[lev]->ok()) { + amrex::Abort("PML not yet implemented with implicit solvers."); + } + +} diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H new file mode 100644 index 00000000000..89a0b82b700 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -0,0 +1,233 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WarpXSolverVec_H_ +#define WarpXSolverVec_H_ + +#include "Utils/TextMsg.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/** + * \brief This is a wrapper class around a Vector of array of pointers to MultiFabs that + * contains basic math operators and functionality needed to interact with nonlinear + * solvers in WarpX and linear solvers in AMReX, such as GMRES. + * + * The size of the Vector is the number of amr levels. Hardcoded for 1 right now. The + * size of the array is the number of MultiFabs. It is hardcoded for 3 right now as it + * is only used for the electric field in the implicit electromagnetic time solvers. In + * the future, the array size can be made a template parameter so that this class can + * be used for other solver vectors, such as electrostatic (array size 1) or Darwin (array size 4). + */ + +class WarpXSolverVec +{ +public: + + WarpXSolverVec() = default; + + WarpXSolverVec(const WarpXSolverVec&) = delete; + + ~WarpXSolverVec() = default; + + using value_type = amrex::Real; + using RT = value_type; + + [[nodiscard]] inline bool IsDefined () const { return m_is_defined; } + + inline + void Define (const WarpXSolverVec& a_vec) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_vec.IsDefined(), + "WarpXSolverVec::Define(a_vec) called with undefined a_vec"); + Define( a_vec.getVec() ); + } + + inline + void Define ( const amrex::Vector, 3 > >& a_solver_vec ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !IsDefined(), + "WarpXSolverVec::Define() called on undefined WarpXSolverVec"); + m_field_vec.resize(m_num_amr_levels); + const int lev = 0; + for (int n=0; n<3; n++) { + const amrex::MultiFab& mf_model = *a_solver_vec[lev][n]; + m_field_vec[lev][n] = std::make_unique( mf_model.boxArray(), mf_model.DistributionMap(), + mf_model.nComp(), amrex::IntVect::TheZeroVector() ); + } + m_is_defined = true; + } + + void SetDotMask( const amrex::Vector& a_Geom ); + [[nodiscard]] RT dotProduct( const WarpXSolverVec& a_X ) const; + + inline + void Copy ( const amrex::Vector, 3 > >& a_solver_vec ) + { + AMREX_ASSERT_WITH_MESSAGE( + IsDefined(), + "WarpXSolverVec::Copy() called on undefined WarpXSolverVec"); + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n = 0; n < 3; ++n) { + amrex::MultiFab::Copy(*m_field_vec[lev][n], *a_solver_vec[lev][n], 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); + } + } + } + + inline + void Copy ( const WarpXSolverVec& a_vec ) + { + AMREX_ASSERT_WITH_MESSAGE( + a_vec.IsDefined(), + "WarpXSolverVec::Copy(a_vec) called with undefined a_vec"); + if (!IsDefined()) { Define(a_vec); } + const amrex::Vector, 3 > >& field_vec = a_vec.getVec(); + Copy(field_vec); + } + + // Prohibit Copy assignment operator + WarpXSolverVec& operator= ( const WarpXSolverVec& a_vec ) = delete; + + // Move assignment operator + WarpXSolverVec(WarpXSolverVec&&) noexcept = default; + WarpXSolverVec& operator= ( WarpXSolverVec&& a_vec ) noexcept + { + if (this != &a_vec) { + m_field_vec = std::move(a_vec.m_field_vec); + m_is_defined = true; + } + return *this; + } + + inline + void operator+= ( const WarpXSolverVec& a_vec ) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->plus(*(a_vec.getVec()[lev][n]), 0, 1, 0); + } + } + } + + inline + void operator-= (const WarpXSolverVec& a_vec) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->minus(*(a_vec.getVec()[lev][n]), 0, 1, 0); + } + } + } + + /** + * \brief Y = a*X + b*Y + */ + inline + void linComb (const RT a, const WarpXSolverVec& X, const RT b, const WarpXSolverVec& Y) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + amrex::MultiFab::LinComb(*m_field_vec[lev][n], a, *X.getVec()[lev][n], 0, + b, *Y.getVec()[lev][n], 0, + 0, 1, 0); + } + } + } + + /** + * \brief Increment Y by a*X (Y += a*X) + */ + void increment (const WarpXSolverVec& X, const RT a) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + amrex::MultiFab::Saxpy( *m_field_vec[lev][n], a, *X.getVec()[lev][n], + 0, 0, 1, amrex::IntVect::TheZeroVector() ); + } + } + } + + /** + * \brief Scale Y by a (Y *= a) + */ + inline + void scale (RT a_a) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->mult(a_a, 0, 1); + } + } + } + + inline + void zero () { setVal(0.0); } + + inline + void setVal ( const RT a_val ) + { + AMREX_ASSERT_WITH_MESSAGE( + IsDefined(), + "WarpXSolverVec::ones() called on undefined WarpXSolverVec"); + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->setVal(a_val); + } + } + } + + [[nodiscard]] inline RT norm2 () const + { + auto const norm = dotProduct(*this); + return std::sqrt(norm); + } + + [[nodiscard]] const amrex::Vector, 3 > >& getVec() const {return m_field_vec;} + amrex::Vector, 3 > >& getVec() {return m_field_vec;} + + // clearDotMask() must be called by the highest class that owns WarpXSolverVec() + // after it is done being used ( typically in the destructor ) to avoid the + // following error message after the simulation finishes: + // malloc_consolidate(): unaligned fastbin chunk detected + static void clearDotMask() { m_dotMask.clear(); } + +private: + + bool m_is_defined = false; + amrex::Vector, 3 > > m_field_vec; + + static constexpr int m_ncomp = 1; + static constexpr int m_num_amr_levels = 1; + + inline static bool m_dot_mask_defined = false; + inline static amrex::Vector,3>> m_dotMask; + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp new file mode 100644 index 00000000000..b181f038fb5 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -0,0 +1,51 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +void WarpXSolverVec::SetDotMask( const amrex::Vector& a_Geom ) +{ + if (m_dot_mask_defined) { return; } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + IsDefined(), + "WarpXSolverVec::SetDotMask() called from undefined instance "); + + m_dotMask.resize(m_num_amr_levels); + for ( int n = 0; n < 3; n++) { + const amrex::BoxArray& grids = m_field_vec[0][n]->boxArray(); + const amrex::MultiFab tmp( grids, m_field_vec[0][n]->DistributionMap(), + 1, 0, amrex::MFInfo().SetAlloc(false) ); + const amrex::Periodicity& period = a_Geom[0].periodicity(); + m_dotMask[0][n] = tmp.OwnerMask(period); + } + m_dot_mask_defined = true; + + // If the function below is not called, then the following + // error message occurs after the simulation finishes: + // malloc_consolidate(): unaligned fastbin chunk detected + amrex::ExecOnFinalize(WarpXSolverVec::clearDotMask); +} + +[[nodiscard]] amrex::Real WarpXSolverVec::dotProduct ( const WarpXSolverVec& a_X ) const +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_dot_mask_defined, + "WarpXSolverVec::dotProduct called with m_dotMask not yet defined"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_X.IsDefined(), + "WarpXSolverVec::dotProduct(a_X) called with undefined a_X"); + amrex::Real result = 0.0; + const int lev = 0; + const bool local = true; + for (int n = 0; n < 3; ++n) { + auto rtmp = amrex::MultiFab::Dot( *m_dotMask[lev][n], + *m_field_vec[lev][n], 0, + *a_X.getVec()[lev][n], 0, 1, 0, local); + result += rtmp; + } + amrex::ParallelAllReduce::Sum(result, amrex::ParallelContext::CommunicatorSub()); + return result; +} diff --git a/Source/FieldSolver/Make.package b/Source/FieldSolver/Make.package index 3b88f4627e5..a8af4c2de97 100644 --- a/Source/FieldSolver/Make.package +++ b/Source/FieldSolver/Make.package @@ -7,5 +7,6 @@ ifeq ($(USE_FFT),TRUE) endif include $(WARPX_HOME)/Source/FieldSolver/FiniteDifferenceSolver/Make.package include $(WARPX_HOME)/Source/FieldSolver/MagnetostaticSolver/Make.package +include $(WARPX_HOME)/Source/FieldSolver/ImplicitSolvers/Make.package VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index bd40c5e0cd4..68b38153f8d 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -501,6 +501,9 @@ WarpX::InitData () CheckGuardCells(); PrintMainPICparameters(); + if (m_implicit_solver) { + m_implicit_solver->PrintParameters(); + } WriteUsedInputsFile(); if (restart_chkfile.empty()) @@ -568,10 +571,33 @@ WarpX::InitFromScratch () AmrCore::InitFromScratch(time); // This will call MakeNewLevelFromScratch + if (m_implicit_solver) { + + m_implicit_solver->Define(this); + m_implicit_solver->GetParticleSolverParams( max_particle_its_in_implicit_scheme, + particle_tol_in_implicit_scheme ); + + // Add space to save the positions and velocities at the start of the time steps + for (auto const& pc : *mypc) { +#if (AMREX_SPACEDIM >= 2) + pc->AddRealComp("x_n"); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + pc->AddRealComp("y_n"); +#endif + pc->AddRealComp("z_n"); + pc->AddRealComp("ux_n"); + pc->AddRealComp("uy_n"); + pc->AddRealComp("uz_n"); + } + + } + mypc->AllocData(); mypc->InitData(); InitPML(); + } void diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 29f047c14e6..519eed52820 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -76,6 +76,7 @@ include $(WARPX_HOME)/Source/BoundaryConditions/Make.package include $(WARPX_HOME)/Source/Diagnostics/Make.package include $(WARPX_HOME)/Source/EmbeddedBoundary/Make.package include $(WARPX_HOME)/Source/FieldSolver/Make.package +include $(WARPX_HOME)/Source/NonlinearSolvers/Make.package include $(WARPX_HOME)/Source/Filter/Make.package include $(WARPX_HOME)/Source/Fluids/Make.package include $(WARPX_HOME)/Source/Initialization/Make.package @@ -90,7 +91,7 @@ include $(AMREX_HOME)/Src/Base/Make.package include $(AMREX_HOME)/Src/Particle/Make.package include $(AMREX_HOME)/Src/Boundary/Make.package include $(AMREX_HOME)/Src/AmrCore/Make.package -include $(AMREX_HOME)/Src/LinearSolvers/MLMG/Make.package +include $(AMREX_HOME)/Src/LinearSolvers/Make.package ifeq ($(USE_SENSEI_INSITU),TRUE) include $(AMREX_HOME)/Src/Amr/Make.package diff --git a/Source/NonlinearSolvers/CMakeLists.txt b/Source/NonlinearSolvers/CMakeLists.txt new file mode 100644 index 00000000000..938cdaad547 --- /dev/null +++ b/Source/NonlinearSolvers/CMakeLists.txt @@ -0,0 +1,6 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + ) +endforeach() diff --git a/Source/NonlinearSolvers/JacobianFunctionMF.H b/Source/NonlinearSolvers/JacobianFunctionMF.H new file mode 100644 index 00000000000..823523df23c --- /dev/null +++ b/Source/NonlinearSolvers/JacobianFunctionMF.H @@ -0,0 +1,234 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef JacobianFunctionMF_H_ +#define JacobianFunctionMF_H_ + +/** + * \brief This is a linear function class for computing the action of a + * Jacobian on a vector using a matrix-free finite-difference method. + * This class has all of the required functions to be used as the + * linear operator template parameter in AMReX_GMRES. + */ + +template +class JacobianFunctionMF +{ + public: + + using RT = typename T::value_type; + + JacobianFunctionMF() = default; + ~JacobianFunctionMF() = default; + + // Default move and copy operations + JacobianFunctionMF(const JacobianFunctionMF&) = default; + JacobianFunctionMF& operator=(const JacobianFunctionMF&) = default; + JacobianFunctionMF(JacobianFunctionMF&&) noexcept = default; + JacobianFunctionMF& operator=(JacobianFunctionMF&&) noexcept = default; + + void apply ( T& a_dF, const T& a_dU ); + + inline + void precond ( T& a_U, const T& a_X ) + { + if (m_usePreCond) { a_U.zero(); } + else { a_U.Copy(a_X); } + } + + inline + void updatePreCondMat ( const T& a_X ) + { + amrex::ignore_unused(a_X); + } + + inline + void create ( T& a_Z, const T& a_U ) + { + a_Z.define(a_U); + } + + T makeVecLHS () const; + T makeVecRHS () const; + + inline + void assign( T& a_Z, const T& a_U ) { + a_Z.Copy(a_U); + } + + inline + void increment( T& a_Z, const T& a_U, RT a_scale ) + { + a_Z.increment(a_U,a_scale); + } + + inline + void scale ( T& a_U, RT a_scale ) + { + a_U.scale(a_scale); + } + + inline + void linComb ( T& a_U, RT a, const T& X, RT b, const T& Y ) + { + a_U.linComb( a, X, b, Y ); + } + + inline + void setToZero ( T& a_U ) + { + a_U.zero(); + } + + inline + void setVal ( T& a_U, RT a_val ) + { + a_U.setVal(a_val); + } + + inline + RT dotProduct( const T& a_X, const T& a_Y ) + { + return( a_X.dotProduct(a_Y) ); + } + + inline + RT norm2( const T& a_U ) + { + return ( a_U.norm2() ); + } + + [[nodiscard]] inline + bool isDefined() const { return m_is_defined; } + + inline + void setBaseSolution ( const T& a_U ) + { + m_Y0.Copy(a_U); + m_normY0 = norm2(m_Y0); + } + + inline + void setBaseRHS ( const T& a_R ) + { + m_R0.Copy(a_R); + } + + inline + void setJFNKEps ( RT a_eps ) + { + m_epsJFNK = a_eps; + } + + inline + void setIsLinear ( bool a_isLinear ) + { + m_is_linear = a_isLinear; + } + + inline + void curTime ( RT a_time ) + { + m_cur_time = a_time; + } + + inline + void curTimeStep ( RT a_dt ) + { + m_dt = a_dt; + } + + void define( const T&, Ops* ); + + private: + + bool m_is_defined = false; + bool m_is_linear = false; + bool m_usePreCond = false; + RT m_epsJFNK = RT(1.0e-6); + RT m_normY0; + RT m_cur_time, m_dt; + std::string m_pc_type; + + T m_Z, m_Y0, m_R0, m_R; + Ops* m_ops; + +}; + +template +void JacobianFunctionMF::define ( const T& a_U, + Ops* a_ops ) +{ + m_Z.Define(a_U); + m_Y0.Define(a_U); + m_R0.Define(a_U); + m_R.Define(a_U); + + m_ops = a_ops; + + m_is_defined = true; +} + +template +auto JacobianFunctionMF::makeVecRHS () const -> T +{ + T Vec; + Vec.Define(m_R); + return Vec; +} + +template +auto JacobianFunctionMF::makeVecLHS () const -> T +{ + T Vec; + Vec.Define(m_R); + return Vec; +} + +template +void JacobianFunctionMF::apply (T& a_dF, const T& a_dU) +{ + BL_PROFILE("JacobianFunctionMF::apply()"); + using namespace amrex::literals; + RT const normY = norm2(a_dU); // always 1 when called from GMRES + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + isDefined(), + "JacobianFunction::apply() called on undefined JacobianFunction"); + + if (normY < 1.0e-15) { a_dF.zero(); } + else { + + RT eps; + if (m_is_linear) { + eps = 1.0_rt; + } else { + /* eps = error_rel * sqrt(1 + ||Y0||) / ||dU|| + * M. Pernice and H. F. Walker, "NITSOL: A Newton Iterative Solver for + * Nonlinear Systems", SIAM J. Sci. Stat. Comput., 1998, vol 19, + * pp. 302--318. */ + if (m_normY0==0.0) { eps = m_epsJFNK * norm2(m_R0) / normY; } + else { + // m_epsJFNK * sqrt(1.0 + m_normY0) / normY + // above commonly used form not recommend for poorly scaled Y0 + eps = m_epsJFNK * m_normY0 / normY; + } + } + const RT eps_inv = 1.0_rt/eps; + + m_Z.linComb( 1.0, m_Y0, eps, a_dU ); // Z = Y0 + eps*dU + m_ops->ComputeRHS(m_R, m_Z, m_cur_time, m_dt, -1, true ); + + // F(Y) = Y - b - R(Y) ==> dF = dF/dY*dU = [1 - dR/dY]*dU + // = dU - (R(Z)-R(Y0))/eps + a_dF.linComb( 1.0, a_dU, eps_inv, m_R0 ); + a_dF.increment(m_R,-eps_inv); + + } + +} + +#endif diff --git a/Source/NonlinearSolvers/Make.package b/Source/NonlinearSolvers/Make.package new file mode 100644 index 00000000000..4f2dcfa9617 --- /dev/null +++ b/Source/NonlinearSolvers/Make.package @@ -0,0 +1 @@ +VPATH_LOCATIONS += $(WARPX_HOME)/Source/NonlinearSolvers diff --git a/Source/NonlinearSolvers/NewtonSolver.H b/Source/NonlinearSolvers/NewtonSolver.H new file mode 100644 index 00000000000..93ad432208a --- /dev/null +++ b/Source/NonlinearSolvers/NewtonSolver.H @@ -0,0 +1,345 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef NEWTON_SOLVER_H_ +#define NEWTON_SOLVER_H_ + +#include "NonlinearSolver.H" +#include "JacobianFunctionMF.H" + +#include +#include +#include "Utils/TextMsg.H" + +#include + +/** + * \brief Newton method to solve nonlinear equation of form: + * F(U) = U - b - R(U) = 0. U is the solution vector, b is a constant, + * and R(U) is some nonlinear function of U, which is computed in the + * ComputeRHS() Ops function. + */ + +template +class NewtonSolver : public NonlinearSolver +{ +public: + + NewtonSolver() = default; + + ~NewtonSolver() override = default; + + // Prohibit Move and Copy operations + NewtonSolver(const NewtonSolver&) = delete; + NewtonSolver& operator=(const NewtonSolver&) = delete; + NewtonSolver(NewtonSolver&&) noexcept = delete; + NewtonSolver& operator=(NewtonSolver&&) noexcept = delete; + + void Define ( const Vec& a_U, + Ops* a_ops ) override; + + void Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const override; + + void GetSolverParams ( amrex::Real& a_rtol, + amrex::Real& a_atol, + int& a_maxits ) override + { + a_rtol = m_rtol; + a_atol = m_atol; + a_maxits = m_maxits; + } + + inline void CurTime ( amrex::Real a_time ) const + { + m_cur_time = a_time; + m_linear_function->curTime( a_time ); + } + + inline void CurTimeStep ( amrex::Real a_dt ) const + { + m_dt = a_dt; + m_linear_function->curTimeStep( a_dt ); + } + + void PrintParams () const override + { + amrex::Print() << "Newton verbose: " << (this->m_verbose?"true":"false") << std::endl; + amrex::Print() << "Newton max iterations: " << m_maxits << std::endl; + amrex::Print() << "Newton relative tolerance: " << m_rtol << std::endl; + amrex::Print() << "Newton absolute tolerance: " << m_atol << std::endl; + amrex::Print() << "Newton require convergence: " << (m_require_convergence?"true":"false") << std::endl; + amrex::Print() << "GMRES verbose: " << m_gmres_verbose_int << std::endl; + amrex::Print() << "GMRES restart length: " << m_gmres_restart_length << std::endl; + amrex::Print() << "GMRES max iterations: " << m_gmres_maxits << std::endl; + amrex::Print() << "GMRES relative tolerance: " << m_gmres_rtol << std::endl; + amrex::Print() << "GMRES absolute tolerance: " << m_gmres_atol << std::endl; + } + +private: + + /** + * \brief Intermediate Vec containers used by the solver. + */ + mutable Vec m_dU, m_F, m_R; + + /** + * \brief Pointer to Ops class. + */ + Ops* m_ops = nullptr; + + /** + * \brief Flag to determine whether convergence is required. + */ + bool m_require_convergence = true; + + /** + * \brief Relative tolerance for the Newton solver. + */ + amrex::Real m_rtol = 1.0e-6; + + /** + * \brief Absolute tolerance for the Newton solver. + */ + amrex::Real m_atol = 0.; + + /** + * \brief Maximum iterations for the Newton solver. + */ + int m_maxits = 100; + + /** + * \brief Relative tolerance for GMRES. + */ + amrex::Real m_gmres_rtol = 1.0e-4; + + /** + * \brief Absolute tolerance for GMRES. + */ + amrex::Real m_gmres_atol = 0.; + + /** + * \brief Maximum iterations for GMRES. + */ + int m_gmres_maxits = 1000; + + /** + * \brief Verbosity flag for GMRES. + */ + int m_gmres_verbose_int = 2; + + /** + * \brief Restart iteration for GMRES. + */ + int m_gmres_restart_length = 30; + + mutable amrex::Real m_cur_time, m_dt; + mutable bool m_update_pc = false; + mutable bool m_update_pc_init = false; + + /** + * \brief The linear function used by GMRES to compute A*v. + * In the contect of JFNK, A = dF/dU (i.e., system Jacobian) + */ + std::unique_ptr> m_linear_function; + + /** + * \brief The linear solver (GMRES) object. + */ + std::unique_ptr>> m_linear_solver; + + void ParseParameters (); + + /** + * \brief Compute the nonlinear residual: F(U) = U - b - R(U). + */ + void EvalResidual ( Vec& a_F, + const Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt, + int a_iter ) const; + +}; + +template +void NewtonSolver::Define ( const Vec& a_U, + Ops* a_ops ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !this->m_is_defined, + "Newton nonlinear solver object is already defined!"); + + ParseParameters(); + + m_dU.Define(a_U); + m_F.Define(a_U); // residual function F(U) = U - b - R(U) = 0 + m_R.Define(a_U); // right hand side function R(U) + + m_ops = a_ops; + + m_linear_function = std::make_unique>(); + m_linear_function->define(m_F, m_ops); + + m_linear_solver = std::make_unique>>(); + m_linear_solver->define(*m_linear_function); + m_linear_solver->setVerbose( m_gmres_verbose_int ); + m_linear_solver->setRestartLength( m_gmres_restart_length ); + m_linear_solver->setMaxIters( m_gmres_maxits ); + + this->m_is_defined = true; + +} + +template +void NewtonSolver::ParseParameters () +{ + const amrex::ParmParse pp_newton("newton"); + pp_newton.query("verbose", this->m_verbose); + pp_newton.query("absolute_tolerance", m_atol); + pp_newton.query("relative_tolerance", m_rtol); + pp_newton.query("max_iterations", m_maxits); + pp_newton.query("require_convergence", m_require_convergence); + + const amrex::ParmParse pp_gmres("gmres"); + pp_gmres.query("verbose_int", m_gmres_verbose_int); + pp_gmres.query("restart_length", m_gmres_restart_length); + pp_gmres.query("absolute_tolerance", m_gmres_atol); + pp_gmres.query("relative_tolerance", m_gmres_rtol); + pp_gmres.query("max_iterations", m_gmres_maxits); +} + +template +void NewtonSolver::Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const +{ + BL_PROFILE("NewtonSolver::Solve()"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + this->m_is_defined, + "NewtonSolver::Solve() called on undefined object"); + using namespace amrex::literals; + + // + // Newton routine to solve nonlinear equation of form: + // F(U) = U - b - R(U) = 0 + // + + CurTime(a_time); + CurTimeStep(a_dt); + + amrex::Real norm_abs = 0.; + amrex::Real norm0 = 1._rt; + amrex::Real norm_rel = 0.; + + int iter; + for (iter = 0; iter < m_maxits;) { + + // Compute residual: F(U) = U - b - R(U) + EvalResidual(m_F, a_U, a_b, a_time, a_dt, iter); + + // Compute norm of the residual + norm_abs = m_F.norm2(); + if (iter == 0) { + if (norm_abs > 0.) { norm0 = norm_abs; } + else { norm0 = 1._rt; } + } + norm_rel = norm_abs/norm0; + + // Check for convergence criteria + if (this->m_verbose || iter == m_maxits) { + amrex::Print() << "Newton: iteration = " << std::setw(3) << iter << ", norm = " + << std::scientific << std::setprecision(5) << norm_abs << " (abs.), " + << std::scientific << std::setprecision(5) << norm_rel << " (rel.)" << "\n"; + } + + if (norm_abs < m_rtol) { + amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter + << ". Satisfied absolute tolerance " << m_atol << std::endl; + break; + } + + if (norm_rel < m_rtol) { + amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter + << ". Satisfied relative tolerance " << m_rtol << std::endl; + break; + } + + if (norm_abs > 100._rt*norm0) { + amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter + << ". SOLVER DIVERGED! relative tolerance = " << m_rtol << std::endl; + std::stringstream convergenceMsg; + convergenceMsg << "Newton: exiting at iteration " << std::setw(3) << iter << + ". SOLVER DIVERGED! absolute norm = " << norm_abs << + " has increased by 100X from that after first iteration."; + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } + + // Solve linear system for Newton step [Jac]*dU = F + m_dU.zero(); + m_linear_solver->solve( m_dU, m_F, m_gmres_rtol, m_gmres_atol ); + + // Update solution + a_U -= m_dU; + + iter++; + if (iter >= m_maxits) { + amrex::Print() << "Newton: exiting at iter = " << std::setw(3) << iter + << ". Maximum iteration reached: iter = " << m_maxits << std::endl; + break; + } + + } + + if (m_rtol > 0. && iter == m_maxits) { + std::stringstream convergenceMsg; + convergenceMsg << "Newton solver failed to converge after " << iter << + " iterations. Relative norm is " << norm_rel << + " and the relative tolerance is " << m_rtol << + ". Absolute norm is " << norm_abs << + " and the absolute tolerance is " << m_atol; + if (this->m_verbose) { amrex::Print() << convergenceMsg.str() << std::endl; } + if (m_require_convergence) { + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } else { + ablastr::warn_manager::WMRecordWarning("NewtonSolver", convergenceMsg.str()); + } + } + +} + +template +void NewtonSolver::EvalResidual ( Vec& a_F, + const Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt, + int a_iter ) const +{ + + m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, a_iter, false ); + + // set base U and R(U) for matrix-free Jacobian action calculation + m_linear_function->setBaseSolution(a_U); + m_linear_function->setBaseRHS(m_R); + + // update preconditioner + if (m_update_pc || m_update_pc_init) { + m_linear_function->updatePreCondMat(a_U); + } + m_update_pc_init = false; + + // Compute residual: F(U) = U - b - R(U) + a_F.Copy(a_U); + a_F -= m_R; + a_F -= a_b; + +} + +#endif diff --git a/Source/NonlinearSolvers/NonlinearSolver.H b/Source/NonlinearSolvers/NonlinearSolver.H new file mode 100644 index 00000000000..5587826474c --- /dev/null +++ b/Source/NonlinearSolvers/NonlinearSolver.H @@ -0,0 +1,87 @@ +#ifndef WARPX_NONLINEAR_SOLVER_H_ +#define WARPX_NONLINEAR_SOLVER_H_ + +#include +#include + +#include + +#include +#include +#include + +/** + * \brief Top-level class for the nonlinear solver + * + * This class is templated on a vector class Vec, and an operator class Ops. + * + * The Ops class must have the following function: + * ComputeRHS( R_vec, U_vec, time, dt, nl_iter, from_jacobian ), + * where U_vec and R_vec are of type Vec. + * + * The Vec class must have basic math operators, such as Copy, +=, -=, + * increment(), linComb(), scale(), etc.. See WarpXSolverVec.H for an example. + */ + +template +class NonlinearSolver +{ +public: + + NonlinearSolver() = default; + + virtual ~NonlinearSolver() = default; + + // Prohibit Move and Copy operations + NonlinearSolver(const NonlinearSolver&) = delete; + NonlinearSolver& operator=(const NonlinearSolver&) = delete; + NonlinearSolver(NonlinearSolver&&) noexcept = delete; + NonlinearSolver& operator=(NonlinearSolver&&) noexcept = delete; + + /** + * \brief Read user-provided parameters that control the nonlinear solver. + * Allocate intermediate data containers needed by the solver. For Newton, + * setup the linear solver for computing the Newton step. + */ + virtual void Define ( const Vec&, + Ops* ) = 0; + + /** + * \brief Check if the nonlinear solver has been defined. + */ + [[nodiscard]] bool IsDefined () const { return m_is_defined; } + + /** + * \brief Solve the specified nonlinear equation for U. + * Picard: U = b + R(U). + * Newton: F(U) = U - b - R(U) = 0. + */ + virtual void Solve ( Vec&, + const Vec&, + amrex::Real, + amrex::Real ) const = 0; + + /** + * \brief Print parameters used by the nonlinear solver. + */ + virtual void PrintParams () const = 0; + + /** + * \brief Return the convergence parameters used by the nonlinear solver. + */ + virtual void GetSolverParams (amrex::Real&, amrex::Real&, int&) = 0; + + /** + * \brief Allow caller to dynamically change the verbosity flag. For + * example, one may want to only print solver information every 100 steps. + */ + void Verbose ( bool a_verbose ) { m_verbose = a_verbose; } + +protected: + + bool m_is_defined = false; + mutable bool m_verbose = true; + +}; + +#endif diff --git a/Source/NonlinearSolvers/NonlinearSolverLibrary.H b/Source/NonlinearSolvers/NonlinearSolverLibrary.H new file mode 100644 index 00000000000..5803d6ca753 --- /dev/null +++ b/Source/NonlinearSolvers/NonlinearSolverLibrary.H @@ -0,0 +1,15 @@ +#ifndef NONLINEAR_SOLVER_LIBRARY_H_ +#define NONLINEAR_SOLVER_LIBRARY_H_ + +#include "PicardSolver.H" // IWYU pragma: export +#include "NewtonSolver.H" // IWYU pragma: export + +/** + * \brief struct to select the nonlinear solver for implicit schemes + */ +enum NonlinearSolverType { + Picard = 0, + Newton = 1 +}; + +#endif diff --git a/Source/NonlinearSolvers/PicardSolver.H b/Source/NonlinearSolvers/PicardSolver.H new file mode 100644 index 00000000000..f05b9a106e6 --- /dev/null +++ b/Source/NonlinearSolvers/PicardSolver.H @@ -0,0 +1,217 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef PICARD_SOLVER_H_ +#define PICARD_SOLVER_H_ + +#include "NonlinearSolver.H" + +#include +#include "Utils/TextMsg.H" + +#include + +/** + * \brief Picard fixed-point iteration method to solve nonlinear + * equation of form: U = b + R(U). U is the solution vector. b + * is a constant. R(U) is some nonlinear function of U, which + * is computed in the Ops function ComputeRHS(). + */ + +template +class PicardSolver : public NonlinearSolver +{ +public: + + PicardSolver() = default; + + ~PicardSolver() override = default; + + // Prohibit Move and Copy operations + PicardSolver(const PicardSolver&) = delete; + PicardSolver& operator=(const PicardSolver&) = delete; + PicardSolver(PicardSolver&&) noexcept = delete; + PicardSolver& operator=(PicardSolver&&) noexcept = delete; + + void Define ( const Vec& a_U, + Ops* a_ops ) override; + + void Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const override; + + void GetSolverParams ( amrex::Real& a_rtol, + amrex::Real& a_atol, + int& a_maxits ) override + { + a_rtol = m_rtol; + a_atol = m_atol; + a_maxits = m_maxits; + } + + void PrintParams () const override + { + amrex::Print() << "Picard max iterations: " << m_maxits << std::endl; + amrex::Print() << "Picard relative tolerance: " << m_rtol << std::endl; + amrex::Print() << "Picard absolute tolerance: " << m_atol << std::endl; + amrex::Print() << "Picard require convergence: " << (m_require_convergence?"true":"false") << std::endl; + } + +private: + + /** + * \brief Intermediate Vec containers used by the solver. + */ + mutable Vec m_Usave, m_R; + + /** + * \brief Pointer to Ops class. + */ + Ops* m_ops = nullptr; + + /** + * \brief Flag to determine whether convergence is required. + */ + bool m_require_convergence = true; + + /** + * \brief Relative tolerance for the Picard nonlinear solver + */ + amrex::Real m_rtol = 1.0e-6; + + /** + * \brief Absolute tolerance for the Picard nonlinear solver + */ + amrex::Real m_atol = 0.; + + /** + * \brief Maximum iterations for the Picard nonlinear solver + */ + int m_maxits = 100; + + void ParseParameters( ); + +}; + +template +void PicardSolver::Define ( const Vec& a_U, + Ops* a_ops ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !this->m_is_defined, + "Picard nonlinear solver object is already defined!"); + + ParseParameters(); + + m_Usave.Define(a_U); + m_R.Define(a_U); + + m_ops = a_ops; + + this->m_is_defined = true; + +} + +template +void PicardSolver::ParseParameters () +{ + const amrex::ParmParse pp_picard("picard"); + pp_picard.query("verbose", this->m_verbose); + pp_picard.query("absolute_tolerance", m_atol); + pp_picard.query("relative_tolerance", m_rtol); + pp_picard.query("max_iterations", m_maxits); + pp_picard.query("require_convergence", m_require_convergence); + +} + +template +void PicardSolver::Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const +{ + BL_PROFILE("PicardSolver::Solve()"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + this->m_is_defined, + "PicardSolver::Solve() called on undefined object"); + using namespace amrex::literals; + + // + // Picard fixed-point iteration method to solve nonlinear + // equation of form: U = b + R(U) + // + + amrex::Real norm_abs = 0.; + amrex::Real norm0 = 1._rt; + amrex::Real norm_rel = 0.; + + int iter; + for (iter = 0; iter < m_maxits;) { + + // Save previous state for norm calculation + m_Usave.Copy(a_U); + + // Update the solver state (a_U = a_b + m_R) + m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, iter, false ); + a_U.Copy(a_b); + a_U += m_R; + + // Compute the step norm and update iter + m_Usave -= a_U; + norm_abs = m_Usave.norm2(); + if (iter == 0) { + if (norm_abs > 0.) { norm0 = norm_abs; } + else { norm0 = 1._rt; } + } + norm_rel = norm_abs/norm0; + iter++; + + // Check for convergence criteria + if (this->m_verbose || iter == m_maxits) { + amrex::Print() << "Picard: iter = " << std::setw(3) << iter << ", norm = " + << std::scientific << std::setprecision(5) << norm_abs << " (abs.), " + << std::scientific << std::setprecision(5) << norm_rel << " (rel.)" << "\n"; + } + + if (norm_abs < m_atol) { + amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter + << ". Satisfied absolute tolerance " << m_atol << std::endl; + break; + } + + if (norm_rel < m_rtol) { + amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter + << ". Satisfied relative tolerance " << m_rtol << std::endl; + break; + } + + if (iter >= m_maxits) { + amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter + << ". Maximum iteration reached: iter = " << m_maxits << std::endl; + break; + } + + } + + if (m_rtol > 0. && iter == m_maxits) { + std::stringstream convergenceMsg; + convergenceMsg << "Picard solver failed to converge after " << iter << + " iterations. Relative norm is " << norm_rel << + " and the relative tolerance is " << m_rtol << + ". Absolute norm is " << norm_abs << + " and the absolute tolerance is " << m_atol; + if (this->m_verbose) { amrex::Print() << convergenceMsg.str() << std::endl; } + if (m_require_convergence) { + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } else { + ablastr::warn_manager::WMRecordWarning("PicardSolver", convergenceMsg.str()); + } + } + +} + +#endif diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index fac94ff20a3..197cb897602 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -94,7 +94,7 @@ public: amrex::ParticleReal * AMREX_RESTRICT puzp, amrex::ParticleReal const * AMREX_RESTRICT pwp, amrex::Real const * AMREX_RESTRICT amplitude, - amrex::Real dt); + amrex::Real dt, PushType push_type=PushType::Explicit); protected: diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 7b735053f0b..0693a13c1f9 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -653,7 +653,7 @@ LaserParticleContainer::Evolve (int lev, // Calculate the corresponding momentum and position for the particles update_laser_particle(pti, static_cast(np), uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), wp.dataPtr(), - amplitude_E.dataPtr(), dt); + amplitude_E.dataPtr(), dt, push_type ); WARPX_PROFILE_VAR_STOP(blp_pp); // Current Deposition @@ -851,7 +851,7 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, ParticleReal * AMREX_RESTRICT const puzp, ParticleReal const * AMREX_RESTRICT const pwp, Real const * AMREX_RESTRICT const amplitude, - const Real dt) + const Real dt, PushType push_type) { const auto GetPosition = GetParticlePosition(pti); auto SetPosition = SetParticlePosition(pti); @@ -863,6 +863,26 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, const Real tmp_nvec_1 = m_nvec[1]; const Real tmp_nvec_2 = m_nvec[2]; + // When using the implicit solver, this function is called multiple times per timestep + // (within the linear and nonlinear solver). Thus, the position of the particles needs to be reset + // to the initial position (at the beginning of the timestep), before updating the particle position +#if (AMREX_SPACEDIM >= 2) + ParticleReal* x_n = nullptr; + if (push_type == PushType::Implicit) { + x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); + } +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal* y_n = nullptr; + if (push_type == PushType::Implicit) { + y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); + } +#endif + ParticleReal* z_n = nullptr; + if (push_type == PushType::Implicit) { + z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + } + // Copy member variables to tmp copies for GPU runs. const Real tmp_mobility = m_mobility; const Real gamma_boost = WarpX::gamma_boost; @@ -894,15 +914,33 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, puzp[i] = gamma * vz; // Push the the particle positions - ParticleReal x, y, z; - GetPosition(i, x, y, z); + + // When using the implicit solver, this function is called multiple times per timestep + // (within the linear and nonlinear solver). Thus, the position of the particles needs to be reset + // to the initial position (at the beginning of the timestep), before updating the particle position + + ParticleReal x=0., y=0., z=0.; + if (push_type == PushType::Explicit) { + GetPosition(i, x, y, z); + } + #if !defined(WARPX_DIM_1D_Z) + if (push_type == PushType::Implicit) { + x = x_n[i]; + } x += vx * dt; #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + if (push_type == PushType::Implicit) { + y = y_n[i]; + } y += vy * dt; #endif + if (push_type == PushType::Implicit) { + z = z_n[i]; + } z += vz * dt; + SetPosition(i, x, y, z); } ); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 3d134393f06..e2be4f948ca 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -61,6 +61,7 @@ #include #include #include +#include #include #include #include @@ -3118,6 +3119,12 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const int qed_runtime_flag = no_qed; #endif + const int max_iterations = WarpX::max_particle_its_in_implicit_scheme; + const amrex::ParticleReal particle_tolerance = WarpX::particle_tol_in_implicit_scheme; + + amrex::Gpu::Buffer unconverged_particles({0}); + amrex::Long* unconverged_particles_ptr = unconverged_particles.data(); + // Using this version of ParallelFor with compile time options // improves performance when qed or external EB are not used by reducing // register pressure. @@ -3129,107 +3136,163 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, { // Position advance starts from the position at the start of the step // but uses the most recent velocity. + #if (AMREX_SPACEDIM >= 2) amrex::ParticleReal xp = x_n[ip]; const amrex::ParticleReal xp_n = x_n[ip]; #else - amrex::ParticleReal xp = 0._rt; + const amrex::ParticleReal xp = 0._rt; const amrex::ParticleReal xp_n = 0._rt; #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) amrex::ParticleReal yp = y_n[ip]; const amrex::ParticleReal yp_n = y_n[ip]; #else - amrex::ParticleReal yp = 0._rt; + const amrex::ParticleReal yp = 0._rt; const amrex::ParticleReal yp_n = 0._rt; #endif amrex::ParticleReal zp = z_n[ip]; const amrex::ParticleReal zp_n = z_n[ip]; - UpdatePositionImplicit(xp, yp, zp, ux_n[ip], uy_n[ip], uz_n[ip], ux[ip], uy[ip], uz[ip], 0.5_rt*dt); - setPosition(ip, xp, yp, zp); + amrex::ParticleReal dxp, dxp_save; + amrex::ParticleReal dyp, dyp_save; + amrex::ParticleReal dzp, dzp_save; + auto idxg2 = static_cast(1._rt/(dx[0]*dx[0])); + auto idyg2 = static_cast(1._rt/(dx[1]*dx[1])); + auto idzg2 = static_cast(1._rt/(dx[2]*dx[2])); - amrex::ParticleReal Exp = Ex_external_particle; - amrex::ParticleReal Eyp = Ey_external_particle; - amrex::ParticleReal Ezp = Ez_external_particle; - amrex::ParticleReal Bxp = Bx_external_particle; - amrex::ParticleReal Byp = By_external_particle; - amrex::ParticleReal Bzp = Bz_external_particle; + amrex::ParticleReal step_norm = 1._prt; + for (int iter=0; iter(ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 1, - m, q, pusher_algo, do_crr, -#ifdef WARPX_QED - t_chi_max, + if (!do_sync) #endif - dt); - } -#ifdef WARPX_QED - else { - if constexpr (qed_control == has_qed) { - doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + { + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, +#ifdef WARPX_QED t_chi_max, +#endif dt); } - } +#ifdef WARPX_QED + else { + if constexpr (qed_control == has_qed) { + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 1, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); + } + } #endif #ifdef WARPX_QED - [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; - [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; - [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc - if constexpr (qed_control == has_qed) { - if (local_has_quantum_sync) { - evolve_opt(ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp,Bxp, Byp, Bzp, - dt, p_optical_depth_QSR[ip]); + [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + if constexpr (qed_control == has_qed) { + if (local_has_quantum_sync) { + evolve_opt(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp,Bxp, Byp, Bzp, + dt, p_optical_depth_QSR[ip]); + } } - } #else amrex::ignore_unused(qed_control); #endif - // Take average to get the time centered value - ux[ip] = 0.5_rt*(ux[ip] + ux_n[ip]); - uy[ip] = 0.5_rt*(uy[ip] + uy_n[ip]); - uz[ip] = 0.5_rt*(uz[ip] + uz_n[ip]); + // Take average to get the time centered value + ux[ip] = 0.5_rt*(ux[ip] + ux_n[ip]); + uy[ip] = 0.5_rt*(uy[ip] + uy_n[ip]); + uz[ip] = 0.5_rt*(uz[ip] + uz_n[ip]); + + iter++; + + // particle did not converge + if ( iter > 1 && iter == max_iterations ) { +#if !defined(AMREX_USE_GPU) + std::stringstream convergenceMsg; + convergenceMsg << "Picard solver for particle failed to converge after " << + iter << " iterations. " << std::endl; + convergenceMsg << "Position step norm is " << step_norm << + " and the tolerance is " << particle_tolerance << std::endl; + convergenceMsg << " ux = " << ux[ip] << ", uy = " << uy[ip] << ", uz = " << uz[ip] << std::endl; + convergenceMsg << " xp = " << xp << ", yp = " << yp << ", zp = " << zp; + ablastr::warn_manager::WMRecordWarning("ImplicitPushXP", convergenceMsg.str()); +#endif + + // write signaling flag: how many particles did not converge? + amrex::Gpu::Atomic::Add(unconverged_particles_ptr, amrex::Long(1)); + } + + } // end Picard iterations }); + + auto const num_unconverged_particles = *(unconverged_particles.copyToHost()); + if (num_unconverged_particles > 0) { + ablastr::warn_manager::WMRecordWarning("ImplicitPushXP", + "Picard solver for " + + std::to_string(num_unconverged_particles) + + " particles failed to converge after " + + std::to_string(max_iterations) + " iterations." + ); + } } void diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H index 6e2e82632ee..89c2de88e47 100644 --- a/Source/Particles/Pusher/UpdatePosition.H +++ b/Source/Particles/Pusher/UpdatePosition.H @@ -85,4 +85,46 @@ void UpdatePositionImplicit(amrex::ParticleReal& x, amrex::ParticleReal& y, amre z += uz * inv_gamma * dt; } +/** \brief Check particle position for convergence. This is used by the theta-implicit + * and semi-implicit time solvers to obtain a self-consistent time-centered update + * of the particles for given electric and magnetic fields on the grid. + */ +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void PositionNorm( amrex::ParticleReal dxp, amrex::ParticleReal dyp, amrex::ParticleReal dzp, + amrex::ParticleReal& dxp_save, amrex::ParticleReal& dyp_save, amrex::ParticleReal& dzp_save, + amrex::ParticleReal idxg2, amrex::ParticleReal idyg2, amrex::ParticleReal idzg2, + amrex::ParticleReal& step_norm, const int iter ) +{ + using namespace amrex::literals; + +#if defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(dxp, dxp_save, idxg2); +#endif +#if !defined(WARPX_DIM_3D) + amrex::ignore_unused(dyp, dyp_save, idyg2); +#endif + + if (iter==0) { step_norm = 1.0_prt; } + else { + step_norm = (dzp - dzp_save)*(dzp - dzp_save)*idzg2; +#if !defined(WARPX_DIM_1D_Z) + step_norm += (dxp - dxp_save)*(dxp - dxp_save)*idxg2; +#endif +#if defined(WARPX_DIM_3D) + step_norm += (dyp - dyp_save)*(dyp - dyp_save)*idyg2; +#elif defined(WARPX_DIM_RZ) + step_norm += (dyp - dyp_save)*(dyp - dyp_save)*idxg2; +#endif + step_norm = std::sqrt(step_norm); + } + dzp_save = dzp; +#if !defined(WARPX_DIM_1D_Z) + dxp_save = dxp; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + dyp_save = dyp; +#endif + +} + #endif // WARPX_PARTICLES_PUSHER_UPDATEPOSITION_H_ diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index e067ed03a80..527b7766895 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -29,8 +29,8 @@ struct MediumForEM { struct EvolveScheme { enum { Explicit = 0, - ImplicitPicard = 1, - SemiImplicitPicard = 2 + ThetaImplicitEM = 1, + SemiImplicitEM = 2 }; }; diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 2449ea2f8dd..89784c3b86c 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -24,10 +24,10 @@ // and corresponding integer for use inside the code const std::map evolve_scheme_to_int = { - {"explicit", EvolveScheme::Explicit }, - {"implicit_picard", EvolveScheme::ImplicitPicard }, - {"semi_implicit_picard", EvolveScheme::SemiImplicitPicard }, - {"default", EvolveScheme::Explicit } + {"explicit", EvolveScheme::Explicit }, + {"theta_implicit_em", EvolveScheme::ThetaImplicitEM }, + {"semi_implicit_em", EvolveScheme::SemiImplicitEM }, + {"default", EvolveScheme::Explicit } }; const std::map grid_to_int = { diff --git a/Source/WarpX.H b/Source/WarpX.H index df06a1723ca..a0a1379d9e2 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -41,6 +41,8 @@ #include "FieldSolver/Fields.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" +#include "FieldSolver/ImplicitSolvers/ImplicitSolver.H" +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" #include "Filter/BilinearFilter.H" #include "Parallelization/GuardCellManager.H" #include "Utils/Parser/IntervalsParser.H" @@ -113,11 +115,27 @@ public: void Evolve (int numsteps = -1); - void EvolveImplicitPicardInit (int lev); - void SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev); - void FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev); - void FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Efield_fp, - amrex::Vector, 3 > >& Efield_n); + // + // Functions used by implicit solvers + // + void ImplicitPreRHSOp ( amrex::Real cur_time, + amrex::Real a_full_dt, + int a_nl_iter, + bool a_from_jacobian ); + void SaveParticlesAtImplicitStepStart (); + void FinishImplicitParticleUpdate (); + void SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ); + void UpdateMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_thetadt ); + void ApplyMagneticFieldBCs (); + void FinishMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_theta ); + void FinishImplicitField ( amrex::Vector, 3 > >& Field_fp, + const amrex::Vector, 3 > >& Field_n, + amrex::Real theta ); + void ImplicitComputeRHSE ( amrex::Real dt, WarpXSolverVec& a_Erhs_vec); + void ImplicitComputeRHSE (int lev, amrex::Real dt, WarpXSolverVec& a_Erhs_vec); + void ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real dt, WarpXSolverVec& a_Erhs_vec); MultiParticleContainer& GetPartContainer () { return *mypc; } MultiFluidContainer& GetFluidContainer () { return *myfl; } @@ -159,14 +177,12 @@ public: static short particle_pusher_algo; //! Integer that corresponds to the type of Maxwell solver (Yee, CKC, PSATD, ECT) static short electromagnetic_solver_id; - //! Integer that corresponds to the evolve scheme (explicit, implicit_picard, semi_implicit_picard) + //! Integer that corresponds to the evolve scheme (explicit, semi_implicit_em, theta_implicit_em) static short evolve_scheme; - //! The maximum number of Picard iterations to do each time step - static int max_picard_iterations; - //! The tolerance for the Picard iteration convergence - static amrex::Real picard_iteration_tolerance; - //! Flags whether the Picard iterations are required to converge - static bool require_picard_convergence; + //! Maximum iterations used for self-consistent particle update in implicit particle-suppressed evolve schemes + static int max_particle_its_in_implicit_scheme; + //! Relative tolerance used for self-consistent particle update in implicit particle-suppressed evolve schemes + static amrex::ParticleReal particle_tol_in_implicit_scheme; /** Records a number corresponding to the load balance cost update strategy * being used (0 or 1 corresponding to timers or heuristic). */ @@ -506,6 +522,17 @@ public: [[nodiscard]] const amrex::MultiFab& getField(warpx::fields::FieldType field_type, int lev, int direction = 0) const; + /** + * \brief + * Get a constant reference to the specified vector field on the different MR levels + * + * \param field_type[in] the field type + * + * \return a vector (which one element per MR level) of arrays of three pointers (for 3 vector components) amrex::MultiFab* containing the field data + */ + [[nodiscard]] const amrex::Vector,3>>& + getMultiLevelField(warpx::fields::FieldType field_type) const; + [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} @@ -1268,8 +1295,6 @@ private: void OneStep_nosub (amrex::Real cur_time); void OneStep_sub1 (amrex::Real cur_time); - void OneStep_ImplicitPicard(amrex::Real cur_time); - /** * \brief Perform one PIC iteration, with the multiple J deposition per time step */ @@ -1477,12 +1502,6 @@ private: amrex::Vector, 3 > > Efield_avg_fp; amrex::Vector, 3 > > Bfield_avg_fp; - // Implicit, fields at start of step and from the previous iteration - amrex::Vector, 3 > > Efield_n; - amrex::Vector, 3 > > Bfield_n; - amrex::Vector, 3 > > Efield_save; - amrex::Vector, 3 > > Bfield_save; - // Memory buffers for computing magnetostatic fields // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order amrex::Vector, 3 > > vector_potential_fp_nodal; @@ -1936,6 +1955,10 @@ private: amrex::Vector> m_fdtd_solver_fp; amrex::Vector> m_fdtd_solver_cp; + + // implicit solver object + std::unique_ptr m_implicit_solver; + }; #endif diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 512ed7fe9d0..d3f91002ef4 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -41,6 +41,8 @@ #include "Utils/WarpXProfilerWrapper.H" #include "Utils/WarpXUtil.H" +#include "FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H" + #include #include @@ -113,9 +115,8 @@ short WarpX::field_gathering_algo; short WarpX::particle_pusher_algo; short WarpX::electromagnetic_solver_id; short WarpX::evolve_scheme; -int WarpX::max_picard_iterations = 10; -Real WarpX::picard_iteration_tolerance = 1.e-7; -bool WarpX::require_picard_convergence = true; +int WarpX::max_particle_its_in_implicit_scheme = 21; +ParticleReal WarpX::particle_tol_in_implicit_scheme = 1.e-10; short WarpX::psatd_solution_type; short WarpX::J_in_time; short WarpX::rho_in_time; @@ -1195,6 +1196,21 @@ WarpX::ReadParameters () particle_pusher_algo = static_cast(GetAlgorithmInteger(pp_algo, "particle_pusher")); evolve_scheme = static_cast(GetAlgorithmInteger(pp_algo, "evolve_scheme")); + // check for implicit evolve scheme + if (evolve_scheme == EvolveScheme::SemiImplicitEM) { + m_implicit_solver = std::make_unique(); + } + else if (evolve_scheme == EvolveScheme::ThetaImplicitEM) { + m_implicit_solver = std::make_unique(); + } + + // implicit evolve schemes not setup to use mirrors + if (evolve_scheme == EvolveScheme::SemiImplicitEM || + evolve_scheme == EvolveScheme::ThetaImplicitEM) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( num_mirrors == 0, + "Mirrors cannot be used with Implicit evolve schemes."); + } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo != CurrentDepositionAlgo::Esirkepov || !do_current_centering, @@ -1231,8 +1247,8 @@ WarpX::ReadParameters () if (current_deposition_algo == CurrentDepositionAlgo::Villasenor) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard, + evolve_scheme == EvolveScheme::SemiImplicitEM || + evolve_scheme == EvolveScheme::ThetaImplicitEM, "Villasenor current deposition can only" "be used with Implicit evolve schemes."); } @@ -1293,11 +1309,8 @@ WarpX::ReadParameters () macroscopic_solver_algo = GetAlgorithmInteger(pp_algo,"macroscopic_sigma_method"); } - if (evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard) { - utils::parser::queryWithParser(pp_algo, "max_picard_iterations", max_picard_iterations); - utils::parser::queryWithParser(pp_algo, "picard_iteration_tolerance", picard_iteration_tolerance); - utils::parser::queryWithParser(pp_algo, "require_picard_convergence", require_picard_convergence); + if (evolve_scheme == EvolveScheme::SemiImplicitEM || + evolve_scheme == EvolveScheme::ThetaImplicitEM) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo == CurrentDepositionAlgo::Esirkepov || @@ -2177,11 +2190,6 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d AllocLevelMFs(lev, ba, dm, guard_cells.ng_alloc_EB, guard_cells.ng_alloc_J, guard_cells.ng_alloc_Rho, guard_cells.ng_alloc_F, guard_cells.ng_alloc_G, aux_is_nodal); - if (evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard) { - EvolveImplicitPicardInit(lev); - } - m_accelerator_lattice[lev] = std::make_unique(); m_accelerator_lattice[lev]->InitElementFinder(lev, ba, dm); @@ -3513,3 +3521,32 @@ WarpX::getField(FieldType field_type, const int lev, const int direction) const { return *getFieldPointer(field_type, lev, direction); } + +const amrex::Vector,3>>& +WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const +{ + switch(field_type) + { + case FieldType::Efield_aux : + return Efield_aux; + case FieldType::Bfield_aux : + return Bfield_aux; + case FieldType::Efield_fp : + return Efield_fp; + case FieldType::Bfield_fp : + return Bfield_fp; + case FieldType::current_fp : + return current_fp; + case FieldType::current_fp_nodal : + return current_fp_nodal; + case FieldType::Efield_cp : + return Efield_cp; + case FieldType::Bfield_cp : + return Bfield_cp; + case FieldType::current_cp : + return current_cp; + default: + WARPX_ABORT_WITH_MESSAGE("Invalid field type"); + return Efield_fp; + } +} From fd8b4b885c62e0c1f5e794907a1a124579179b4e Mon Sep 17 00:00:00 2001 From: Avigdor Veksler <124003120+aveksler1@users.noreply.github.com> Date: Tue, 4 Jun 2024 20:34:08 -0700 Subject: [PATCH 115/190] Add upper bound for weight of product particles from particle resampling (#4969) * add upper bound for weight parameter on particle merging * increase readability, fix clang-tidy test * rework from ignoring particles above some maximum weight to instead resample particles such that they stay under a target weight * avoid implicit capture of this, cleanup * remove leftover code Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> * clarify docstring * clarifying comment --------- Co-authored-by: Avigdor Veksler Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- Python/pywarpx/picmi.py | 5 +++++ .../Resampling/VelocityCoincidenceThinning.H | 1 + .../Resampling/VelocityCoincidenceThinning.cpp | 15 +++++++++++++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 34c21c2a34a..61fad85ec94 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -137,6 +137,9 @@ class Species(picmistandard.PICMI_Species): Cells with fewer particles than this number will be skipped during resampling. + warpx_resampling_algorithm_target_weight: float + Weight that the product particles from resampling will not exceed. + warpx_resampling_trigger_intervals: bool, default=0 Timesteps at which to resample @@ -249,6 +252,7 @@ def init(self, kw): self.resampling_min_ppc = kw.pop('warpx_resampling_min_ppc', None) self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) + self.resampling_algorithm_target_weight = kw.pop('warpx_resampling_algorithm_target_weight', None) self.resampling_algorithm_velocity_grid_type = kw.pop('warpx_resampling_algorithm_velocity_grid_type', None) self.resampling_algorithm_delta_ur = kw.pop('warpx_resampling_algorithm_delta_ur', None) self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) @@ -298,6 +302,7 @@ def species_initialize_inputs(self, layout, resampling_min_ppc=self.resampling_min_ppc, resampling_trigger_intervals=self.resampling_trigger_intervals, resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc, + resampling_algorithm_target_weight=self.resampling_algorithm_target_weight, resampling_algorithm_velocity_grid_type=self.resampling_algorithm_velocity_grid_type, resampling_algorithm_delta_ur=self.resampling_algorithm_delta_ur, resampling_algorithm_n_theta=self.resampling_algorithm_n_theta, diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.H b/Source/Particles/Resampling/VelocityCoincidenceThinning.H index b8d67d198a4..bb325734777 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.H +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.H @@ -200,5 +200,6 @@ private: int m_ntheta, m_nphi; amrex::ParticleReal m_delta_ur; amrex::Vector m_delta_u; + amrex::ParticleReal m_cluster_weight = std::numeric_limits::max(); }; #endif // WARPX_VELOCITY_COINCIDENCE_THINNING_H_ diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp index db4ddb801c6..2880ace200d 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp @@ -24,6 +24,14 @@ VelocityCoincidenceThinning::VelocityCoincidenceThinning (const std::string& spe "Resampling min_ppc should be greater than or equal to 1" ); + amrex::ParticleReal target_weight = 0; + if (utils::parser::queryWithParser( + pp_species_name, "resampling_algorithm_target_weight", target_weight + )) { + // factor of 2 since each cluster is reduced to 2 particles + m_cluster_weight = target_weight * 2.0_prt; + } + std::string velocity_grid_type_str = "spherical"; pp_species_name.query( "resampling_algorithm_velocity_grid_type", velocity_grid_type_str @@ -84,7 +92,7 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, auto *const cell_offsets = bins.offsetsPtr(); const auto min_ppc = m_min_ppc; - + const auto cluster_weight = m_cluster_weight; const auto mass = pc->getMass(); // check if species mass > 0 @@ -207,10 +215,13 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, ux[part_idx], uy[part_idx], uz[part_idx], mass ); - // check if this is the last particle in the current momentum bin + // check if this is the last particle in the current momentum bin, + // or if the next particle would push the current cluster weight + // to exceed the maximum specified cluster weight if ( (i == cell_stop - 1) || (momentum_bin_number_data[sorted_indices_data[i]] != momentum_bin_number_data[sorted_indices_data[i + 1]]) + || (total_weight + w[indices[sorted_indices_data[i+1]]] > cluster_weight) ) { // check if the bin has more than 2 particles in it if ( particles_in_bin > 2 && total_weight > std::numeric_limits::min() ){ From 67419c6ad3edbbbfafb4d5e2ecbca4cece524bbe Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Wed, 5 Jun 2024 19:15:10 -0700 Subject: [PATCH 116/190] Add initialization of extra attribs to picmi (#4972) * add initialization of extra attribs to picmi * avoid new `Bucket` * update CI checksum * update ionization test analysis to always include runtime attribute * Update Examples Further, Fix CI * update docstring; update checksum values * allow user defined attributes to be added to `ParticleDiagnostic` output * more CI fixes * revert adding `z_orig` to inputs_2d_bf_rt --------- Co-authored-by: Axel Huebl --- .../laser_acceleration/PICMI_inputs_3d.py | 4 ++- .../laser_acceleration/inputs_3d | 2 +- .../laser_ion/PICMI_inputs_2d.py | 5 ++-- Examples/Tests/ionization/PICMI_inputs_2d.py | 8 +++--- .../Tests/ionization/analysis_ionization.py | 9 ++++--- Examples/Tests/ionization/inputs_2d_bf_rt | 2 +- Python/pywarpx/picmi.py | 25 +++++++++++++++++++ .../Python_LaserAcceleration.json | 4 ++- .../benchmarks_json/Python_LaserIonAcc2d.json | 4 ++- .../benchmarks_json/Python_ionization.json | 20 ++++++++------- 10 files changed, 59 insertions(+), 24 deletions(-) diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py index ac3398e43fc..13bf492e203 100755 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py +++ b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py @@ -55,7 +55,9 @@ electrons = picmi.Species( particle_type = 'electron', name = 'electrons', - initial_distribution = uniform_distribution) + initial_distribution = uniform_distribution, + warpx_add_int_attributes = {'regionofinterest': "(z>12.0e-6) * (z<13.0e-6)"}, + warpx_add_real_attributes = {'initialenergy': "ux*ux + uy*uy + uz*uz"}) # Particles: beam electrons q_tot = 1e-12 diff --git a/Examples/Physics_applications/laser_acceleration/inputs_3d b/Examples/Physics_applications/laser_acceleration/inputs_3d index bdcfd7676a4..fb8810dd34b 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_3d +++ b/Examples/Physics_applications/laser_acceleration/inputs_3d @@ -55,7 +55,7 @@ electrons.do_continuous_injection = 1 electrons.addIntegerAttributes = regionofinterest electrons.attribute.regionofinterest(x,y,z,ux,uy,uz,t) = "(z>12.0e-6) * (z<13.0e-6)" electrons.addRealAttributes = initialenergy -electrons.attribute.initialenergy(x,y,z,ux,uy,uz,t) = " ux*ux + uy*uy + uz*uz" +electrons.attribute.initialenergy(x,y,z,ux,uy,uz,t) = "ux*ux + uy*uy + uz*uz" ################################# ############ LASER ############# diff --git a/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py index 844501992c3..9f7a2aacfca 100755 --- a/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py +++ b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py @@ -91,18 +91,17 @@ rms_velocity=[c*ux_th, 0., c*uz_th] # thermal velocity spread in m/s ) -# TODO: add additional attributes orig_x and orig_z electrons = picmi.Species( particle_type='electron', name='electrons', initial_distribution=slab_with_ramp_dist_electrons, ) -# TODO: add additional attributes orig_x and orig_z hydrogen = picmi.Species( particle_type='proton', name='hydrogen', - initial_distribution=slab_with_ramp_dist_hydrogen + initial_distribution=slab_with_ramp_dist_hydrogen, + warpx_add_real_attributes = {"orig_x": "x", "orig_z": "z"} ) # Laser diff --git a/Examples/Tests/ionization/PICMI_inputs_2d.py b/Examples/Tests/ionization/PICMI_inputs_2d.py index a076361bf50..802bf5435ac 100644 --- a/Examples/Tests/ionization/PICMI_inputs_2d.py +++ b/Examples/Tests/ionization/PICMI_inputs_2d.py @@ -47,12 +47,14 @@ fill_in = True) electrons = picmi.Species( particle_type = 'electron', - name = 'electrons') + name = 'electrons', + warpx_add_real_attributes = {'orig_z': 'z'}) ions = picmi.Species( particle_type = 'N', name = 'ions', charge_state = 2, - initial_distribution = uniform_distribution) + initial_distribution = uniform_distribution, + warpx_add_real_attributes = {'orig_z': 'z'}) # Field ionization nitrogen_ionization = picmi.FieldIonization( @@ -88,7 +90,7 @@ name = 'diag1', period = 10000, species = [electrons, ions], - data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting'], + data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting', 'orig_z'], write_dir = '.', warpx_file_prefix = 'Python_ionization_plt') field_diag = picmi.FieldDiagnostic( diff --git a/Examples/Tests/ionization/analysis_ionization.py b/Examples/Tests/ionization/analysis_ionization.py index 95732b03e36..90657915b50 100755 --- a/Examples/Tests/ionization/analysis_ionization.py +++ b/Examples/Tests/ionization/analysis_ionization.py @@ -52,7 +52,7 @@ ad = ds.all_data() # Plot ions with ionization levels - species = 'ions'; + species = 'ions' xi = ad[species, 'particle_position_x'].v zi = ad[species, 'particle_position_y'].v ii = ad[species, 'particle_ionizationLevel'].v @@ -75,7 +75,7 @@ plt.colorbar() # Plot electrons - species = 'electrons'; + species = 'electrons' if species in [x[0] for x in ds.field_list]: xe = ad[species, 'particle_position_x'].v ze = ad[species, 'particle_position_y'].v @@ -96,10 +96,11 @@ # Check that the user runtime component (if it exists) worked as expected try: orig_z = ad['electrons', 'particle_orig_z'].v - assert np.all( (orig_z > 0) & (orig_z < 1.5e-5) ) + print(f"orig_z: min = {np.min(orig_z)}, max = {np.max(orig_z)}") + assert np.all( (orig_z > 0.0) & (orig_z < 1.5e-5) ) print('particle_orig_z has reasonable values') except yt.utilities.exceptions.YTFieldNotFound: - pass # Some of the tested script to not have the quantity orig_z + pass # The backtransformed diagnostic version of the test does not have orig_z test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/ionization/inputs_2d_bf_rt b/Examples/Tests/ionization/inputs_2d_bf_rt index 52af5d0d40f..8bcb66595d2 100644 --- a/Examples/Tests/ionization/inputs_2d_bf_rt +++ b/Examples/Tests/ionization/inputs_2d_bf_rt @@ -48,7 +48,7 @@ electrons.zmax = 50.e-6 electrons.profile = constant electrons.density = 2. electrons.momentum_distribution_type = at_rest -electrons.do_continuous_injection=1 +electrons.do_continuous_injection = 1 lasers.names = laser1 laser1.profile = Gaussian diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 61fad85ec94..b981bc5e1d3 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -171,6 +171,14 @@ class Species(picmistandard.PICMI_Species): during grid-based merging, with `velocity_grid_type == "cartesian"`. If a single number is given the same du value will be used in all three directions. + + warpx_add_int_attributes: dict + Dictionary of extra integer particle attributes initialized from an + expression that is a function of the variables (x, y, z, ux, uy, uz, t). + + warpx_add_real_attributes: dict + Dictionary of extra real particle attributes initialized from an + expression that is a function of the variables (x, y, z, ux, uy, uz, t). """ def init(self, kw): @@ -261,6 +269,10 @@ def init(self, kw): if self.resampling_algorithm_delta_u is not None and np.size(self.resampling_algorithm_delta_u) == 1: self.resampling_algorithm_delta_u = [self.resampling_algorithm_delta_u]*3 + # extra particle attributes + self.extra_int_attributes = kw.pop('warpx_add_int_attributes', None) + self.extra_real_attributes = kw.pop('warpx_add_real_attributes', None) + def species_initialize_inputs(self, layout, initialize_self_fields = False, injection_plane_position = None, @@ -318,6 +330,16 @@ def species_initialize_inputs(self, layout, self.species.add_new_attr("reflection_model_zhi(E)", self.reflection_model_zhi) # self.species.add_new_attr("reflection_model_eb(E)", self.reflection_model_eb) + # extra particle attributes + if self.extra_int_attributes is not None: + self.species.addIntegerAttributes = self.extra_int_attributes.keys() + for attr, function in self.extra_int_attributes.items(): + self.species.add_new_attr('attribute.'+attr+'(x,y,z,ux,uy,uz,t)', function) + if self.extra_real_attributes is not None: + self.species.addRealAttributes = self.extra_real_attributes.keys() + for attr, function in self.extra_real_attributes.items(): + self.species.add_new_attr('attribute.'+attr+'(x,y,z,ux,uy,uz,t)', function) + pywarpx.Particles.particles_list.append(self.species) if self.initial_distribution is not None: @@ -2596,6 +2618,9 @@ def diagnostic_initialize_inputs(self): ) else: variables.add(dataname) + else: + # possibly add user defined attributes + variables.add(dataname) # --- Convert the set to a sorted list so that the order # --- is the same on all processors. diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json index 08969db023e..474e812480f 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json @@ -21,12 +21,14 @@ "particle_weight": 6241509.074460764 }, "electrons": { + "particle_initialenergy": 0.0, "particle_momentum_x": 1.7921232210868553e-20, "particle_momentum_y": 7.225819896136567e-20, "particle_momentum_z": 4.2317254599358777e-20, "particle_position_x": 0.713912262116188, "particle_position_y": 0.7150340887578024, "particle_position_z": 1.31757706006908, + "particle_regionofinterest": 1936.0, "particle_weight": 12926557617.187498 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json index baaf29bec59..d2fd4841cc4 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json @@ -23,6 +23,8 @@ "particle_weight": 2.6507336926909222e+17 }, "hydrogen": { + "particle_origX": 0.008198291015625001, + "particle_origZ": 0.0365664599609375, "particle_position_x": 0.008197892199782453, "particle_position_y": 0.0, "particle_position_z": 0.0365646600930625, @@ -31,4 +33,4 @@ "particle_momentum_z": 1.0873094324185116e-18, "particle_weight": 2.703612070965676e+17 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Python_ionization.json b/Regression/Checksum/benchmarks_json/Python_ionization.json index 31f426aa362..a5e65fcf765 100644 --- a/Regression/Checksum/benchmarks_json/Python_ionization.json +++ b/Regression/Checksum/benchmarks_json/Python_ionization.json @@ -10,21 +10,23 @@ "jy": 0.0, "jz": 1.3483401471475687e-07 }, - "electrons": { - "particle_momentum_x": 4.4206237143449475e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.6361297302081026e-18, - "particle_position_x": 0.11009154442846772, - "particle_position_y": 0.6414658436421568, - "particle_weight": 3.4450781249999996e-10 - }, "ions": { "particle_ionizationLevel": 72897.0, "particle_momentum_x": 1.76132401934254e-18, "particle_momentum_y": 0.0, "particle_momentum_z": 3.644887053263054e-23, + "particle_orig_z": 0.128, "particle_position_x": 0.03200001189420337, "particle_position_y": 0.1280000046901387, "particle_weight": 9.999999999999999e-11 + }, + "electrons": { + "particle_momentum_x": 4.4206237143449475e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.6361297302081026e-18, + "particle_orig_z": 0.4305565137391907, + "particle_position_x": 0.11009154442846772, + "particle_position_y": 0.6414658436421568, + "particle_weight": 3.4450781249999996e-10 } -} \ No newline at end of file +} From dc2c2f690c85eb648036c0710fd773972b9bfde4 Mon Sep 17 00:00:00 2001 From: David Grote Date: Fri, 7 Jun 2024 11:16:23 -0700 Subject: [PATCH 117/190] Add temperature diagnostic (#4942) * Add temperature diagnostic * Add to CI test * Add input to picmi * Add comment on the method * const cleanup * Fix particle array declaration * Update benchmark for CI --- Docs/source/usage/parameters.rst | 4 +- Examples/Tests/collision/inputs_3d | 2 +- Python/pywarpx/picmi.py | 3 + .../benchmarks_json/collisionXYZ.json | 4 +- .../ComputeDiagFunctors/CMakeLists.txt | 1 + .../ComputeDiagFunctors/Make.package | 1 + .../ComputeDiagFunctors/TemperatureFunctor.H | 35 ++++ .../TemperatureFunctor.cpp | 165 ++++++++++++++++++ Source/Diagnostics/Diagnostics.H | 2 + Source/Diagnostics/Diagnostics.cpp | 24 +++ Source/Diagnostics/FullDiagnostics.cpp | 17 ++ 11 files changed, 255 insertions(+), 3 deletions(-) create mode 100644 Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H create mode 100644 Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index fb53e81012f..a3ca703d15e 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2693,7 +2693,9 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``.fields_to_plot`` (list of `strings`, optional) Fields written to output. - Possible scalar fields: ``part_per_cell`` ``rho`` ``phi`` ``F`` ``part_per_grid`` ``divE`` ``divB`` and ``rho_``, where ```` must match the name of one of the available particle species. Note that ``phi`` will only be written out when do_electrostatic==labframe. Also, note that for ``.diag_type = BackTransformed``, the only scalar field currently supported is ``rho``. + Possible scalar fields: ``part_per_cell`` ``rho`` ``phi`` ``F`` ``part_per_grid`` ``divE`` ``divB`` ``rho_`` and ``T_``, where ```` must match the name of one of the available particle species. + ``T_`` is the temperature in eV. + Note that ``phi`` will only be written out when do_electrostatic==labframe. Also, note that for ``.diag_type = BackTransformed``, the only scalar field currently supported is ``rho``. Possible vector field components in Cartesian geometry: ``Ex`` ``Ey`` ``Ez`` ``Bx`` ``By`` ``Bz`` ``jx`` ``jy`` ``jz``. Possible vector field components in RZ geometry: ``Er`` ``Et`` ``Ez`` ``Br`` ``Bt`` ``Bz`` ``jr`` ``jt`` ``jz``. The default is ``.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz`` in Cartesian geometry and ``.fields_to_plot = Er Et Ez Br Bt Bz jr jt jz`` in RZ geometry. diff --git a/Examples/Tests/collision/inputs_3d b/Examples/Tests/collision/inputs_3d index 3cc06061bed..ed413ba2776 100644 --- a/Examples/Tests/collision/inputs_3d +++ b/Examples/Tests/collision/inputs_3d @@ -75,7 +75,7 @@ collision3.ndt = 10 diagnostics.diags_names = diag1 diag_parser_filter diag_uniform_filter diag_random_filter diag1.intervals = 10 diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz +diag1.fields_to_plot = Ex Ey Ez Bx By Bz T_electron T_ion ## diag_parser_filter is a diag used to test the particle filter function. diag_parser_filter.intervals = 150:150: diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index b981bc5e1d3..0048cb1e51b 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2411,6 +2411,9 @@ def diagnostic_initialize_inputs(self): elif dataname.startswith('rho_'): # Adds rho_species diagnostic fields_to_plot.add(dataname) + elif dataname.startswith('T_'): + # Adds T_species diagnostic + fields_to_plot.add(dataname) elif dataname == 'dive': fields_to_plot.add('divE') elif dataname == 'divb': diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 927848745a8..6e4b9abf965 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -5,6 +5,8 @@ "Bz": 0.0, "Ex": 0.0, "Ey": 0.0, - "Ez": 0.0 + "Ez": 0.0, + "T_electron": 362230.52300397365, + "T_ion": 338312.83502136066 } } diff --git a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt index 5e0eeaab73a..2a5cc87c0cb 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt +++ b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt @@ -13,5 +13,6 @@ foreach(D IN LISTS WarpX_DIMS) BackTransformFunctor.cpp BackTransformParticleFunctor.cpp ParticleReductionFunctor.cpp + TemperatureFunctor.cpp ) endforeach() diff --git a/Source/Diagnostics/ComputeDiagFunctors/Make.package b/Source/Diagnostics/ComputeDiagFunctors/Make.package index fd1624b8708..0fd618748e3 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/Make.package +++ b/Source/Diagnostics/ComputeDiagFunctors/Make.package @@ -9,5 +9,6 @@ CEXE_sources += RhoFunctor.cpp CEXE_sources += BackTransformFunctor.cpp CEXE_sources += BackTransformParticleFunctor.cpp CEXE_sources += ParticleReductionFunctor.cpp +CEXE_sources += TemperatureFunctor.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Diagnostics/ComputeDiagFunctors diff --git a/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H new file mode 100644 index 00000000000..f6c425e74d5 --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H @@ -0,0 +1,35 @@ +#ifndef WARPX_TEMPERATUREFUNCTOR_H_ +#define WARPX_TEMPERATUREFUNCTOR_H_ + +#include "ComputeDiagFunctor.H" + +#include + +/** + * \brief Functor to calculate per-cell averages of particle temperature + */ +class TemperatureFunctor final : public ComputeDiagFunctor +{ +public: + /** \brief Constructor + * \param[in] lev level of multifab + * \param[in] crse_ratio for interpolating field values from simulation MultiFabs + to the output diagnostic MultiFab mf_dst + * \param[in] ispec index of the species over which to calculate the temperature + * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab + */ + TemperatureFunctor(int lev, amrex::IntVect crse_ratio, int ispec, int ncomp=1); + + /** \brief Compute the temperature in each grid cell. + * + * \param[out] mf_dst output MultiFab where the result is written + * \param[in] dcomp first component of mf_dst in which cell-centered + * data is stored + */ + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; +private: + int const m_lev; /**< level on which mf_src is defined */ + int const m_ispec; /**< index of species to average over */ +}; + +#endif // WARPX_TEMPERATUREFUNCTOR_H_ diff --git a/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp new file mode 100644 index 00000000000..c42f8970d5e --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp @@ -0,0 +1,165 @@ + +#include "TemperatureFunctor.H" + +#include "Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/Parser/ParserUtils.H" +#include "WarpX.H" + +#include + +#include +#include +#include +#include +#include + +TemperatureFunctor::TemperatureFunctor (const int lev, + const amrex::IntVect crse_ratio, const int ispec, const int ncomp) + : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev), m_ispec(ispec) +{ + // Write only in one output component. + AMREX_ALWAYS_ASSERT(ncomp == 1); +} + +void +TemperatureFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const int /*i_buffer*/) const +{ + using namespace amrex::literals; + auto& warpx = WarpX::GetInstance(); + + // Guard cell is set to 1 for generality. However, for a cell-centered + // output Multifab, mf_dst, the guard-cell data is not needed especially considering + // the operations performend in the CoarsenAndInterpolate function. + constexpr int ng = 1; + + // Temporary cell-centered, multi-component MultiFab for storing particles sums and result + amrex::MultiFab sum_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 7, ng); + + auto& pc = warpx.GetPartContainer().GetParticleContainer(m_ispec); + amrex::Real const mass = pc.getMass(); // Note, implicit conversion from ParticleReal + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(mass > 0., + "The temperature diagnostic can not be calculated for a massless species."); + + // Calculate the averages in two steps, first the average velocity , then the + // average velocity squared >**2. This method is more robust than the + // single step using - **2 when >> u_rms. + ParticleToMesh(pc, sum_mf, m_lev, + [=] AMREX_GPU_DEVICE (const WarpXParticleContainer::SuperParticleType& p, + amrex::Array4 const& out_array, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi) + { + // Get position in AMReX convention to calculate corresponding index. + // Ideally this will be replaced with the AMReX NGP interpolator + // Always do x direction. + int ii = 0, jj = 0, kk = 0; + const amrex::ParticleReal x = p.pos(0); + const amrex::Real lx = (x - plo[0]) * dxi[0]; + ii = static_cast(amrex::Math::floor(lx)); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + const amrex::ParticleReal y = p.pos(1); + const amrex::Real ly = (y - plo[1]) * dxi[1]; + jj = static_cast(amrex::Math::floor(ly)); +#endif +#if defined(WARPX_DIM_3D) + const amrex::ParticleReal z = p.pos(2); + const amrex::Real lz = (z - plo[2]) * dxi[2]; + kk = static_cast(amrex::Math::floor(lz)); +#endif + + const amrex::ParticleReal w = p.rdata(PIdx::w); + const amrex::ParticleReal ux = p.rdata(PIdx::ux); + const amrex::ParticleReal uy = p.rdata(PIdx::uy); + const amrex::ParticleReal uz = p.rdata(PIdx::uz); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 0), (amrex::Real)(w)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 1), (amrex::Real)(w*ux)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 2), (amrex::Real)(w*uy)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 3), (amrex::Real)(w*uz)); + }); + + // Divide value by number of particles for average + for (amrex::MFIter mfi(sum_mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const amrex::Box& box = mfi.tilebox(); + amrex::Array4 const& out_array = sum_mf.array(mfi); + amrex::ParallelFor(box, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + if (out_array(i,j,k,0) > 0) { + const amrex::Real invsum = 1._rt/out_array(i,j,k,0); + out_array(i,j,k,1) *= invsum; + out_array(i,j,k,2) *= invsum; + out_array(i,j,k,3) *= invsum; + } + }); + } + + // Calculate the sum of the squares, subtracting the averages + // These loops must be written out since ParticleToMesh always zeros out the mf. + const auto plo = pc.Geom(m_lev).ProbLoArray(); + const auto dxi = pc.Geom(m_lev).InvCellSizeArray(); + for (WarpXParIter pti(pc, m_lev); pti.isValid(); ++pti) + { + const long np = pti.numParticles(); + amrex::ParticleReal* wp = pti.GetAttribs(PIdx::w).dataPtr(); + amrex::ParticleReal* uxp = pti.GetAttribs(PIdx::ux).dataPtr(); + amrex::ParticleReal* uyp = pti.GetAttribs(PIdx::uy).dataPtr(); + amrex::ParticleReal* uzp = pti.GetAttribs(PIdx::uz).dataPtr(); + + auto const GetPosition = GetParticlePosition(pti); + + amrex::Array4 const& out_array = sum_mf.array(pti); + + amrex::ParallelFor(np, + [=] AMREX_GPU_DEVICE (long ip) { + // --- Get particle quantities + amrex::ParticleReal xp, yp, zp; + GetPosition.AsStored(ip, xp, yp, zp); + + // Get position in AMReX convention to calculate corresponding index. + int ii = 0, jj = 0, kk = 0; + const amrex::Real lx = (xp - plo[0]) * dxi[0]; + ii = static_cast(amrex::Math::floor(lx)); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real lz = (zp - plo[1]) * dxi[1]; + jj = static_cast(amrex::Math::floor(lz)); +#elif defined(WARPX_DIM_3D) + const amrex::Real ly = (yp - plo[1]) * dxi[1]; + jj = static_cast(amrex::Math::floor(ly)); + const amrex::Real lz = (zp - plo[2]) * dxi[2]; + kk = static_cast(amrex::Math::floor(lz)); +#endif + + const amrex::ParticleReal w = wp[ip]; + const amrex::ParticleReal ux = uxp[ip] - out_array(ii, jj, kk, 1); + const amrex::ParticleReal uy = uyp[ip] - out_array(ii, jj, kk, 2); + const amrex::ParticleReal uz = uzp[ip] - out_array(ii, jj, kk, 3); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 4), (amrex::Real)(w*ux*ux)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 5), (amrex::Real)(w*uy*uy)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 6), (amrex::Real)(w*uz*uz)); + }); + } + + // Divide the squares by number of particles for average and calculate the temperature + for (amrex::MFIter mfi(sum_mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const amrex::Box& box = mfi.tilebox(); + amrex::Array4 const& out_array = sum_mf.array(mfi); + amrex::ParallelFor(box, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + if (out_array(i,j,k,0) > 0) { + const amrex::Real invsum = 1._rt/out_array(i,j,k,0); + out_array(i,j,k,4) *= invsum; + out_array(i,j,k,5) *= invsum; + out_array(i,j,k,6) *= invsum; + out_array(i,j,k,0) = mass*(out_array(i,j,k,4) + out_array(i,j,k,5) + out_array(i,j,k,6))/(3._rt*PhysConst::q_e); + } + }); + } + + // Coarsen and interpolate from sum_mf to the output diagnostic MultiFab, mf_dst. + ablastr::coarsen::sample::Coarsen(mf_dst, sum_mf, dcomp, 0, nComp(), 0, m_crse_ratio); + +} diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index c0d2a9f0d53..20550364fb7 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -304,6 +304,8 @@ protected: int m_num_buffers; /** Array of species indices that dump rho per species */ amrex::Vector m_rho_per_species_index; + /** Array of species indices that dump temperature per species */ + amrex::Vector m_T_per_species_index; /** Vector of particle buffer vectors for each snapshot */ amrex::Vector< amrex::Vector > > m_particles_buffer; /** Vector of pointers to functors to compute particle output per species*/ diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index 3b5daabaffa..f6e2da74127 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -278,6 +278,30 @@ Diagnostics::BaseReadParameters () + ".fields_to_plot does not match any species" ); } + // Check if m_varnames contains a string of the form T_ + if (var.rfind("T_", 0) == 0) { + // Extract species name from the string T_ + const std::string species = var.substr(var.find("T_") + 2); + // Boolean used to check if species name was misspelled + bool species_name_is_wrong = true; + // Loop over all species + for (int i = 0, n = int(m_all_species_names.size()); i < n; i++) { + // Check if species name extracted from the string T_ + // matches any of the species in the simulation + if (species == m_all_species_names[i]) { + // Store species index: will be used in TemperatureFunctor to dump + // T for this species + m_T_per_species_index.push_back(i); + species_name_is_wrong = false; + } + } + // If species name was misspelled, abort with error message + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !species_name_is_wrong, + "Input error: string " + var + " in " + m_diag_name + + ".fields_to_plot does not match any species" + ); + } } const bool checkpoint_compatibility = ( diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index b25f899e29d..55af73d6408 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -8,6 +8,7 @@ #include "ComputeDiagFunctors/PartPerCellFunctor.H" #include "ComputeDiagFunctors/PartPerGridFunctor.H" #include "ComputeDiagFunctors/ParticleReductionFunctor.H" +#include "ComputeDiagFunctors/TemperatureFunctor.H" #include "ComputeDiagFunctors/RhoFunctor.H" #include "Diagnostics/Diagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" @@ -186,6 +187,8 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) // Species index to loop over species that dump rho per species int i = 0; + // Species index to loop over species that dump temperature per species + int i_T_species = 0; const int ncomp = ncomp_multimodefab; // This function is called multiple times, for different values of `lev` // but the `varnames` need only be updated once. @@ -303,6 +306,13 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) AddRZModesToOutputNames(std::string("rho_") + m_all_species_names[m_rho_per_species_index[i]], ncomp); } i++; + } else if ( m_varnames_fields[comp].rfind("T_", 0) == 0 ){ + // Initialize temperature functor to dump temperature per species + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_T_per_species_index[i_T_species]); + if (update_varnames) { + AddRZModesToOutputNames(std::string("T_") + m_all_species_names[m_T_per_species_index[i_T_species]], ncomp); + } + i_T_species++; } else if ( m_varnames_fields[comp] == "F" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio, false, ncomp); @@ -637,6 +647,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // Species index to loop over species that dump rho per species int i = 0; + // Species index to loop over species that dump temperature per species + int i_T_species = 0; + const auto nvar = static_cast(m_varnames_fields.size()); const auto nspec = static_cast(m_pfield_species.size()); const auto ntot = static_cast(nvar + m_pfield_varnames.size() * nspec); @@ -666,6 +679,10 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // Initialize rho functor to dump rho per species m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, m_rho_per_species_index[i]); i++; + } else if ( m_varnames[comp].rfind("T_", 0) == 0 ){ + // Initialize temperature functor to dump temperature per species + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_T_per_species_index[i_T_species]); + i_T_species++; } else if ( m_varnames[comp] == "F" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ From e80ca983aa27b023b0528c0ed2875a5745ea6599 Mon Sep 17 00:00:00 2001 From: David Grote Date: Fri, 7 Jun 2024 17:54:08 -0700 Subject: [PATCH 118/190] Cleanup diagnostic dimension macros (#4973) * Clean up BeamRelavent diagnostics, using get_particle_position * In Field diagnostics, simplify calculation of dV * Clean up of ParticleExtrema * In WarpXOpenPMD, removed duplicate particle coordinate transformation with RZ * Added more amrex prefixes in ParticleExtrema * Fix const * Small cleanup in WarpXOpenPMD * Clean up calculation of dV * Cleanup call to CellSize * Fix in FieldEnergy --- .../Diagnostics/ReducedDiags/BeamRelevant.cpp | 54 +--- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 14 +- .../ReducedDiags/FieldMomentum.cpp | 12 +- .../ReducedDiags/ParticleExtrema.cpp | 271 ++++++------------ Source/Diagnostics/WarpXOpenPMD.cpp | 43 +-- 5 files changed, 119 insertions(+), 275 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index ae7d5230b4c..9b5a28a3516 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -175,15 +175,6 @@ void BeamRelevant::ComputeDiags (int step) // inverse of speed of light squared Real constexpr inv_c2 = 1.0_rt / (PhysConst::c * PhysConst::c); - // If 2D-XZ, p.pos(1) is z, rather than p.pos(2). -#if (defined WARPX_DIM_3D) - int const index_z = 2; -#elif (defined WARPX_DIM_XZ || defined WARPX_DIM_RZ) - int const index_z = 1; -#elif (defined WARPX_DIM_1D_Z) - int const index_z = 0; -#endif - // loop over species for (int i_s = 0; i_s < nSpecies; ++i_s) { @@ -212,26 +203,14 @@ void BeamRelevant::ComputeDiags (int step) const ParticleReal p_uy = p.rdata(PIdx::uy); const ParticleReal p_uz = p.rdata(PIdx::uz); const ParticleReal p_us = p_ux*p_ux + p_uy*p_uy + p_uz*p_uz; - const ParticleReal p_pos0 = p.pos(0); const ParticleReal p_w = p.rdata(PIdx::w); -#if defined(WARPX_DIM_3D) - const ParticleReal p_pos1 = p.pos(1); - const ParticleReal p_x_mean = p_pos0*p_w; - const ParticleReal p_y_mean = p_pos1*p_w; -#elif defined(WARPX_DIM_RZ) - const ParticleReal p_theta = p.rdata(PIdx::theta); - const ParticleReal p_x_mean = p_pos0*std::cos(p_theta)*p_w; - const ParticleReal p_y_mean = p_pos0*std::sin(p_theta)*p_w; -#elif defined(WARPX_DIM_XZ) - const ParticleReal p_x_mean = p_pos0*p_w; - const ParticleReal p_y_mean = 0; -#elif defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(p_pos0); - const ParticleReal p_x_mean = 0; - const ParticleReal p_y_mean = 0; -#endif - const ParticleReal p_z_mean = p.pos(index_z)*p_w; + ParticleReal p_x, p_y, p_z; + get_particle_position(p, p_x, p_y, p_z); + + const ParticleReal p_x_mean = p_x*p_w; + const ParticleReal p_y_mean = p_y*p_w; + const ParticleReal p_z_mean = p_z*p_w; const ParticleReal p_ux_mean = p_ux*p_w; const ParticleReal p_uy_mean = p_uy*p_w; @@ -292,25 +271,8 @@ void BeamRelevant::ComputeDiags (int step) const ParticleReal p_gm = std::sqrt(1.0_rt+p_us*inv_c2); const ParticleReal p_w = p.rdata(PIdx::w); -#if (defined WARPX_DIM_1D_Z) - const ParticleReal p_x = 0.0; - const ParticleReal p_y = 0.0; -#elif (defined WARPX_DIM_RZ) - const ParticleReal p_pos0 = p.pos(0); - const ParticleReal p_theta = p.rdata(PIdx::theta); - const ParticleReal p_x = p_pos0*std::cos(p_theta); - const ParticleReal p_y = p_pos0*std::sin(p_theta); -#elif (defined WARPX_DIM_XZ) - const ParticleReal p_pos0 = p.pos(0); - const ParticleReal p_x = p_pos0; - const ParticleReal p_y = 0.0; -#else - const ParticleReal p_pos0 = p.pos(0); - const ParticleReal p_pos1 = p.pos(1); - const ParticleReal p_x = p_pos0; - const ParticleReal p_y = p_pos1; -#endif - const ParticleReal p_z = p.pos(index_z); + ParticleReal p_x, p_y, p_z; + get_particle_position(p, p_x, p_y, p_z); const ParticleReal p_x_ms = (p_x-x_mean)*(p_x-x_mean)*p_w; const ParticleReal p_y_ms = (p_y-y_mean)*(p_y-y_mean)*p_w; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index f4b4e2a39a1..40ef1a088e6 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -98,15 +98,9 @@ void FieldEnergy::ComputeDiags (int step) const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); - // get cell size - Geometry const & geom = warpx.Geom(lev); -#if defined(WARPX_DIM_1D_Z) - auto dV = geom.CellSize(0); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif + // get cell volume + const std::array &dx = WarpX::CellSize(lev); + const amrex::Real dV = dx[0]*dx[1]*dx[2]; #if defined(WARPX_DIM_RZ) amrex::Real const tmpEx = ComputeNorm2RZ(Ex, lev); @@ -119,6 +113,8 @@ void FieldEnergy::ComputeDiags (int step) amrex::Real const tmpBz = ComputeNorm2RZ(Bz, lev); amrex::Real const Bs = tmpBx + tmpBy + tmpBz; #else + Geometry const & geom = warpx.Geom(lev); + // compute E squared Real const tmpEx = Ex.norm2(0,geom.periodicity()); Real const tmpEy = Ey.norm2(0,geom.periodicity()); diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index f182acd5ba2..7eb16efecff 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -183,15 +183,9 @@ void FieldMomentum::ComputeDiags (int step) amrex::Real ExB_z = amrex::get<2>(r); amrex::ParallelDescriptor::ReduceRealSum({ExB_x,ExB_y,ExB_z}); - // Get cell size - amrex::Geometry const & geom = warpx.Geom(lev); -#if defined(WARPX_DIM_1D_Z) - auto dV = geom.CellSize(0); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif + // Get cell volume + const std::array &dx = WarpX::CellSize(lev); + const amrex::Real dV = dx[0]*dx[1]*dx[2]; // Save data (offset: 3 values for each refinement level) const int offset = lev*3; diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 0cc5429be7a..9adfd3f238c 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -51,7 +51,7 @@ #include #include -using namespace amrex; +using namespace amrex::literals; using namespace warpx::fields; // constructor @@ -59,7 +59,7 @@ ParticleExtrema::ParticleExtrema (const std::string& rd_name) : ReducedDiags{rd_name} { // read species name - const ParmParse pp_rd_name(rd_name); + const amrex::ParmParse pp_rd_name(rd_name); pp_rd_name.get("species",m_species_name); // get WarpX class object @@ -122,7 +122,7 @@ ParticleExtrema::ParticleExtrema (const std::string& rd_name) m_data.resize(all_diag_names.size()); - if (ParallelDescriptor::IOProcessor()) + if (amrex::ParallelDescriptor::IOProcessor()) { if ( m_write_header ) { @@ -165,16 +165,7 @@ void ParticleExtrema::ComputeDiags (int step) const auto species_names = mypc.GetSpeciesNames(); // inverse of speed of light squared - Real constexpr inv_c2 = 1.0_rt / (PhysConst::c * PhysConst::c); - - // If 2D-XZ, p.pos(1) is z, rather than p.pos(2). -#if (defined WARPX_DIM_3D) - int const index_z = 2; -#elif (defined WARPX_DIM_XZ || defined WARPX_DIM_RZ) - int const index_z = 1; -#elif (defined WARPX_DIM_1D_Z) - int const index_z = 0; -#endif + amrex::Real constexpr inv_c2 = 1.0_rt / (PhysConst::c * PhysConst::c); // loop over species for (int i_s = 0; i_s < nSpecies; ++i_s) @@ -193,172 +184,72 @@ void ParticleExtrema::ComputeDiags (int step) } using PType = typename WarpXParticleContainer::SuperParticleType; - - // xmin -#if (defined WARPX_DIM_RZ) - Real xmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::cos(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_1D_Z) - Real xmin = 0.0_rt; -#else - Real xmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0); }); -#endif - - // xmax -#if (defined WARPX_DIM_RZ) - Real xmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::cos(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_1D_Z) - Real xmax = 0.0_rt; -#else - Real xmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0); }); -#endif - - // ymin -#if (defined WARPX_DIM_RZ) - Real ymin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::sin(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_XZ || WARPX_DIM_1D_Z) - Real ymin = 0.0_rt; -#else - Real ymin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(1); }); -#endif - - // ymax -#if (defined WARPX_DIM_RZ) - Real ymax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::sin(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_XZ || WARPX_DIM_1D_Z) - Real ymax = 0.0_rt; -#else - Real ymax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(1); }); -#endif - - // zmin - Real zmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(index_z); }); - - // zmax - Real zmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(index_z); }); - - // uxmin - Real uxmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::ux); }); - - // uxmax - Real uxmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::ux); }); - - // uymin - Real uymin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uy); }); - - // uymax - Real uymax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uy); }); - - // uzmin - Real uzmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uz); }); - - // uzmax - Real uzmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uz); }); - - // gmin - Real gmin = 0.0_rt; - if ( is_photon ) { - gmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) + using OpMin = amrex::ReduceOpMin; + using OpMax = amrex::ReduceOpMax; + + amrex::ReduceOps reduce_ops; + auto posminmax = amrex::ParticleReduce>( + myspc, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(us*inv_c2); - }); - } else { - gmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) + amrex::ParticleReal x, y, z; + get_particle_position(p, x, y, z); + amrex::Real const w = p.rdata(PIdx::w); + return {w, x, y, z, w, x, y, z}; + }, + reduce_ops); + + amrex::Real wmin = amrex::get<0>(posminmax); + amrex::Real xmin = amrex::get<1>(posminmax); + amrex::Real ymin = amrex::get<2>(posminmax); + amrex::Real zmin = amrex::get<3>(posminmax); + amrex::Real wmax = amrex::get<4>(posminmax); + amrex::Real xmax = amrex::get<5>(posminmax); + amrex::Real ymax = amrex::get<6>(posminmax); + amrex::Real zmax = amrex::get<7>(posminmax); + + amrex::Real const gfactor = (is_photon ? 0._rt : 1._rt); + auto uminmax = amrex::ParticleReduce>( + myspc, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(1.0_rt + us*inv_c2); - }); - } - - // gmax - Real gmax = 0.0_rt; - if ( is_photon ) { - gmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(us*inv_c2); - }); - } else { - gmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(1.0_rt + us*inv_c2); - }); - } - - // wmin - Real wmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::w); }); - - // wmax - Real wmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::w); }); - - ParallelDescriptor::ReduceRealMin({xmin,ymin,zmin,uxmin,uymin,uzmin,gmin,wmin}); - ParallelDescriptor::ReduceRealMax({xmax,ymax,zmax,uxmax,uymax,uzmax,gmax,wmax}); + amrex::Real const ux = p.rdata(PIdx::ux); + amrex::Real const uy = p.rdata(PIdx::uy); + amrex::Real const uz = p.rdata(PIdx::uz); + amrex::Real const g = std::sqrt(gfactor + (ux*ux + uy*uy + uz*uz)*inv_c2); + return {g, ux, uy, uz, g, ux, uy, uz}; + }, + reduce_ops); + + amrex::Real gmin = amrex::get<0>(uminmax); + amrex::Real uxmin = amrex::get<1>(uminmax); + amrex::Real uymin = amrex::get<2>(uminmax); + amrex::Real uzmin = amrex::get<3>(uminmax); + amrex::Real gmax = amrex::get<4>(uminmax); + amrex::Real uxmax = amrex::get<5>(uminmax); + amrex::Real uymax = amrex::get<6>(uminmax); + amrex::Real uzmax = amrex::get<7>(uminmax); + + amrex::ParallelDescriptor::ReduceRealMin({xmin,ymin,zmin,uxmin,uymin,uzmin,gmin,wmin}); + amrex::ParallelDescriptor::ReduceRealMax({xmax,ymax,zmax,uxmax,uymax,uzmax,gmax,wmax}); #if (defined WARPX_QED) // get number of level (int) const auto level_number = WarpX::GetInstance().finestLevel(); // compute chimin and chimax - Real chimin_f = 0.0_rt; - Real chimax_f = 0.0_rt; + amrex::Real chimin_f = 0.0_rt; + amrex::Real chimax_f = 0.0_rt; if (myspc.DoQED()) { // declare chi arrays - std::vector chimin, chimax; + std::vector chimin, chimax; chimin.resize(level_number+1,0.0_rt); chimax.resize(level_number+1,0.0_rt); @@ -374,17 +265,17 @@ void ParticleExtrema::ComputeDiags (int step) { // define variables in preparation for field gathering const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - const GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // declare reduce_op - ReduceOps reduce_op; - ReduceData reduce_data(reduce_op); + amrex::ReduceOps reduce_op; + amrex::ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; // Loop over boxes @@ -408,28 +299,28 @@ void ParticleExtrema::ComputeDiags (int step) // define variables in preparation for field gathering amrex::Box box = pti.tilebox(); box.grow(ngEB); - const Dim3 lo = amrex::lbound(box); + const amrex::Dim3 lo = amrex::lbound(box); const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; const auto& ex_arr = Ex[pti].array(); const auto& ey_arr = Ey[pti].array(); const auto& ez_arr = Ez[pti].array(); const auto& bx_arr = Bx[pti].array(); const auto& by_arr = By[pti].array(); const auto& bz_arr = Bz[pti].array(); - const IndexType ex_type = Ex[pti].box().ixType(); - const IndexType ey_type = Ey[pti].box().ixType(); - const IndexType ez_type = Ez[pti].box().ixType(); - const IndexType bx_type = Bx[pti].box().ixType(); - const IndexType by_type = By[pti].box().ixType(); - const IndexType bz_type = Bz[pti].box().ixType(); + const amrex::IndexType ex_type = Ex[pti].box().ixType(); + const amrex::IndexType ey_type = Ey[pti].box().ixType(); + const amrex::IndexType ez_type = Ez[pti].box().ixType(); + const amrex::IndexType bx_type = Bx[pti].box().ixType(); + const amrex::IndexType by_type = By[pti].box().ixType(); + const amrex::IndexType bz_type = Bz[pti].box().ixType(); // evaluate reduce_op reduce_op.eval(pti.numParticles(), reduce_data, [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple { // get external fields - ParticleReal xp, yp, zp; + amrex::ParticleReal xp, yp, zp; GetPosition(i, xp, yp, zp); amrex::ParticleReal ex = Ex_external_particle; amrex::ParticleReal ey = Ey_external_particle; @@ -449,7 +340,7 @@ void ParticleExtrema::ComputeDiags (int step) dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); // compute chi - Real chi = 0.0_rt; + amrex::Real chi = 0.0_rt; if ( is_photon ) { chi = QedUtils::chi_photon(ux[i]*m, uy[i]*m, uz[i]*m, ex, ey, ez, bx, by, bz); @@ -461,13 +352,13 @@ void ParticleExtrema::ComputeDiags (int step) }); } auto val = reduce_data.value(); - chimin[lev] = get<0>(val); - chimax[lev] = get<1>(val); + chimin[lev] = amrex::get<0>(val); + chimax[lev] = amrex::get<1>(val); } chimin_f = *std::min_element(chimin.begin(), chimin.end()); chimax_f = *std::max_element(chimax.begin(), chimax.end()); - ParallelDescriptor::ReduceRealMin(chimin_f, ParallelDescriptor::IOProcessorNumber()); - ParallelDescriptor::ReduceRealMax(chimax_f, ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::ReduceRealMin(chimin_f, amrex::ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::ReduceRealMax(chimax_f, amrex::ParallelDescriptor::IOProcessorNumber()); } #endif diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 45d5dc1cdee..36827bd316a 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -911,33 +911,34 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, { auto const real_counter = std::min(write_real_comp.size(), real_comp_names.size()); +#if defined(WARPX_DIM_RZ) // reconstruct Cartesian positions for RZ simulations // r,z,theta -> x,y,z -#if defined(WARPX_DIM_RZ) - auto const * const r = soa.GetRealData(PIdx::x).data(); - auto const * const theta = soa.GetRealData(PIdx::theta).data(); + // If each comp is being written, create a temporary array, otherwise create an empty array. + std::shared_ptr const x( + new amrex::ParticleReal[(write_real_comp[0] ? numParticleOnTile : 0)], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + std::shared_ptr const y( + new amrex::ParticleReal[(write_real_comp[1] ? numParticleOnTile : 0)], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + const auto& tile = pti.GetParticleTile(); + const auto& ptd = tile.getConstParticleTileData(); + + for (int i = 0; i < numParticleOnTile; ++i) { + const auto& p = ptd.getSuperParticle(i); + amrex::ParticleReal xp, yp, zp; + get_particle_position(p, xp, yp, zp); + if (write_real_comp[0]) { x.get()[i] = xp; } + if (write_real_comp[1]) { y.get()[i] = yp; } + } if (write_real_comp[0]) { - std::shared_ptr const x( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (int i = 0; i < numParticleOnTile; ++i) { - x.get()[i] = r[i] * std::cos(theta[i]); - } - getComponentRecord(real_comp_names[0]).storeChunk( - x, {offset}, {numParticleOnTile64}); + getComponentRecord(real_comp_names[0]).storeChunk(x, {offset}, {numParticleOnTile64}); } if (write_real_comp[1]) { - std::shared_ptr const y( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (int i = 0; i < numParticleOnTile; ++i) { - y.get()[i] = r[i] * std::sin(theta[i]); - } - getComponentRecord(real_comp_names[1]).storeChunk( - y, {offset}, {numParticleOnTile64}); + getComponentRecord(real_comp_names[1]).storeChunk(y, {offset}, {numParticleOnTile64}); } #endif From 7269f09307ce608428bdfc406544c0df7284ab40 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 11 Jun 2024 01:03:29 +0200 Subject: [PATCH 119/190] slightly increase tolerance for embedded_circle test (#4968) --- Examples/Tests/embedded_circle/inputs_2d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Tests/embedded_circle/inputs_2d b/Examples/Tests/embedded_circle/inputs_2d index 7100f245917..7c76c1659da 100644 --- a/Examples/Tests/embedded_circle/inputs_2d +++ b/Examples/Tests/embedded_circle/inputs_2d @@ -5,7 +5,7 @@ max_step = 11 warpx.const_dt = 3.99e-13 warpx.do_electrostatic = labframe -warpx.self_fields_required_precision = 1e-06 +warpx.self_fields_required_precision = 2e-06 warpx.eb_implicit_function = -((x-0.00005)**2+(z-0.00005)**2-1e-05**2) warpx.eb_potential(x,y,z,t) = -10 warpx.self_fields_absolute_tolerance = 0.02 From a1eb908e897c5fb8cdad40fa0d041a1508c19d50 Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 10 Jun 2024 17:05:06 -0700 Subject: [PATCH 120/190] Remove unneeded macros from AllocLevelData (#4979) --- Source/Parallelization/GuardCellManager.H | 2 +- Source/Parallelization/GuardCellManager.cpp | 2 +- Source/WarpX.cpp | 8 +------- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 341db01bef6..561456943f1 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -52,7 +52,7 @@ public: */ void Init( amrex::Real dt, - amrex::RealVect dx, + const amrex::Real * dx, bool do_subcycling, bool do_fdtd_nci_corr, short grid_type, diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 28157e09d8c..321be15df7e 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -33,7 +33,7 @@ using namespace amrex; void guardCellManager::Init ( const amrex::Real dt, - const amrex::RealVect dx, + const amrex::Real *dx, const bool do_subcycling, const bool do_fdtd_nci_corr, const short grid_type, diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index d3f91002ef4..ef81aef4482 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2120,13 +2120,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d { const bool aux_is_nodal = (field_gathering_algo == GatheringAlgo::MomentumConserving); -#if defined(WARPX_DIM_1D_Z) - const amrex::RealVect dx(WarpX::CellSize(lev)[2]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::RealVect dx = {WarpX::CellSize(lev)[0], WarpX::CellSize(lev)[2]}; -#elif defined(WARPX_DIM_3D) - const amrex::RealVect dx = {WarpX::CellSize(lev)[0], WarpX::CellSize(lev)[1], WarpX::CellSize(lev)[2]}; -#endif + const Real* dx = Geom(lev).CellSize(); // Initialize filter before guard cells manager // (needs info on length of filter's stencil) From a3cd47d19b16722721bb206ebdb4da52c4dca8af Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 11 Jun 2024 06:24:39 +0200 Subject: [PATCH 121/190] Release 24.06 (#4980) * AMReX: 24.06 * pyAMReX: 24.06 * WarpX: 24.06 --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 07422c36e15..3a03ea01c0c 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd - + cd ../amrex && git checkout --detach 24.06 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 85b4a42b040..81c12a3df2f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.05) +project(WarpX VERSION 24.06) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 471f6f2b6a3..18ca8370e80 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.05' +version = u'24.06' # The full version, including alpha/beta/rc tags. -release = u'24.05' +release = u'24.06' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index fbf3330ada0..9f9c8d7d736 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.05', + version = '24.06', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 8542e5f35d9..05ed74fe1b1 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 28b010126a1b39297d8a496ba81f171d8563953b +branch = 24.06 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index e2d69be9c60..41db9a15bdc 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 28b010126a1b39297d8a496ba81f171d8563953b +branch = 24.06 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index f80ec0d5af7..eda45dc9f77 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.05 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.06 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "28b010126a1b39297d8a496ba81f171d8563953b" +set(WarpX_amrex_branch "24.06" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index cdef8f277f6..18cab89e347 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.05 CONFIG REQUIRED) + find_package(pyAMReX 24.06 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "d4d409bd21bc4c48487883ac2331efdb1a6b3d61" +set(WarpX_pyamrex_branch "24.06" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index e9dbb0f2533..f397d31048e 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 28b010126a1b39297d8a496ba81f171d8563953b && cd - +cd amrex && git checkout --detach 24.06 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index 3d61cc5be36..670e376986f 100644 --- a/setup.py +++ b/setup.py @@ -278,7 +278,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.05', + version = '24.06', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From c7b203e5670f57e6b9955db8c3f80d718f070b1c Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Tue, 11 Jun 2024 23:27:08 +0200 Subject: [PATCH 122/190] group initialization and cleanup of external libraries (#4964) --- Source/Evolve/WarpXEvolve.cpp | 3 +++ Source/Initialization/CMakeLists.txt | 1 + Source/Initialization/Make.package | 1 + Source/Initialization/WarpXInit.H | 30 ++++++++++++++++++++++++++ Source/Initialization/WarpXInit.cpp | 29 +++++++++++++++++++++++++ Source/main.cpp | 32 ++++++---------------------- 6 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 Source/Initialization/WarpXInit.H create mode 100644 Source/Initialization/WarpXInit.cpp diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 4ac50483ad3..8b2cc2c33f1 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -306,6 +306,9 @@ WarpX::Evolve (int numsteps) multi_diags->FilterComputePackFlushLastTimestep( istep[0] ); if (m_exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } } + + amrex::Print() << + ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("THE END"); } /* /brief Perform one PIC iteration, without subcycling diff --git a/Source/Initialization/CMakeLists.txt b/Source/Initialization/CMakeLists.txt index 8931de740ad..e5e2334fd7e 100644 --- a/Source/Initialization/CMakeLists.txt +++ b/Source/Initialization/CMakeLists.txt @@ -11,6 +11,7 @@ foreach(D IN LISTS WarpX_DIMS) TemperatureProperties.cpp VelocityProperties.cpp WarpXAMReXInit.cpp + WarpXInit.cpp WarpXInitData.cpp ) endforeach() diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index 8b4a4c1d669..831e3fc3f89 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -7,6 +7,7 @@ CEXE_sources += PlasmaInjector.cpp CEXE_sources += TemperatureProperties.cpp CEXE_sources += VelocityProperties.cpp CEXE_sources += WarpXAMReXInit.cpp +CEXE_sources += WarpXInit.cpp CEXE_sources += WarpXInitData.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization diff --git a/Source/Initialization/WarpXInit.H b/Source/Initialization/WarpXInit.H new file mode 100644 index 00000000000..ce179e2e997 --- /dev/null +++ b/Source/Initialization/WarpXInit.H @@ -0,0 +1,30 @@ +/* Copyright 2024 Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_INIT_H_ +#define WARPX_INIT_H_ + +namespace warpx::initialization +{ + /** Initializes, in the following order: + * - the MPI library through the mpi_init helper function in ablastr + * - the AMReX library + * - the FFT library through the anyfft::setup() function in ablastr + * + * @param[in] argc number of arguments from main() + * @param[in] argv argument strings from main() + */ + void initialize_external_libraries(int argc, char* argv[]); + + /** Initializes, in the following order: + * - the FFT library through the anyfft::cleanup() function in ablastr + * - the AMReX library + * - the MPI library through the mpi_finalize helper function in ablastr + */ + void finalize_external_libraries(); +} + +#endif //WARPX_INIT_H_ diff --git a/Source/Initialization/WarpXInit.cpp b/Source/Initialization/WarpXInit.cpp new file mode 100644 index 00000000000..7e00760bf30 --- /dev/null +++ b/Source/Initialization/WarpXInit.cpp @@ -0,0 +1,29 @@ +/* Copyright 2024 Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "WarpXInit.H" + +#include "Initialization/WarpXAMReXInit.H" + +#include + +#include +#include + +void warpx::initialization::initialize_external_libraries(int argc, char* argv[]) +{ + ablastr::parallelization::mpi_init(argc, argv); + warpx::initialization::amrex_init(argc, argv); + ablastr::math::anyfft::setup(); +} + +void warpx::initialization::finalize_external_libraries() +{ + ablastr::math::anyfft::cleanup(); + amrex::Finalize(); + ablastr::parallelization::mpi_finalize(); +} diff --git a/Source/main.cpp b/Source/main.cpp index 2a1b828c64f..9273cd3928b 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -8,25 +8,16 @@ */ #include "WarpX.H" -#include "Initialization/WarpXAMReXInit.H" +#include "Initialization/WarpXInit.H" #include "Utils/WarpXProfilerWrapper.H" -#include -#include #include -#include #include - int main(int argc, char* argv[]) { - ablastr::parallelization::mpi_init(argc, argv); - - warpx::initialization::amrex_init(argc, argv); - - ablastr::math::anyfft::setup(); - + warpx::initialization::initialize_external_libraries(argc, argv); { WARPX_PROFILE_VAR("main()", pmain); @@ -34,29 +25,18 @@ int main(int argc, char* argv[]) timer.record_start_time(); auto& warpx = WarpX::GetInstance(); - warpx.InitData(); - warpx.Evolve(); - - amrex::Print() << - ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("THE END"); + const auto is_warpx_verbose = warpx.Verbose(); + WarpX::Finalize(); timer.record_stop_time(); - if (warpx.Verbose()) - { + if (is_warpx_verbose){ amrex::Print() << "Total Time : " << timer.get_global_duration() << '\n'; } WARPX_PROFILE_VAR_STOP(pmain); - - WarpX::Finalize(); } - - ablastr::math::anyfft::cleanup(); - - amrex::Finalize(); - - ablastr::parallelization::mpi_finalize (); + warpx::initialization::finalize_external_libraries(); } From 2602b540efcd15f3ffebc3017312e98f46cdc7af Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 12 Jun 2024 00:20:20 +0200 Subject: [PATCH 123/190] Fix FieldProbe Check: Particle Shape (#4983) The constructor of FieldProbe might be called earlier than the WarpX class parameter init. That could lead to relying on an uninitialized particle shape static. Use the parser instead, similar to our general efforts to reduce the static members stored in the WarpX class. --- Source/Diagnostics/ReducedDiags/FieldProbe.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 7364d5989f1..1fc3b957ec9 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -154,7 +154,12 @@ FieldProbe::FieldProbe (const std::string& rd_name) ablastr::warn_manager::WarnPriority::low); } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(interp_order <= WarpX::nox , + // ensure assumption holds: we read the fields in the interpolation kernel as they are, + // without further communication of guard/ghost/halo regions + int particle_shape; + const ParmParse pp_algo("algo"); + utils::parser::getWithParser(pp_algo, "particle_shape", particle_shape); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(interp_order <= particle_shape , "Field probe interp_order should be less than or equal to algo.particle_shape"); if (ParallelDescriptor::IOProcessor()) { From 3c4e523e255dfa20663e45b416f828924366f2a1 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 13 Jun 2024 23:16:14 +0200 Subject: [PATCH 124/190] CMake: heFFTe Support (#4986) * CMake: heFFTe * Doc: Perlmutter heFFTe * Update Spack Dev Envs * Finalize After Testing Co-authored-by: Alfred Mishi <140518333+Haavaan@users.noreply.github.com> --------- Co-authored-by: Alfred Mishi <140518333+Haavaan@users.noreply.github.com> --- CMakeLists.txt | 37 ++++++++++++++++++ Docs/source/install/cmake.rst | 2 + Docs/source/install/dependencies.rst | 7 ++-- Docs/source/install/hpc/perlmutter.rst | 8 ++-- .../machines/desktop/spack-macos-openmp.yaml | 1 + Tools/machines/desktop/spack-ubuntu-cuda.yaml | 1 + .../machines/desktop/spack-ubuntu-openmp.yaml | 1 + Tools/machines/desktop/spack-ubuntu-rocm.yaml | 1 + .../install_cpu_dependencies.sh | 39 +++++++++++++++++++ .../install_gpu_dependencies.sh | 38 ++++++++++++++++++ .../perlmutter_cpu_warpx.profile.example | 2 + .../perlmutter_gpu_warpx.profile.example | 2 + cmake/WarpXFunctions.cmake | 1 + setup.py | 2 + 14 files changed, 135 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 81c12a3df2f..5e2d1ebba9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -79,6 +79,7 @@ option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) option(WarpX_FFT "FFT-based solvers" OFF) +option(WarpX_HEFFTE "Multi-node FFT-based solvers" OFF) option(WarpX_PYTHON "Python bindings" OFF) option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF) option(WarpX_QED "QED support (requires PICSAR)" ON) @@ -136,6 +137,10 @@ mark_as_advanced(WarpX_MPI_THREAD_MULTIPLE) option(WarpX_amrex_internal "Download & build AMReX" ON) +if(WarpX_HEFFTE AND NOT WarpX_MPI) + message(FATAL_ERROR "WarpX_HEFFTE (${WarpX_HEFFTE}) can only be used if WarpX_MPI is ON.") +endif() + # change the default build type to Release (or RelWithDebInfo) instead of Debug set_default_build_type("Release") @@ -174,6 +179,10 @@ option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT}) if(WarpX_FFT) set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE) endif() +option(ABLASTR_HEFFTE "compile AnyFFT wrappers" ${WarpX_HEFFTE}) +if(WarpX_HEFFTE) + set(ABLASTR_HEFFTE ON CACHE STRING "Multi-Node FFT-based solvers" FORCE) +endif() # this defined the variable BUILD_TESTING which is ON by default #include(CTest) @@ -215,6 +224,23 @@ if(WarpX_FFT) endif() endif() +# multi-node FFT +if(WarpX_HEFFTE) + if(WarpX_COMPUTE STREQUAL CUDA) + set(_heFFTe_COMPS CUDA) + elseif(WarpX_COMPUTE STREQUAL HIP) + set(_heFFTe_COMPS ROCM) + elseif(WarpX_COMPUTE STREQUAL SYCL) + set(_heFFTe_COMPS ONEAPI) + else() # NOACC, OMP + set(_heFFTe_COMPS FFTW) # or MKL + endif() + # note: we could also enforce GPUAWARE for CUDA and HIP, which can still be + # disabled at runtime + + find_package(Heffte REQUIRED COMPONENTS ${_heFFTe_COMPS}) +endif() + # Python if(WarpX_PYTHON) find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) @@ -455,6 +481,10 @@ foreach(D IN LISTS WarpX_DIMS) endif() endif() + if(ABLASTR_HEFFTE) + target_link_libraries(ablastr_${SD} PUBLIC Heffte::Heffte) + endif() + if(WarpX_PYTHON) target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras) if(WarpX_PYTHON_IPO) @@ -539,6 +569,13 @@ foreach(D IN LISTS WarpX_DIMS) target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_FFT) endif() + if(WarpX_HEFFTE) + target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_HEFFTE) + endif() + if(ABLASTR_HEFFTE) + target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_HEFFTE) + endif() + if(WarpX_PYTHON AND pyWarpX_VERSION_INFO) # for module __version__ target_compile_definitions(pyWarpX_${SD} PRIVATE diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index b9fd0b2be45..fde927c10e1 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -97,6 +97,7 @@ CMake Option Default & Values Descr ``WarpX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WarpX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set ``WarpX_FFT`` ON/**OFF** FFT-based solvers +``WarpX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WarpX_PYTHON`` ON/**OFF** Python bindings ``WarpX_QED`` **ON**/OFF QED support (requires PICSAR) ``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost) @@ -271,6 +272,7 @@ Environment Variable Default & Values Descr ``WARPX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WARPX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set ``WARPX_FFT`` ON/**OFF** FFT-based solvers +``WARPX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WARPX_QED`` **ON**/OFF PICSAR QED (requires PICSAR) ``WARPX_QED_TABLE_GEN`` ON/**OFF** QED table generation (requires PICSAR and Boost) ``BUILD_PARALLEL`` ``2`` Number of threads to use for parallel builds diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index ce9f9dca520..3bab32b7502 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -23,12 +23,13 @@ Optional dependencies include: - for on-node accelerated compute *one of either*: - `OpenMP 3.1+ `__: for threaded CPU execution or - - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `_) or + - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `__) or - `ROCm 5.2+ (5.5+ recommended) `__: for AMD GPU support -- `FFTW3 `_: for spectral solver (PSATD) support when running on CPU or SYCL +- `FFTW3 `__: for spectral solver (PSATD or IGF) support when running on CPU or SYCL - also needs the ``pkg-config`` tool on Unix -- `BLAS++ `_ and `LAPACK++ `_: for spectral solver (PSATD) support in RZ geometry +- `heFFTe 2.4.0+ `__ and `LAPACK++ `__: for spectral solver (PSATD) support in RZ geometry - `Boost 1.66.0+ `__: for QED lookup tables generation support - `openPMD-api 0.15.1+ `__: we automatically download and compile a copy of openPMD-api for openPMD I/O support diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst index 9612b64476d..dc5a985e99f 100644 --- a/Docs/source/install/hpc/perlmutter.rst +++ b/Docs/source/install/hpc/perlmutter.rst @@ -153,7 +153,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu - cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. @@ -164,7 +164,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu_py - cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu_py -j 16 --target pip_install .. tab-item:: CPU Nodes @@ -174,7 +174,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_cpu - cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_cpu/bin/``. @@ -184,7 +184,7 @@ Use the following :ref:`cmake commands ` to compile the applicat rm -rf build_pm_cpu_py - cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu_py -j 16 --target pip_install Now, you can :ref:`submit Perlmutter compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Tools/machines/desktop/spack-macos-openmp.yaml b/Tools/machines/desktop/spack-macos-openmp.yaml index 820cf7069fd..3ea78625b78 100644 --- a/Tools/machines/desktop/spack-macos-openmp.yaml +++ b/Tools/machines/desktop/spack-macos-openmp.yaml @@ -23,6 +23,7 @@ spack: - conduit ~fortran - fftw - hdf5 ~fortran + - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - llvm-openmp diff --git a/Tools/machines/desktop/spack-ubuntu-cuda.yaml b/Tools/machines/desktop/spack-ubuntu-cuda.yaml index 08d0c95ee4b..19b9ae12e24 100644 --- a/Tools/machines/desktop/spack-ubuntu-cuda.yaml +++ b/Tools/machines/desktop/spack-ubuntu-cuda.yaml @@ -25,6 +25,7 @@ spack: - cuda - fftw - hdf5 + - heffte - lapackpp - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-openmp.yaml b/Tools/machines/desktop/spack-ubuntu-openmp.yaml index b658f1e009d..1eb7d4074a7 100644 --- a/Tools/machines/desktop/spack-ubuntu-openmp.yaml +++ b/Tools/machines/desktop/spack-ubuntu-openmp.yaml @@ -22,6 +22,7 @@ spack: - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - fftw - hdf5 + - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-rocm.yaml b/Tools/machines/desktop/spack-ubuntu-rocm.yaml index 45c9b0f776e..7eee1baa13c 100644 --- a/Tools/machines/desktop/spack-ubuntu-rocm.yaml +++ b/Tools/machines/desktop/spack-ubuntu-rocm.yaml @@ -21,6 +21,7 @@ spack: - cmake - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - hdf5 + - heffte - hip - lapackpp - llvm-amdgpu diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index f65e43891d0..a5a7c28b85e 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -109,6 +109,45 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-cpu-build +# heFFTe +if [ -d $HOME/src/heffte ] +then + cd $HOME/src/heffte + git fetch --prune + git checkout v2.4.0 + git pull + cd - +else + git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte +fi +rm -rf ${HOME}/src/heffte-pm-cpu-build +cmake \ + -S ${HOME}/src/heffte \ + -B ${build_dir}/heffte-pm-cpu-build \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ + -DHeffte_DISABLE_GPU_AWARE_MPI=ON \ + -DHeffte_ENABLE_AVX=ON \ + -DHeffte_ENABLE_AVX512=OFF \ + -DHeffte_ENABLE_FFTW=ON \ + -DHeffte_ENABLE_CUDA=OFF \ + -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF \ + -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_SEQUENTIAL_TESTING=OFF \ + -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_TRACING=OFF \ + -DHeffte_ENABLE_PYTHON=OFF \ + -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_SWIG=OFF \ + -DHeffte_ENABLE_MAGMA=OFF +cmake --build ${build_dir}/heffte-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/heffte-pm-cpu-build + # Python ###################################################################### # diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index 9ac5800c6ce..125c63104be 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -109,6 +109,44 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-gpu-build +# heFFTe +if [ -d $HOME/src/heffte ] +then + cd $HOME/src/heffte + git fetch --prune + git checkout v2.4.0 + cd - +else + git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte +fi +rm -rf ${HOME}/src/heffte-pm-gpu-build +cmake \ + -S ${HOME}/src/heffte \ + -B ${build_dir}/heffte-pm-gpu-build \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ + -DHeffte_DISABLE_GPU_AWARE_MPI=OFF \ + -DHeffte_ENABLE_AVX=OFF \ + -DHeffte_ENABLE_AVX512=OFF \ + -DHeffte_ENABLE_FFTW=OFF \ + -DHeffte_ENABLE_CUDA=ON \ + -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF \ + -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_SEQUENTIAL_TESTING=OFF \ + -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_TRACING=OFF \ + -DHeffte_ENABLE_PYTHON=OFF \ + -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_SWIG=OFF \ + -DHeffte_ENABLE_MAGMA=OFF +cmake --build ${build_dir}/heffte-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/heffte-pm-gpu-build + # Python ###################################################################### # diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 1b0ac3182d5..3fea9b3aa39 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -19,11 +19,13 @@ export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 759df0b923a..9e1465d6c02 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -23,11 +23,13 @@ export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.2 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index a68ebca6c60..c12cedca3b2 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -453,6 +453,7 @@ function(warpx_print_summary) message(" PARTICLE PRECISION: ${WarpX_PARTICLE_PRECISION}") message(" PRECISION: ${WarpX_PRECISION}") message(" FFT Solvers: ${WarpX_FFT}") + message(" heFFTe: ${WarpX_HEFFTE}") message(" PYTHON: ${WarpX_PYTHON}") if(WarpX_PYTHON) message(" PYTHON IPO: ${WarpX_PYTHON_IPO}") diff --git a/setup.py b/setup.py index 670e376986f..6ad6fd81960 100644 --- a/setup.py +++ b/setup.py @@ -97,6 +97,7 @@ def build_extension(self, ext): '-DWarpX_PRECISION=' + WARPX_PRECISION, '-DWarpX_PARTICLE_PRECISION=' + WARPX_PARTICLE_PRECISION, '-DWarpX_FFT:BOOL=' + WARPX_FFT, + '-DWarpX_HEFFTE:BOOL=' + WARPX_HEFFTE, '-DWarpX_PYTHON:BOOL=ON', '-DWarpX_PYTHON_IPO:BOOL=' + WARPX_PYTHON_IPO, '-DWarpX_QED:BOOL=' + WARPX_QED, @@ -206,6 +207,7 @@ def build_extension(self, ext): WARPX_PRECISION = env.pop('WARPX_PRECISION', 'DOUBLE') WARPX_PARTICLE_PRECISION = env.pop('WARPX_PARTICLE_PRECISION', WARPX_PRECISION) WARPX_FFT = env.pop('WARPX_FFT', 'OFF') +WARPX_HEFFTE = env.pop('WARPX_HEFFTE', 'OFF') WARPX_QED = env.pop('WARPX_QED', 'ON') WARPX_QED_TABLE_GEN = env.pop('WARPX_QED_TABLE_GEN', 'OFF') WARPX_DIMS = env.pop('WARPX_DIMS', '1;2;RZ;3') From 828332089e58ef7686a8a8753062eb505514cd7e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Jun 2024 00:25:51 -0700 Subject: [PATCH 125/190] Resetting collisionXYY Temperature (#4999) Resetting after a recent change. Might need more investigation. --- Regression/Checksum/benchmarks_json/collisionXYZ.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 6e4b9abf965..c87bbd98e42 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,7 +6,7 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 362230.52300397365, - "T_ion": 338312.83502136066 + "T_electron": 358778.6506903592, + "T_ion": 341562.3085776466 } } From 41fc45bfb83d27e68964e83916fe154d05921f10 Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 18 Jun 2024 11:21:48 -0700 Subject: [PATCH 126/190] Cleanup deposition and gather routines (#4978) * Initial clean up of current deposition * Fix of current deposition * Further cleanup of current deposition * Cleanup SharedDepositionUtils * Clean up in charge deposition * Clean up field gather plus other fixes * Change LowerCorner to return XDim3, add InvCellSize * Change variable name in Lower and UpperCorner to avoid confusion * Add const * Update RepellingParticles CI test * Update Python_ionization CI test * Update ionization_boost and ionization_lab CI tests * Update PEC_particle CI test * Update BTD_rz CI test --- .../repelling_particles/analysis_repelling.py | 10 +- .../Checksum/benchmarks_json/BTD_rz.json | 12 +- .../benchmarks_json/PEC_particle.json | 40 +- .../benchmarks_json/Python_ionization.json | 30 +- .../benchmarks_json/RepellingParticles.json | 44 +- .../benchmarks_json/ionization_boost.json | 30 +- .../benchmarks_json/ionization_lab.json | 32 +- .../LatticeElementFinder.cpp | 2 +- .../WarpXFieldBoundaries.cpp | 4 +- .../ReducedDiags/ColliderRelevant.cpp | 8 +- .../Diagnostics/ReducedDiags/FieldEnergy.cpp | 4 +- .../Diagnostics/ReducedDiags/FieldProbe.cpp | 9 +- .../ReducedDiags/ParticleExtrema.cpp | 9 +- .../SpectralSolver/SpectralFieldDataRZ.cpp | 4 +- Source/FieldSolver/WarpXPushFieldsEM.cpp | 16 +- .../Particles/Deposition/ChargeDeposition.H | 115 +--- .../Particles/Deposition/CurrentDeposition.H | 548 +++++------------- .../Deposition/SharedDepositionUtils.H | 32 +- .../Particles/ElementaryProcess/Ionization.H | 6 +- .../ElementaryProcess/Ionization.cpp | 5 +- .../ElementaryProcess/QEDPairGeneration.H | 6 +- .../ElementaryProcess/QEDPairGeneration.cpp | 5 +- .../ElementaryProcess/QEDPhotonEmission.H | 6 +- .../ElementaryProcess/QEDPhotonEmission.cpp | 7 +- Source/Particles/Gather/FieldGather.H | 205 +++---- Source/Particles/PhotonParticleContainer.cpp | 11 +- .../Particles/PhysicalParticleContainer.cpp | 33 +- .../RigidInjectedParticleContainer.cpp | 8 +- Source/Particles/WarpXParticleContainer.cpp | 81 +-- Source/Utils/WarpXUtil.cpp | 4 +- Source/WarpX.H | 5 +- Source/WarpX.cpp | 27 +- Source/ablastr/particles/DepositCharge.H | 14 +- 33 files changed, 496 insertions(+), 876 deletions(-) diff --git a/Examples/Tests/repelling_particles/analysis_repelling.py b/Examples/Tests/repelling_particles/analysis_repelling.py index ebff010fc40..bda3d74d274 100755 --- a/Examples/Tests/repelling_particles/analysis_repelling.py +++ b/Examples/Tests/repelling_particles/analysis_repelling.py @@ -36,7 +36,7 @@ # Check plotfile name specified in command line last_filename = sys.argv[1] -filename_radical = re.findall('(.*?)\d+/*$', last_filename)[0] +filename_radical = re.findall(r'(.*?)\d+/*$', last_filename)[0] # Loop through files, and extract the position and velocity of both particles x1 = [] @@ -48,10 +48,10 @@ ds = yt.load(filename) ad = ds.all_data() - x1.append( float(ad[('electron1','particle_position_x')]) ) - x2.append( float(ad[('electron2','particle_position_x')]) ) - beta1.append( float(ad[('electron1','particle_momentum_x')])/(m_e*c) ) - beta2.append( float(ad[('electron2','particle_momentum_x')])/(m_e*c) ) + x1.append( float(ad[('electron1','particle_position_x')][0]) ) + x2.append( float(ad[('electron2','particle_position_x')][0]) ) + beta1.append( float(ad[('electron1','particle_momentum_x')][0])/(m_e*c) ) + beta2.append( float(ad[('electron2','particle_momentum_x')][0])/(m_e*c) ) # Convert to numpy array x1 = np.array(x1) diff --git a/Regression/Checksum/benchmarks_json/BTD_rz.json b/Regression/Checksum/benchmarks_json/BTD_rz.json index 01e4c687292..cf83ce01346 100644 --- a/Regression/Checksum/benchmarks_json/BTD_rz.json +++ b/Regression/Checksum/benchmarks_json/BTD_rz.json @@ -1,13 +1,13 @@ { "lev=0": { - "Br": 1.8705552264208163e-08, - "Bt": 2380179.5677922587, - "Bz": 2.4079077116116535e-08, + "Br": 1.8705569155952808e-08, + "Bt": 2380179.567792259, + "Bz": 2.4079063431898616e-08, "Er": 497571119514841.25, - "Et": 7.048225464720808, - "Ez": 137058870936728.28, + "Et": 7.048223219552, + "Ez": 137058870936728.23, "jr": 0.0, "jt": 0.0, "jz": 0.0 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/PEC_particle.json b/Regression/Checksum/benchmarks_json/PEC_particle.json index 53253cebb0a..0c502a6d615 100644 --- a/Regression/Checksum/benchmarks_json/PEC_particle.json +++ b/Regression/Checksum/benchmarks_json/PEC_particle.json @@ -1,31 +1,31 @@ { "lev=0": { - "Bx": 6.52080713617892e-07, - "By": 5.323555640562078e-18, - "Bz": 1.1834601056649145e-06, - "Ex": 300.06725745121753, - "Ey": 320.82687281173185, - "Ez": 185.62208934593673, - "jx": 1.857782359527958e-06, - "jy": 257766.55567863808, + "Bx": 6.5208071361789e-07, + "By": 5.323579900863346e-18, + "Bz": 1.1834601056649094e-06, + "Ex": 300.0672574512165, + "Ey": 320.82687281173, + "Ez": 185.62208934593605, + "jx": 1.8577823595279576e-06, + "jy": 257766.5556786359, "jz": 0.0 }, + "electron": { + "particle_momentum_x": 1.1142179380730402e-32, + "particle_momentum_y": 4.5160903214571184e-36, + "particle_momentum_z": 5.735409609139057e-50, + "particle_position_x": 3.199800000000006e-05, + "particle_position_y": 6.267072536608953e-23, + "particle_position_z": 3.3938629027782103e-37, + "particle_weight": 1.0 + }, "proton": { - "particle_momentum_x": 1.0975605406855061e-33, + "particle_momentum_x": 1.097560540685506e-33, "particle_momentum_y": 1.002878875615427e-18, - "particle_momentum_z": 5.084424741580567e-51, + "particle_momentum_z": 8.484417024161229e-51, "particle_position_x": 3.1998e-05, "particle_position_y": 6.572670690061996e-06, - "particle_position_z": 5.681444715216788e-38, - "particle_weight": 1.0 - }, - "electron": { - "particle_momentum_x": 1.1142179380730476e-32, - "particle_momentum_y": 4.516090321457367e-36, - "particle_momentum_z": 2.2354046360787e-50, - "particle_position_x": 3.199800000000006e-05, - "particle_position_y": 6.267072536609136e-23, - "particle_position_z": 1.4145775327446577e-37, + "particle_position_z": 4.372139867361774e-39, "particle_weight": 1.0 } } diff --git a/Regression/Checksum/benchmarks_json/Python_ionization.json b/Regression/Checksum/benchmarks_json/Python_ionization.json index a5e65fcf765..7918a30244d 100644 --- a/Regression/Checksum/benchmarks_json/Python_ionization.json +++ b/Regression/Checksum/benchmarks_json/Python_ionization.json @@ -3,30 +3,30 @@ "Bx": 0.0, "By": 26296568.434868, "Bz": 0.0, - "Ex": 7878103122971888.0, + "Ex": 7878103122971890.0, "Ey": 0.0, - "Ez": 3027.995293554466, - "jx": 1.2111358330750162e+16, + "Ez": 3027.738911163427, + "jx": 1.2111358330750164e+16, "jy": 0.0, - "jz": 1.3483401471475687e-07 + "jz": 1.3523766702993914e-07 + }, + "electrons": { + "particle_momentum_x": 4.4237309006289776e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.643146520891287e-18, + "particle_orig_z": 0.4306994012391905, + "particle_position_x": 0.11013574385450274, + "particle_position_y": 0.6415447480129455, + "particle_weight": 3.4456249999999997e-10 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.76132401934254e-18, + "particle_momentum_x": 1.7613240193425407e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.644887053263054e-23, + "particle_momentum_z": 3.6448870533515125e-23, "particle_orig_z": 0.128, "particle_position_x": 0.03200001189420337, "particle_position_y": 0.1280000046901387, "particle_weight": 9.999999999999999e-11 - }, - "electrons": { - "particle_momentum_x": 4.4206237143449475e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.6361297302081026e-18, - "particle_orig_z": 0.4305565137391907, - "particle_position_x": 0.11009154442846772, - "particle_position_y": 0.6414658436421568, - "particle_weight": 3.4450781249999996e-10 } } diff --git a/Regression/Checksum/benchmarks_json/RepellingParticles.json b/Regression/Checksum/benchmarks_json/RepellingParticles.json index 57b36cdc90c..0331363053a 100644 --- a/Regression/Checksum/benchmarks_json/RepellingParticles.json +++ b/Regression/Checksum/benchmarks_json/RepellingParticles.json @@ -1,32 +1,32 @@ { + "lev=0": { + "Bx": 0.0, + "By": 10603.681421961119, + "Bz": 0.0, + "Ex": 12084698236273.04, + "Ey": 0.0, + "Ez": 16282812598779.527, + "F": 21124.39775810062, + "divE": 1.495787831639977e+18, + "jx": 496020344341010.9, + "jy": 0.0, + "jz": 10.641859867595102, + "rho": 6408706.535999998 + }, "electron1": { - "particle_momentum_x": 7.303686586478214e-23, + "particle_momentum_x": 7.303686586478215e-23, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.556862195127266e-36, - "particle_position_x": 1.293487226551532e-05, - "particle_position_y": 1.777889117487881e-19, + "particle_momentum_z": 1.552083282915395e-36, + "particle_position_x": 1.293487226551533e-05, + "particle_position_y": 1.7783750855614515e-19, "particle_weight": 5000000000000.0 }, "electron2": { - "particle_momentum_x": 7.303686586477884e-23, + "particle_momentum_x": 7.303686586477887e-23, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.567350238825104e-36, + "particle_momentum_z": 1.5731022908002534e-36, "particle_position_x": 1.293487226551495e-05, - "particle_position_y": 1.797289839952283e-19, + "particle_position_y": 1.7989574553330234e-19, "particle_weight": 5000000000000.0 - }, - "lev=0": { - "Bx": 0.0, - "By": 10603.68142196111, - "Bz": 0.0, - "Ex": 12084698236273.04, - "Ey": 0.0, - "Ez": 16282812598779.53, - "F": 21124.39775810061, - "divE": 1.495787831639981e+18, - "jx": 496020344341011.4, - "jy": 0.0, - "jz": 10.93513947024536, - "rho": 6408706.535999998 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/ionization_boost.json b/Regression/Checksum/benchmarks_json/ionization_boost.json index 35b2db84a1a..dbde29d19f3 100644 --- a/Regression/Checksum/benchmarks_json/ionization_boost.json +++ b/Regression/Checksum/benchmarks_json/ionization_boost.json @@ -1,30 +1,30 @@ { "lev=0": { "Bx": 0.0, - "By": 18263123.342891, + "By": 18263123.342890993, "Bz": 0.0, - "Ex": 5472992180428804.0, + "Ex": 5472992180428803.0, "Ey": 0.0, - "Ez": 922.5589707939612, - "jx": 12440856508004.96, + "Ez": 922.2798874201462, + "jx": 12440856508004.957, "jy": 0.0, - "jz": 78616.0000011086 - }, - "electrons": { - "particle_momentum_x": 2.1386770170623736e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7287241262743654e-17, - "particle_position_x": 0.11064981928849067, - "particle_position_y": 1.7121826057017473, - "particle_weight": 3.0755014319061045e-09 + "jz": 78616.00000110897 }, "ions": { "particle_ionizationLevel": 52741.0, - "particle_momentum_x": 3.630619728387593e-18, + "particle_momentum_x": 3.630619728387586e-18, "particle_momentum_y": 0.0, "particle_momentum_z": 1.0432995297946715e-13, "particle_position_x": 0.021439968272741083, "particle_position_y": 0.4742770090248555, "particle_weight": 5.000948082142308e-10 + }, + "electrons": { + "particle_momentum_x": 2.138486151076505e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7285619821244245e-17, + "particle_position_x": 0.11066682695880013, + "particle_position_y": 1.712069774427624, + "particle_weight": 3.0755014319061037e-09 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/ionization_lab.json b/Regression/Checksum/benchmarks_json/ionization_lab.json index 82984141b59..99d93020532 100644 --- a/Regression/Checksum/benchmarks_json/ionization_lab.json +++ b/Regression/Checksum/benchmarks_json/ionization_lab.json @@ -3,29 +3,29 @@ "Bx": 0.0, "By": 26296568.434868, "Bz": 0.0, - "Ex": 7878103122971888.0, + "Ex": 7878103122971890.0, "Ey": 0.0, - "Ez": 3027.995293554467, - "jx": 1.2111358330750162e+16, + "Ez": 3027.7389111634266, + "jx": 1.2111358330750164e+16, "jy": 0.0, - "jz": 1.3575651149270143e-07 + "jz": 1.355962687103163e-07 + }, + "electrons": { + "particle_momentum_x": 4.4195594812814575e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.6520195422151263e-18, + "particle_orig_z": 0.43004993872230357, + "particle_position_x": 0.11017033205040265, + "particle_position_y": 0.6412498318709239, + "particle_weight": 3.442265625e-10 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.7613240052056494e-18, + "particle_momentum_x": 1.761324005205651e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.6186967375178554e-23, + "particle_momentum_z": 3.618696737610014e-23, "particle_position_x": 0.0320000118071032, "particle_position_y": 0.12800000462171646, "particle_weight": 9.999999999999999e-11 - }, - "electrons": { - "particle_momentum_x": 4.428135211584547e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.6627518442038486e-18, - "particle_orig_z": 0.43043207622230534, - "particle_position_x": 0.11023346490802505, - "particle_position_y": 0.6417814148540429, - "particle_weight": 3.44453125e-10 } -} \ No newline at end of file +} diff --git a/Source/AcceleratorLattice/LatticeElementFinder.cpp b/Source/AcceleratorLattice/LatticeElementFinder.cpp index ba2e51715fb..ec784049760 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.cpp +++ b/Source/AcceleratorLattice/LatticeElementFinder.cpp @@ -60,7 +60,7 @@ LatticeElementFinder::UpdateIndices (int const lev, amrex::MFIter const& a_mfi, // Note that the current box is used since the box may have been updated since // the initialization in InitElementFinder. const amrex::Box box = a_mfi.tilebox(); - m_zmin = WarpX::LowerCorner(box, lev, 0._rt)[2]; + m_zmin = WarpX::LowerCorner(box, lev, 0._rt).z; m_time = warpx.gett_new(lev); if (accelerator_lattice.h_quad.nelements > 0) { diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index a5e0004325a..2b063b99a15 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -201,8 +201,8 @@ WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex // Lower corner of tile box physical domain // Note that this is done before the tilebox.grow so that // these do not include the guard cells. - const std::array& xyzmin = LowerCorner(tilebox, lev, 0._rt); - const amrex::Real rmin = xyzmin[0]; + const amrex::XDim3 xyzmin = LowerCorner(tilebox, lev, 0._rt); + const amrex::Real rmin = xyzmin.x; // Skip blocks that don't touch the axis if (rmin > 0._rt) { continue; } diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 434ab1d7949..1b0c74cf7fa 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -441,8 +441,7 @@ void ColliderRelevant::ComputeDiags (int step) const int lev = 0; // define variables in preparation for field gathering - const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); @@ -478,8 +477,7 @@ void ColliderRelevant::ComputeDiags (int step) amrex::Box box = pti.tilebox(); box.grow(ngEB); const amrex::Dim3 lo = amrex::lbound(box); - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const amrex::Array4 & ex_arr = Ex[pti].array(); const amrex::Array4 & ey_arr = Ey[pti].array(); const amrex::Array4 & ez_arr = Ez[pti].array(); @@ -515,7 +513,7 @@ void ColliderRelevant::ComputeDiags (int step) ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); // compute chi amrex::Real chi = 0.0_rt; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index 40ef1a088e6..9c0fac101a2 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -178,10 +178,10 @@ FieldEnergy::ComputeNorm2RZ(const amrex::MultiFab& field, const int lev) amrex::Box tb = convert(tilebox, field.ixType().toIntVect()); // Lower corner of tile box physical domain - const std::array& xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); const Dim3 lo = lbound(tilebox); const Dim3 hi = ubound(tilebox); - const Real rmin = xyzmin[0] + (tb.ixType().nodeCentered(0) ? 0._rt : 0.5_rt*dr); + const Real rmin = xyzmin.x + (tb.ixType().nodeCentered(0) ? 0._rt : 0.5_rt*dr); const int irmin = lo.x; const int irmax = hi.x; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 1fc3b957ec9..1113075f4cd 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -489,11 +489,8 @@ void FieldProbe::ComputeDiags (int step) auto * const AMREX_RESTRICT idcpu = pti.GetStructOfArrays().GetIdCPUData().data(); - const auto &xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const std::array &dx = WarpX::CellSize(lev); - - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); + const amrex::XDim3 dinv = WarpX::InvCellSize(lev); const Dim3 lo = lbound(box); // Temporarily defining modes and interp outside ParallelFor to avoid GPU compilation errors. @@ -514,7 +511,7 @@ void FieldProbe::ComputeDiags (int step) doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, arrEx, arrEy, arrEz, arrBx, arrBy, arrBz, Extype, Eytype, Eztype, Bxtype, Bytype, Bztype, - dx_arr, xyzmin_arr, lo, temp_modes, + dinv, xyzmin, lo, temp_modes, temp_interp_order, false); //Calculate the Poynting Vector S diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 9adfd3f238c..811fee7a9de 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -264,8 +264,8 @@ void ParticleExtrema::ComputeDiags (int step) for (int lev = 0; lev <= level_number; ++lev) { // define variables in preparation for field gathering - const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); @@ -300,8 +300,7 @@ void ParticleExtrema::ComputeDiags (int step) amrex::Box box = pti.tilebox(); box.grow(ngEB); const amrex::Dim3 lo = amrex::lbound(box); - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const auto& ex_arr = Ex[pti].array(); const auto& ey_arr = Ey[pti].array(); const auto& ez_arr = Ez[pti].array(); @@ -337,7 +336,7 @@ void ParticleExtrema::ComputeDiags (int step) ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); // compute chi amrex::Real chi = 0.0_rt; diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp index b26248fb2d0..d295d61b43a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp @@ -199,8 +199,8 @@ SpectralFieldDataRZ::SpectralFieldDataRZ (const int lev, #endif // Create the Hankel transformer for each box. - std::array xmax = WarpX::UpperCorner(mfi.tilebox(), lev, 0._rt); - multi_spectral_hankel_transformer[mfi] = SpectralHankelTransformer(grid_size[0], n_rz_azimuthal_modes, xmax[0]); + amrex::XDim3 const xyzmax = WarpX::UpperCorner(mfi.tilebox(), lev, 0._rt); + multi_spectral_hankel_transformer[mfi] = SpectralHankelTransformer(grid_size[0], n_rz_azimuthal_modes, xyzmax.x); } } diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 6631c1b88ee..602a2666b27 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -1245,11 +1245,11 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu // Lower corner of tile box physical domain // Note that this is done before the tilebox.grow so that // these do not include the guard cells. - const std::array& xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); - const Real rmin = xyzmin[0]; - const Real rminr = xyzmin[0] + (tbr.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); - const Real rmint = xyzmin[0] + (tbt.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); - const Real rminz = xyzmin[0] + (tbz.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); + const Real rmin = xyzmin.x; + const Real rminr = xyzmin.x + (tbr.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const Real rmint = xyzmin.x + (tbt.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const Real rminz = xyzmin.x + (tbz.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); const Dim3 lo = lbound(tilebox); const int irmin = lo.x; @@ -1416,10 +1416,10 @@ WarpX::ApplyInverseVolumeScalingToChargeDensity (MultiFab* Rho, int lev) // Lower corner of tile box physical domain // Note that this is done before the tilebox.grow so that // these do not include the guard cells. - const std::array& xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); const Dim3 lo = lbound(tilebox); - const Real rmin = xyzmin[0]; - const Real rminr = xyzmin[0] + (tb.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const Real rmin = xyzmin.x; + const Real rminr = xyzmin.x + (tb.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); const int irmin = lo.x; const int ishift = (rminr > rmin ? 1 : 0); diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index 2912b769c57..d3d36dbfbda 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -28,8 +28,8 @@ ion_lev is a null pointer. * \param rho_fab FArrayBox of charge density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. - * \param dx 3D cell size - * \param xyzmin Physical lower bounds of domain. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. @@ -40,37 +40,19 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, const int* ion_lev, amrex::FArrayBox& rho_fab, long np_to_deposit, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]] int n_rz_azimuthal_modes) { using namespace amrex; // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; amrex::Array4 const& rho_arr = rho_fab.array(); amrex::IntVect const rho_type = rho_fab.box().type(); @@ -98,19 +80,12 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, // Get particle position in grid coordinates #if defined(WARPX_DIM_RZ) const amrex::Real rp = std::sqrt(xp*xp + yp*yp); - amrex::Real costheta; - amrex::Real sintheta; - if (rp > 0.) { - costheta = xp/rp; - sintheta = yp/rp; - } else { - costheta = 1._rt; - sintheta = 0._rt; - } + const amrex::Real costheta = (rp > 0._rt ? xp/rp : 1._rt); + const amrex::Real sintheta = (rp > 0._rt ? yp/rp : 0._rt); const Complex xy0 = Complex{costheta, sintheta}; - const amrex::Real x = (rp - xmin)*dxi; + const amrex::Real x = (rp - xyzmin.x)*dinv.x; #else - const amrex::Real x = (xp - xmin)*dxi; + const amrex::Real x = (xp - xyzmin.x)*dinv.x; #endif // Compute shape factor along x @@ -125,7 +100,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, #endif //defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) #if defined(WARPX_DIM_3D) // y direction - const amrex::Real y = (yp - ymin)*dyi; + const amrex::Real y = (yp - xyzmin.y)*dinv.y; amrex::Real sy[depos_order + 1] = {0._rt}; int j = 0; if (rho_type[1] == NODE) { @@ -135,7 +110,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, } #endif // z direction - const amrex::Real z = (zp - zmin)*dzi; + const amrex::Real z = (zp - xyzmin.z)*dinv.z; amrex::Real sz[depos_order + 1] = {0._rt}; int k = 0; if (rho_type[WARPX_ZINDEX] == NODE) { @@ -151,8 +126,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, &rho_arr(lo.x+k+iz, 0, 0, 0), sz[iz]*wq); } -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ amrex::Gpu::Atomic::AddNoRet( @@ -182,10 +156,6 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, #endif } ); - -#ifndef WARPX_DIM_RZ - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif } /* \brief Perform charge deposition on a tile using shared memory @@ -198,8 +168,8 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, * \param rho_fab FArrayBox of charge density, either full array or tile. * \param ix_type * \param np_to_deposit Number of particles for which charge is deposited. - * \param dx 3D cell size - * \param xyzmin Physical lower bounds of domain. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. @@ -216,11 +186,11 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio amrex::FArrayBox& rho_fab, const amrex::IntVect& ix_type, const long np_to_deposit, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, + [[maybe_unused]]const int n_rz_azimuthal_modes, const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, @@ -239,26 +209,8 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; amrex::Array4 const& rho_arr = rho_fab.array(); auto rho_box = rho_fab.box(); @@ -269,7 +221,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio // Loop over particles and deposit into rho_fab #if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) - const auto dxiarr = geom.InvCellSizeArray(); const auto plo = geom.ProbLoArray(); const auto domain = geom.Domain(); @@ -316,15 +267,14 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio ParticleReal xp, yp, zp; GetPosition(permutation[bin_start], xp, yp, zp); #if defined(WARPX_DIM_3D) - IntVect iv = IntVect(int(amrex::Math::floor((xp-plo[0])*dxiarr[0])), - int(amrex::Math::floor((yp-plo[1])*dxiarr[1])), - int(amrex::Math::floor((zp-plo[2])*dxiarr[2]))); + IntVect iv = IntVect(int(amrex::Math::floor((xp-plo[0])*dinv.x)), + int(amrex::Math::floor((yp-plo[1])*dinv.y)), + int(amrex::Math::floor((zp-plo[2])*dinv.z))); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - IntVect iv = IntVect( - int(amrex::Math::floor((xp-plo[0])*dxiarr[0])), - int(amrex::Math::floor((zp-plo[1])*dxiarr[1]))); + IntVect iv = IntVect(int(amrex::Math::floor((xp-plo[0])*dinv.x)), + int(amrex::Math::floor((zp-plo[1])*dinv.z))); #elif defined(WARPX_DIM_1D_Z) - IntVect iv = IntVect(int(amrex::Math::floor((zp-plo[0])*dxiarr[0]))); + IntVect iv = IntVect(int(amrex::Math::floor((zp-plo[0])*dinv.z))); #endif iv += domain.smallEnd(); getTileIndex(iv, box, true, bin_size, buffer_box); @@ -381,9 +331,9 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio sintheta = 0._rt; } const Complex xy0 = Complex{costheta, sintheta}; - const amrex::Real x = (rp - xmin)*dxi; + const amrex::Real x = (rp - xyzmin.x)*dinv.x; #else - const amrex::Real x = (xp - xmin)*dxi; + const amrex::Real x = (xp - xyzmin.x)*dinv.x; #endif // Compute shape factor along x @@ -398,7 +348,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif //defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) #if defined(WARPX_DIM_3D) // y direction - const amrex::Real y = (yp - ymin)*dyi; + const amrex::Real y = (yp - xyzmin.y)*dinv.y; amrex::Real sy[depos_order + 1] = {0._rt}; int j = 0; if (rho_type[1] == NODE) { @@ -408,7 +358,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio } #endif // z direction - const amrex::Real z = (zp - zmin)*dzi; + const amrex::Real z = (zp - xyzmin.z)*dinv.z; amrex::Real sz[depos_order + 1] = {0._rt}; int k = 0; if (rho_type[WARPX_ZINDEX] == NODE) { @@ -424,8 +374,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio &buf(lo.x+k+iz, 0, 0, 0), sz[iz]*wq); } -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ amrex::Gpu::Atomic::AddNoRet( @@ -462,10 +411,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif // defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) } ); - -#ifndef WARPX_DIM_RZ - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif } #endif // WARPX_CHARGEDEPOSITION_H_ diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 3dce7d28534..cb56c559bc0 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -24,6 +24,7 @@ #include #include #include +#include #include /** @@ -38,17 +39,17 @@ * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dzi, dxi, dyi The inverse cell sizes - * \param zmin, xmin, ymin The lower bounds of the domain + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param invvol The inverse volume of a grid cell * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. */ template AMREX_GPU_HOST_DEVICE AMREX_INLINE -void doDepositionShapeNKernel(const amrex::ParticleReal xp, - const amrex::ParticleReal yp, - const amrex::ParticleReal zp, +void doDepositionShapeNKernel([[maybe_unused]] const amrex::ParticleReal xp, + [[maybe_unused]] const amrex::ParticleReal yp, + [[maybe_unused]] const amrex::ParticleReal zp, const amrex::ParticleReal wq, const amrex::ParticleReal vx, const amrex::ParticleReal vy, @@ -60,28 +61,14 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, amrex::IntVect const& jy_type, amrex::IntVect const& jz_type, const amrex::Real relative_time, - AMREX_D_DECL(const amrex::Real dzi, - const amrex::Real dxi, - const amrex::Real dyi), - AMREX_D_DECL(const amrex::Real zmin, - const amrex::Real xmin, - const amrex::Real ymin), + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Real invvol, const amrex::Dim3 lo, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif -#if defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(xp, yp); -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(yp); -#endif - constexpr int zdir = WARPX_ZINDEX; constexpr int NODE = amrex::IndexType::NODE; constexpr int CELL = amrex::IndexType::CELL; @@ -92,15 +79,8 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, const amrex::Real xpmid = xp + relative_time*vx; const amrex::Real ypmid = yp + relative_time*vy; const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); - amrex::Real costheta; - amrex::Real sintheta; - if (rpmid > 0._rt) { - costheta = xpmid/rpmid; - sintheta = ypmid/rpmid; - } else { - costheta = 1._rt; - sintheta = 0._rt; - } + const amrex::Real costheta = (rpmid > 0._rt ? xpmid/rpmid : 1._rt); + const amrex::Real sintheta = (rpmid > 0._rt ? ypmid/rpmid : 0._rt); const Complex xy0 = Complex{costheta, sintheta}; const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); @@ -115,12 +95,13 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, #if (AMREX_SPACEDIM >= 2) // x direction // Get particle position after 1/2 push back in position -#if defined(WARPX_DIM_RZ) // Keep these double to avoid bug in single precision - const double xmid = (rpmid - xmin)*dxi; +#if defined(WARPX_DIM_RZ) + const double xmid = (rpmid - xyzmin.x)*dinv.x; #else - const double xmid = ((xp - xmin) + relative_time*vx)*dxi; + const double xmid = ((xp - xyzmin.x) + relative_time*vx)*dinv.x; #endif + // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current // sx_j[xyz] shape factor along x for the centering of each current // There are only two possible centerings, node or cell centered, so at most only two shape factor @@ -155,7 +136,7 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, #if defined(WARPX_DIM_3D) // y direction // Keep these double to avoid bug in single precision - const double ymid = ((yp - ymin) + relative_time*vy)*dyi; + const double ymid = ((yp - xyzmin.y) + relative_time*vy)*dinv.y; double sy_node[depos_order + 1] = {0.}; double sy_cell[depos_order + 1] = {0.}; int k_node = 0; @@ -182,7 +163,8 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, // z direction // Keep these double to avoid bug in single precision - const double zmid = ((zp - zmin) + relative_time*vz)*dzi; + constexpr int zdir = WARPX_ZINDEX; + const double zmid = ((zp - xyzmin.z) + relative_time*vz)*dinv.z; double sz_node[depos_order + 1] = {0.}; double sz_cell[depos_order + 1] = {0.}; int l_node = 0; @@ -282,7 +264,7 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -300,41 +282,19 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jz_fab, long np_to_deposit, amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]]int n_rz_azimuthal_modes) { using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; const amrex::Real clightsq = 1.0_rt/PhysConst::c/PhysConst::c; @@ -367,9 +327,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, jx_type, jy_type, jz_type, - relative_time, - AMREX_D_DECL(dzi, dxi, dyi), - AMREX_D_DECL(zmin, xmin, ymin), + relative_time, dinv, xyzmin, invvol, lo, n_rz_azimuthal_modes); } @@ -391,7 +349,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, ion_lev is a null pointer. * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -411,40 +369,19 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, amrex::FArrayBox& jy_fab, amrex::FArrayBox& jz_fab, const long np_to_deposit, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes) + [[maybe_unused]]const int n_rz_azimuthal_modes) { using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; amrex::Array4 const& jx_arr = jx_fab.array(); amrex::Array4 const& jy_arr = jy_fab.array(); @@ -483,9 +420,7 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, const amrex::Real relative_time = 0._rt; doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, jx_type, jy_type, jz_type, - relative_time, - AMREX_D_DECL(dzi, dxi, dyi), - AMREX_D_DECL(zmin, xmin, ymin), + relative_time, dinv, xyzmin, invvol, lo, n_rz_azimuthal_modes); } @@ -509,7 +444,7 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -527,8 +462,8 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jz_fab, long np_to_deposit, const amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, int n_rz_azimuthal_modes, @@ -634,7 +569,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, { const unsigned int ip = permutation[ip_orig]; depositComponent(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_buff, jx_type, - relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, + relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, ip, zdir, NODE, CELL, 0); } @@ -649,7 +584,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, { const unsigned int ip = permutation[ip_orig]; depositComponent(GetPosition, wp, uxp, uyp, uzp, ion_lev, jy_buff, jy_type, - relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, + relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, ip, zdir, NODE, CELL, 1); } @@ -664,7 +599,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, { const unsigned int ip = permutation[ip_orig]; depositComponent(GetPosition, wp, uxp, uyp, uzp, ion_lev, jz_buff, jz_type, - relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, + relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, ip, zdir, NODE, CELL, 2); } @@ -675,7 +610,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, // Note, you should never reach this part of the code. This funcion cannot be called unless // using HIP/CUDA, and those things are checked prior //don't use any args - ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, a_bins, box, geom, a_tbox_max_size); + ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, a_bins, box, geom, a_tbox_max_size); WARPX_ABORT_WITH_MESSAGE("Shared memory only implemented for HIP/CUDA"); #endif } @@ -698,7 +633,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -717,53 +652,28 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, long np_to_deposit, amrex::Real dt, amrex::Real relative_time, - const std::array& dx, - std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]]int n_rz_azimuthal_modes) { using namespace amrex; using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - ignore_unused(n_rz_azimuthal_modes); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; +#if !defined(WARPX_DIM_3D) + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; #endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - Real const invdtdx = 1.0_rt / (dt*dx[1]*dx[2]); - Real const invdtdy = 1.0_rt / (dt*dx[0]*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]*dx[1]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - Real const invdtdx = 1.0_rt / (dt*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[0]*dx[2]); -#elif defined(WARPX_DIM_1D_Z) - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[2]); -#endif + amrex::XDim3 const invdtd = amrex::XDim3{(1.0_rt/dt)*dinv.y*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.y}; -#if defined(WARPX_DIM_RZ) - Complex const I = Complex{0._rt, 1._rt}; -#endif + Real constexpr clightsq = 1.0_rt / ( PhysConst::c * PhysConst::c ); - Real const clightsq = 1.0_rt / ( PhysConst::c * PhysConst::c ); #if !defined(WARPX_DIM_1D_Z) Real constexpr one_third = 1.0_rt / 3.0_rt; Real constexpr one_sixth = 1.0_rt / 6.0_rt; @@ -778,7 +688,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, + uyp[ip]*uyp[ip]*clightsq + uzp[ip]*uzp[ip]*clightsq); - // wqx, wqy wqz are particle current in each direction Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; @@ -787,14 +696,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, ParticleReal xp, yp, zp; GetPosition(ip, xp, yp, zp); -#if !defined(WARPX_DIM_1D_Z) - Real const wqx = wq*invdtdx; -#endif -#if defined(WARPX_DIM_3D) - Real const wqy = wq*invdtdy; -#endif - Real const wqz = wq*invdtdz; - // computes current and old position in grid units #if defined(WARPX_DIM_RZ) Real const xp_new = xp + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv; @@ -806,51 +707,33 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); Real const rp_mid = std::sqrt(xp_mid*xp_mid + yp_mid*yp_mid); Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); - Real costheta_new, sintheta_new; - if (rp_new > 0._rt) { - costheta_new = xp_new/rp_new; - sintheta_new = yp_new/rp_new; - } else { - costheta_new = 1._rt; - sintheta_new = 0._rt; - } - amrex::Real costheta_mid, sintheta_mid; - if (rp_mid > 0._rt) { - costheta_mid = xp_mid/rp_mid; - sintheta_mid = yp_mid/rp_mid; - } else { - costheta_mid = 1._rt; - sintheta_mid = 0._rt; - } - amrex::Real costheta_old, sintheta_old; - if (rp_old > 0._rt) { - costheta_old = xp_old/rp_old; - sintheta_old = yp_old/rp_old; - } else { - costheta_old = 1._rt; - sintheta_old = 0._rt; - } + const amrex::Real costheta_mid = (rp_mid > 0._rt ? xp_mid/rp_mid : 1._rt); + const amrex::Real sintheta_mid = (rp_mid > 0._rt ? yp_mid/rp_mid : 0._rt); + const amrex::Real costheta_new = (rp_new > 0._rt ? xp_new/rp_new : 1._rt); + const amrex::Real sintheta_new = (rp_new > 0._rt ? yp_new/rp_new : 0._rt); + const amrex::Real costheta_old = (rp_old > 0._rt ? xp_old/rp_old : 1._rt); + const amrex::Real sintheta_old = (rp_old > 0._rt ? yp_old/rp_old : 0._rt); const Complex xy_new0 = Complex{costheta_new, sintheta_new}; const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; const Complex xy_old0 = Complex{costheta_old, sintheta_old}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; #else #if !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp - xmin + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv)*dxi; - double const x_old = x_new - dt*dxi*uxp[ip]*gaminv; + double const x_new = (xp - xyzmin.x + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv)*dinv.x; + double const x_old = x_new - dt*dinv.x*uxp[ip]*gaminv; #endif #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp - ymin + (relative_time + 0.5_rt*dt)*uyp[ip]*gaminv)*dyi; - double const y_old = y_new - dt*dyi*uyp[ip]*gaminv; + double const y_new = (yp - xyzmin.y + (relative_time + 0.5_rt*dt)*uyp[ip]*gaminv)*dinv.y; + double const y_old = y_new - dt*dinv.y*uyp[ip]*gaminv; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp - zmin + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv)*dzi; - double const z_old = z_new - dt*dzi*uzp[ip]*gaminv; + double const z_new = (zp - xyzmin.z + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv)*dinv.z; + double const z_old = z_new - dt*dinv.z*uzp[ip]*gaminv; #if defined(WARPX_DIM_RZ) Real const vy = (-uxp[ip]*sintheta_mid + uyp[ip]*costheta_mid)*gaminv; @@ -861,6 +744,12 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, Real const vy = uyp[ip]*gaminv; #endif + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + const Compute_shape_factor< depos_order > compute_shape_factor; + const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + // Shape factor arrays // Note that there are extra values above and below // to possibly hold the factor for the old particle @@ -869,30 +758,17 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #if !defined(WARPX_DIM_1D_Z) double sx_new[depos_order + 3] = {0.}; double sx_old[depos_order + 3] = {0.}; -#endif -#if defined(WARPX_DIM_3D) - // Keep these double to avoid bug in single precision - double sy_new[depos_order + 3] = {0.}; - double sy_old[depos_order + 3] = {0.}; -#endif - // Keep these double to avoid bug in single precision - double sz_new[depos_order + 3] = {0.}; - double sz_old[depos_order + 3] = {0.}; - - // --- Compute shape factors - // Compute shape factors for position as they are now and at old positions - // [ijk]_new: leftmost grid point that the particle touches - const Compute_shape_factor< depos_order > compute_shape_factor; - const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; - -#if !defined(WARPX_DIM_1D_Z) const int i_new = compute_shape_factor(sx_new+1, x_new); const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); #endif #if defined(WARPX_DIM_3D) + double sy_new[depos_order + 3] = {0.}; + double sy_old[depos_order + 3] = {0.}; const int j_new = compute_shape_factor(sy_new+1, y_new); const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); #endif + double sz_new[depos_order + 3] = {0.}; + double sz_old[depos_order + 3] = {0.}; const int k_new = compute_shape_factor(sz_new+1, z_new); const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); @@ -917,7 +793,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int j=djl; j<=depos_order+2-dju; j++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*( + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*( one_third*(sy_new[j]*sz_new[k] + sy_old[j]*sz_old[k]) +one_sixth*(sy_new[j]*sz_old[k] + sy_old[j]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdxi); @@ -928,7 +804,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdyj = 0._rt; for (int j=djl; j<=depos_order+1-dju; j++) { - sdyj += wqy*(sy_old[j] - sy_new[j])*( + sdyj += wq*invdtd.y*(sy_old[j] - sy_new[j])*( one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdyj); @@ -939,7 +815,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*( + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*( one_third*(sx_new[i]*sy_new[j] + sx_old[i]*sy_old[j]) +one_sixth*(sx_new[i]*sy_old[j] + sx_old[i]*sy_new[j])); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdzk); @@ -952,7 +828,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int k=dkl; k<=depos_order+2-dku; k++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdxi); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -973,6 +849,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdyj); #if defined(WARPX_DIM_RZ) + Complex const I = Complex{0._rt, 1._rt}; Complex xy_new = xy_new0; Complex xy_mid = xy_mid0; Complex xy_old = xy_old0; @@ -980,7 +857,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { // The factor 2 comes from the normalization of the modes // The minus sign comes from the different convention with respect to Davidson et al. - const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(amrex::Real)imode + const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xyzmin.x*dinv.x)*wq*invdtd.x/(amrex::Real)imode *(Complex(sx_new[i]*sz_new[k], 0._rt)*(xy_new - xy_mid) + Complex(sx_old[i]*sz_old[k], 0._rt)*(xy_mid - xy_old)); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); @@ -995,7 +872,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int i=dil; i<=depos_order+2-diu; i++) { Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdzk); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -1021,7 +898,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, } amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+k_new-1+k, 0, 0, 0), sdzk); } #endif @@ -1047,16 +924,16 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. */ template -void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, - const amrex::ParticleReal * const yp_n, - const amrex::ParticleReal * const zp_n, +void doChargeConservingDepositionShapeNImplicit ([[maybe_unused]]const amrex::ParticleReal * const xp_n, + [[maybe_unused]]const amrex::ParticleReal * const yp_n, + [[maybe_unused]]const amrex::ParticleReal * const zp_n, const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, [[maybe_unused]]const amrex::ParticleReal * const uxp_n, @@ -1071,49 +948,25 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex; -#if !defined(WARPX_DIM_RZ) - ignore_unused(n_rz_azimuthal_modes); -#endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - Real const invdtdx = 1.0_rt / (dt*dx[1]*dx[2]); - Real const invdtdy = 1.0_rt / (dt*dx[0]*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]*dx[1]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - Real const invdtdx = 1.0_rt / (dt*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[0]*dx[2]); -#elif defined(WARPX_DIM_1D_Z) - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[2]); +#if !defined(WARPX_DIM_3D) + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; #endif -#if defined(WARPX_DIM_RZ) - Complex const I = Complex{0._rt, 1._rt}; -#endif + amrex::XDim3 const invdtd = amrex::XDim3{(1.0_rt/dt)*dinv.y*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.y}; #if !defined(WARPX_DIM_1D_Z) Real constexpr one_third = 1.0_rt / 3.0_rt; @@ -1137,7 +990,6 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); #endif - // wqx, wqy wqz are particle current in each direction Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; @@ -1148,24 +1000,12 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #if !defined(WARPX_DIM_1D_Z) ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; -#else - ignore_unused(xp_n); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; -#else - ignore_unused(yp_n); #endif ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; -#if !defined(WARPX_DIM_1D_Z) - amrex::Real const wqx = wq*invdtdx; -#endif -#if defined(WARPX_DIM_3D) - amrex::Real const wqy = wq*invdtdy; -#endif - amrex::Real const wqz = wq*invdtdz; - // computes current and old position in grid units #if defined(WARPX_DIM_RZ) amrex::Real const xp_new = xp_np1; @@ -1177,51 +1017,33 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; - amrex::Real costheta_new, sintheta_new; - if (rp_new > 0._rt) { - costheta_new = xp_new/rp_new; - sintheta_new = yp_new/rp_new; - } else { - costheta_new = 1._rt; - sintheta_new = 0._rt; - } - amrex::Real costheta_mid, sintheta_mid; - if (rp_mid > 0._rt) { - costheta_mid = xp_mid/rp_mid; - sintheta_mid = yp_mid/rp_mid; - } else { - costheta_mid = 1._rt; - sintheta_mid = 0._rt; - } - amrex::Real costheta_old, sintheta_old; - if (rp_old > 0._rt) { - costheta_old = xp_old/rp_old; - sintheta_old = yp_old/rp_old; - } else { - costheta_old = 1._rt; - sintheta_old = 0._rt; - } + const amrex::Real costheta_mid = (rp_mid > 0._rt ? xp_mid/rp_mid : 1._rt); + const amrex::Real sintheta_mid = (rp_mid > 0._rt ? yp_mid/rp_mid : 0._rt); + const amrex::Real costheta_new = (rp_new > 0._rt ? xp_new/rp_new : 1._rt); + const amrex::Real sintheta_new = (rp_new > 0._rt ? yp_new/rp_new : 0._rt); + const amrex::Real costheta_old = (rp_old > 0._rt ? xp_old/rp_old : 1._rt); + const amrex::Real sintheta_old = (rp_old > 0._rt ? yp_old/rp_old : 0._rt); const Complex xy_new0 = Complex{costheta_new, sintheta_new}; const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; const Complex xy_old0 = Complex{costheta_old, sintheta_old}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; #else #if !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n[ip] - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n[ip] - xyzmin.x)*dinv.x; #endif #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n[ip] - ymin)*dyi; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n[ip] - xyzmin.y)*dinv.y; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n[ip] - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n[ip] - xyzmin.z)*dinv.z; #if defined(WARPX_DIM_RZ) amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; @@ -1232,6 +1054,12 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con amrex::Real const vy = uyp_nph[ip]*gaminv; #endif + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + const Compute_shape_factor< depos_order > compute_shape_factor; + const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + // Shape factor arrays // Note that there are extra values above and below // to possibly hold the factor for the old particle @@ -1240,30 +1068,17 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #if !defined(WARPX_DIM_1D_Z) double sx_new[depos_order + 3] = {0.}; double sx_old[depos_order + 3] = {0.}; -#endif -#if defined(WARPX_DIM_3D) - // Keep these double to avoid bug in single precision - double sy_new[depos_order + 3] = {0.}; - double sy_old[depos_order + 3] = {0.}; -#endif - // Keep these double to avoid bug in single precision - double sz_new[depos_order + 3] = {0.}; - double sz_old[depos_order + 3] = {0.}; - - // --- Compute shape factors - // Compute shape factors for position as they are now and at old positions - // [ijk]_new: leftmost grid point that the particle touches - const Compute_shape_factor< depos_order > compute_shape_factor; - const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; - -#if !defined(WARPX_DIM_1D_Z) const int i_new = compute_shape_factor(sx_new+1, x_new); const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); #endif #if defined(WARPX_DIM_3D) + double sy_new[depos_order + 3] = {0.}; + double sy_old[depos_order + 3] = {0.}; const int j_new = compute_shape_factor(sy_new+1, y_new); const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); #endif + double sz_new[depos_order + 3] = {0.}; + double sz_old[depos_order + 3] = {0.}; const int k_new = compute_shape_factor(sz_new+1, z_new); const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); @@ -1288,7 +1103,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int j=djl; j<=depos_order+2-dju; j++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*( + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*( one_third*(sy_new[j]*sz_new[k] + sy_old[j]*sz_old[k]) +one_sixth*(sy_new[j]*sz_old[k] + sy_old[j]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdxi); @@ -1299,7 +1114,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdyj = 0._rt; for (int j=djl; j<=depos_order+1-dju; j++) { - sdyj += wqy*(sy_old[j] - sy_new[j])*( + sdyj += wq*invdtd.y*(sy_old[j] - sy_new[j])*( one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdyj); @@ -1310,7 +1125,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*( + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*( one_third*(sx_new[i]*sy_new[j] + sx_old[i]*sy_old[j]) +one_sixth*(sx_new[i]*sy_old[j] + sx_old[i]*sy_new[j])); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdzk); @@ -1323,7 +1138,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int k=dkl; k<=depos_order+2-dku; k++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdxi); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -1344,6 +1159,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdyj); #if defined(WARPX_DIM_RZ) + Complex const I = Complex{0._rt, 1._rt}; Complex xy_new = xy_new0; Complex xy_mid = xy_mid0; Complex xy_old = xy_old0; @@ -1351,7 +1167,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { // The factor 2 comes from the normalization of the modes // The minus sign comes from the different convention with respect to Davidson et al. - const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(amrex::Real)imode + const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xyzmin.x*dinv.x)*wq*invdtd.x/(amrex::Real)imode *(Complex(sx_new[i]*sz_new[k], 0._rt)*(xy_new - xy_mid) + Complex(sx_old[i]*sz_old[k], 0._rt)*(xy_mid - xy_old)); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); @@ -1366,7 +1182,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int i=dil; i<=depos_order+2-diu; i++) { Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdzk); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -1392,7 +1208,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con } amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+k_new-1+k, 0, 0, 0), sdzk); } #endif @@ -1420,16 +1236,16 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. */ template -void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, - const amrex::ParticleReal * const yp_n, - const amrex::ParticleReal * const zp_n, +void doVillasenorDepositionShapeNImplicit ([[maybe_unused]]const amrex::ParticleReal * const xp_n, + [[maybe_unused]]const amrex::ParticleReal * const yp_n, + [[maybe_unused]]const amrex::ParticleReal * const zp_n, const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, [[maybe_unused]]const amrex::ParticleReal * const uxp_n, @@ -1444,42 +1260,21 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex; -#if !defined(WARPX_DIM_RZ) - ignore_unused(n_rz_azimuthal_modes); -#endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - Real const invvol = 1.0_rt / (dx[0]*dx[1]*dx[2]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - Real const invvol = 1.0_rt / (dx[0]*dx[2]); -#elif defined(WARPX_DIM_1D_Z) - Real const invvol = 1.0_rt / (dx[2]); -#endif + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; -#if !defined(WARPX_DIM_1D_Z) +#if (AMREX_SPACEDIM > 1) Real constexpr one_third = 1.0_rt / 3.0_rt; Real constexpr one_sixth = 1.0_rt / 6.0_rt; #endif @@ -1502,7 +1297,6 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); #endif - // wqx, wqy wqz are particle current in each direction Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; @@ -1513,13 +1307,9 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ #if !defined(WARPX_DIM_1D_Z) ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; -#else - ignore_unused(xp_n); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; -#else - ignore_unused(yp_n); #endif ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; @@ -1545,14 +1335,14 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; amrex::Real const vx = (rp_new - rp_old)/dt; amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; #elif defined(WARPX_DIM_XZ) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n[ip] - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n[ip] - xyzmin.x)*dinv.x; amrex::Real const vx = (xp_np1 - xp_n[ip])/dt; amrex::Real const vy = uyp_nph[ip]*gaminv; #elif defined(WARPX_DIM_1D_Z) @@ -1560,17 +1350,17 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ amrex::Real const vy = uyp_nph[ip]*gaminv; #elif defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n[ip] - xmin)*dxi; - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n[ip] - ymin)*dyi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n[ip] - xyzmin.x)*dinv.x; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n[ip] - xyzmin.y)*dinv.y; amrex::Real const vx = (xp_np1 - xp_n[ip])/dt; amrex::Real const vy = (yp_np1 - yp_n[ip])/dt; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n[ip] - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n[ip] - xyzmin.z)*dinv.z; amrex::Real const vz = (zp_np1 - zp_n[ip])/dt; // Define velocity kernals to deposit @@ -2057,7 +1847,7 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param[in] dx 3D cell size + * \param[in] dinv 3D cell size inverse * \param[in] xyzmin 3D lower bounds of physical domain * \param[in] lo Dimension-agnostic lower bounds of index domain * \param[in] q Species charge @@ -2076,57 +1866,37 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, long np_to_deposit, amrex::Real dt, amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]]int n_rz_azimuthal_modes) { using namespace amrex::literals; #if defined(WARPX_DIM_RZ) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q); WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in RZ geometry"); #endif #if defined(WARPX_DIM_1D_Z) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); - WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in cartesian 1D geometry"); + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q); + WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in 1D geometry"); #endif #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) - amrex::ignore_unused(n_rz_azimuthal_modes); // If ion_lev is a null pointer, then do_ionization=0, else do_ionization=1 const bool do_ionization = ion_lev; - // Inverse cell volume in each direction - const amrex::Real dxi = 1._rt / dx[0]; - const amrex::Real dzi = 1._rt / dx[2]; -#if defined(WARPX_DIM_3D) - const amrex::Real dyi = 1._rt / dx[1]; -#endif - // Inverse of time step const amrex::Real invdt = 1._rt / dt; - // Total inverse cell volume -#if defined(WARPX_DIM_XZ) - const amrex::Real invvol = dxi * dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real invvol = dxi * dyi * dzi; -#endif - - // Lower bound of physical domain in each direction - const amrex::Real xmin = xyzmin[0]; - const amrex::Real zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; // Allocate temporary arrays #if defined(WARPX_DIM_3D) @@ -2172,23 +1942,18 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, yp += relative_time * vy; zp += relative_time * vz; - // Particle current densities -#if defined(WARPX_DIM_XZ) - const amrex::Real wqy = wq * vy * invvol; -#endif - // Current and old particle positions in grid units // Keep these double to avoid bug in single precision. - double const x_new = (xp - xmin + 0.5_rt*dt*vx) * dxi; - double const x_old = (xp - xmin - 0.5_rt*dt*vx) * dxi; + double const x_new = (xp - xyzmin.x + 0.5_rt*dt*vx) * dinv.x; + double const x_old = (xp - xyzmin.x - 0.5_rt*dt*vx) * dinv.x; #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision. - double const y_new = (yp - ymin + 0.5_rt*dt*vy) * dyi; - double const y_old = (yp - ymin - 0.5_rt*dt*vy) * dyi; + double const y_new = (yp - xyzmin.y + 0.5_rt*dt*vy) * dinv.y; + double const y_old = (yp - xyzmin.y - 0.5_rt*dt*vy) * dinv.y; #endif // Keep these double to avoid bug in single precision. - double const z_new = (zp - zmin + 0.5_rt*dt*vz) * dzi; - double const z_old = (zp - zmin - 0.5_rt*dt*vz) * dzi; + double const z_new = (zp - xyzmin.z + 0.5_rt*dt*vz) * dinv.z; + double const z_old = (zp - xyzmin.z - 0.5_rt*dt*vz) * dinv.z; // Shape factor arrays for current and old positions (nodal) // Keep these double to avoid bug in single precision. @@ -2235,6 +2000,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, // Deposit current into Dx_arr, Dy_arr and Dz_arr #if defined(WARPX_DIM_XZ) + const amrex::Real wqy = wq * vy * invvol; for (int k=0; k<=depos_order; k++) { for (int i=0; i<=depos_order; i++) { diff --git a/Source/Particles/Deposition/SharedDepositionUtils.H b/Source/Particles/Deposition/SharedDepositionUtils.H index 1e2294be3a2..e6deb51b5df 100644 --- a/Source/Particles/Deposition/SharedDepositionUtils.H +++ b/Source/Particles/Deposition/SharedDepositionUtils.H @@ -74,8 +74,8 @@ void depositComponent (const GetParticlePosition& GetPosition, amrex::Array4 const& j_buff, amrex::IntVect const j_type, const amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 dinv, + const amrex::XDim3 xyzmin, const amrex::Dim3 lo, const amrex::Real q, const int n_rz_azimuthal_modes, @@ -91,26 +91,8 @@ void depositComponent (const GetParticlePosition& GetPosition, // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; const amrex::Real clightsq = 1.0_rt/PhysConst::c/PhysConst::c; @@ -171,9 +153,9 @@ void depositComponent (const GetParticlePosition& GetPosition, // Get particle position after 1/2 push back in position #if defined(WARPX_DIM_RZ) // Keep these double to avoid bug in single precision - const double xmid = (rpmid - xmin)*dxi; + const double xmid = (rpmid - xyzmin.x)*dinv.x; #else - const double xmid = ((xp - xmin) + relative_time*vx)*dxi; + const double xmid = ((xp - xyzmin.x) + relative_time*vx)*dinv.x; #endif // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current // sx_j[xyz] shape factor along x for the centering of each current @@ -203,7 +185,7 @@ void depositComponent (const GetParticlePosition& GetPosition, #if defined(WARPX_DIM_3D) // y direction // Keep these double to avoid bug in single precision - const double ymid = ((yp - ymin) + relative_time*vy)*dyi; + const double ymid = ((yp - xyzmin.y) + relative_time*vy)*dinv.y; double sy_node[depos_order + 1] = {0.}; double sy_cell[depos_order + 1] = {0.}; int k_node = 0; @@ -224,7 +206,7 @@ void depositComponent (const GetParticlePosition& GetPosition, // z direction // Keep these double to avoid bug in single precision - const double zmid = ((zp - zmin) + relative_time*vz)*dzi; + const double zmid = ((zp - xyzmin.z) + relative_time*vz)*dinv.z; double sz_node[depos_order + 1] = {0.}; double sz_cell[depos_order + 1] = {0.}; int l_node = 0; diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index cee7ec07eb5..6f98c18959a 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -62,8 +62,8 @@ struct IonizationFilterFunc amrex::IndexType m_by_type; amrex::IndexType m_bz_type; - amrex::GpuArray m_dx_arr; - amrex::GpuArray m_xyzmin_arr; + amrex::XDim3 m_dinv; + amrex::XDim3 m_xyzmin; bool m_galerkin_interpolation; int m_nox; @@ -117,7 +117,7 @@ struct IonizationFilterFunc doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, - m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_dinv, m_xyzmin, m_lo, m_n_rz_azimuthal_modes, m_nox, m_galerkin_interpolation); m_get_externalEB(i, ex, ey, ez, bx, by, bz); diff --git a/Source/Particles/ElementaryProcess/Ionization.cpp b/Source/Particles/ElementaryProcess/Ionization.cpp index b7b91e4d4e3..0568e302eec 100644 --- a/Source/Particles/ElementaryProcess/Ionization.cpp +++ b/Source/Particles/ElementaryProcess/Ionization.cpp @@ -76,11 +76,10 @@ IonizationFilterFunc::IonizationFilterFunc (const WarpXParIter& a_pti, int lev, box.grow(ngEB); const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - m_dx_arr = {dx[0], dx[1], dx[2]}; + m_dinv = amrex::XDim3{1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + m_xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_lo = amrex::lbound(box); } diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index d550c5f03ee..180fdf0fb35 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -144,7 +144,7 @@ public: doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, - m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_dinv, m_xyzmin, m_lo, m_n_rz_azimuthal_modes, m_nox, m_galerkin_interpolation); //Despite the names of the variables, positrons and electrons @@ -198,8 +198,8 @@ private: amrex::IndexType m_by_type; amrex::IndexType m_bz_type; - amrex::GpuArray m_dx_arr; - amrex::GpuArray m_xyzmin_arr; + amrex::XDim3 m_dinv; + amrex::XDim3 m_xyzmin; bool m_galerkin_interpolation; int m_nox; diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp b/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp index 2b380d454f4..82546e43ef4 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp @@ -64,11 +64,10 @@ PairGenerationTransformFunc (BreitWheelerGeneratePairs const generate_functor, box.grow(ngEB); const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - m_dx_arr = {dx[0], dx[1], dx[2]}; + m_dinv = amrex::XDim3{1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + m_xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_lo = amrex::lbound(box); } diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index dd5f57a8f24..514526374bd 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -156,7 +156,7 @@ public: doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, - m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_dinv, m_xyzmin, m_lo, m_n_rz_azimuthal_modes, m_nox, m_galerkin_interpolation); auto& ux = src.m_rdata[PIdx::ux][i_src]; @@ -209,8 +209,8 @@ private: amrex::IndexType m_by_type; amrex::IndexType m_bz_type; - amrex::GpuArray m_dx_arr; - amrex::GpuArray m_xyzmin_arr; + amrex::XDim3 m_dinv; + amrex::XDim3 m_xyzmin; bool m_galerkin_interpolation; int m_nox; diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp b/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp index 077a4659ce5..688c3c0184d 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp @@ -67,13 +67,10 @@ PhotonEmissionTransformFunc (QuantumSynchrotronGetOpticalDepth opt_depth_functor box.grow(ngEB); const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - m_dx_arr = {dx[0], dx[1], dx[2]}; + m_dinv = amrex::XDim3{1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - - + m_xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_lo = amrex::lbound(box); } diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index b5bd4376ba1..4b4590b8642 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -28,16 +28,16 @@ * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. * \param ex_type,ey_type,ez_type IndexType of the electric field * \param bx_type,by_type,bz_type IndexType of the magnetic field - * \param dx 3D cell spacing - * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void doGatherShapeN (const amrex::ParticleReal xp, - const amrex::ParticleReal yp, - const amrex::ParticleReal zp, +void doGatherShapeN ([[maybe_unused]] const amrex::ParticleReal xp, + [[maybe_unused]] const amrex::ParticleReal yp, + [[maybe_unused]] const amrex::ParticleReal zp, amrex::ParticleReal& Exp, amrex::ParticleReal& Eyp, amrex::ParticleReal& Ezp, @@ -56,41 +56,13 @@ void doGatherShapeN (const amrex::ParticleReal xp, const amrex::IndexType bx_type, const amrex::IndexType by_type, const amrex::IndexType bz_type, - const amrex::GpuArray& dx, - const amrex::GpuArray& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex; -#if defined(WARPX_DIM_XZ) - amrex::ignore_unused(yp); -#endif - -#if defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(xp,yp); -#endif - -#ifndef WARPX_DIM_RZ - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif - -#if (AMREX_SPACEDIM >= 2) - const amrex::Real dxi = 1.0_rt/dx[0]; -#endif - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_3D) - const amrex::Real dyi = 1.0_rt/dx[1]; -#endif - -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; - constexpr int zdir = WARPX_ZINDEX; constexpr int NODE = amrex::IndexType::NODE; constexpr int CELL = amrex::IndexType::CELL; @@ -105,9 +77,9 @@ void doGatherShapeN (const amrex::ParticleReal xp, // Get particle position #ifdef WARPX_DIM_RZ const amrex::Real rp = std::sqrt(xp*xp + yp*yp); - const amrex::Real x = (rp - xmin)*dxi; + const amrex::Real x = (rp - xyzmin.x)*dinv.x; #else - const amrex::Real x = (xp-xmin)*dxi; + const amrex::Real x = (xp-xyzmin.x)*dinv.x; #endif // j_[eb][xyz] leftmost grid point in x that the particle touches for the centering of each current @@ -151,7 +123,7 @@ void doGatherShapeN (const amrex::ParticleReal xp, #if defined(WARPX_DIM_3D) // y direction - const amrex::Real y = (yp-ymin)*dyi; + const amrex::Real y = (yp-xyzmin.y)*dinv.y; amrex::Real sy_node[depos_order + 1]; amrex::Real sy_cell[depos_order + 1]; amrex::Real sy_node_v[depos_order + 1 - galerkin_interpolation]; @@ -187,7 +159,7 @@ void doGatherShapeN (const amrex::ParticleReal xp, #endif // z direction - const amrex::Real z = (zp-zmin)*dzi; + const amrex::Real z = (zp-xyzmin.z)*dinv.z; amrex::Real sz_node[depos_order + 1]; amrex::Real sz_cell[depos_order + 1]; amrex::Real sz_node_v[depos_order + 1 - galerkin_interpolation]; @@ -464,10 +436,10 @@ void doGatherShapeN (const amrex::ParticleReal xp, * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. * \param Ex_type,Ey_type,Ez_type IndexType of the electric field * \param Bx_type,By_type,Bz_type IndexType of the magnetic field - * \param dx 3D cell spacing - * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. - * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -496,8 +468,8 @@ void doGatherShapeNEsirkepovStencilImplicit ( [[maybe_unused]] const amrex::IndexType Bx_type, [[maybe_unused]] const amrex::IndexType By_type, [[maybe_unused]] const amrex::IndexType Bz_type, - const amrex::GpuArray& dx, - const amrex::GpuArray& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes) { @@ -506,19 +478,6 @@ void doGatherShapeNEsirkepovStencilImplicit ( ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; - #if !defined(WARPX_DIM_1D_Z) Real constexpr one_third = 1.0_rt / 3.0_rt; Real constexpr one_sixth = 1.0_rt / 6.0_rt; @@ -553,23 +512,23 @@ void doGatherShapeNEsirkepovStencilImplicit ( } const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; #else #if !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n - xyzmin.x)*dinv.x; #endif #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n - ymin)*dyi; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n - xyzmin.y)*dinv.y; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n - xyzmin.z)*dinv.z; // Shape factor arrays // Note that there are extra values above and below @@ -894,8 +853,8 @@ void doGatherShapeNEsirkepovStencilImplicit ( * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. * \param Ex_type,Ey_type,Ez_type IndexType of the electric field * \param Bx_type,By_type,Bz_type IndexType of the magnetic field - * \param dx 3D cell spacing - * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ @@ -926,8 +885,8 @@ void doGatherPicnicShapeN ( [[maybe_unused]] const amrex::IndexType Bx_type, [[maybe_unused]] const amrex::IndexType By_type, [[maybe_unused]] const amrex::IndexType Bz_type, - const amrex::GpuArray& dx, - const amrex::GpuArray& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes) { @@ -936,19 +895,6 @@ void doGatherPicnicShapeN ( ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; - #if !defined(WARPX_DIM_1D_Z) const ParticleReal xp_np1 = 2._prt*xp_nph - xp_n; #endif @@ -983,25 +929,25 @@ void doGatherPicnicShapeN ( } const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; - double const x_bar = (rp_mid - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; + double const x_bar = (rp_mid - xyzmin.x)*dinv.x; #elif !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n - xmin)*dxi; - double const x_bar = (xp_nph - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n - xyzmin.x)*dinv.x; + double const x_bar = (xp_nph - xyzmin.x)*dinv.x; #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n - ymin)*dyi; - double const y_bar = (yp_nph - ymin)*dyi; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n - xyzmin.y)*dinv.y; + double const y_bar = (yp_nph - xyzmin.y)*dinv.y; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n - zmin)*dzi; - double const z_bar = (zp_nph - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n - xyzmin.z)*dinv.z; + double const z_bar = (zp_nph - xyzmin.z)*dinv.z; // 1) Determine the number of segments. // 2) Loop over segments and gather electric field. @@ -1567,8 +1513,8 @@ void doGatherPicnicShapeN ( * \param exfab,eyfab,ezfab Array4 of the electric field, either full array or tile. * \param bxfab,byfab,bzfab Array4 of the magnetic field, either full array or tile. * \param np_to_gather Number of particles for which field is gathered. - * \param dx 3D cell size - * \param xyzmin Physical lower bounds of domain. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ @@ -1585,15 +1531,12 @@ void doGatherShapeN(const GetParticlePosition& getPosition, amrex::FArrayBox const * const byfab, amrex::FArrayBox const * const bzfab, const long np_to_gather, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const int n_rz_azimuthal_modes) { - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -1622,7 +1565,7 @@ void doGatherShapeN(const GetParticlePosition& getPosition, xp, yp, zp, Exp[ip], Eyp[ip], Ezp[ip], Bxp[ip], Byp[ip], Bzp[ip], ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } ); } @@ -1637,8 +1580,8 @@ void doGatherShapeN(const GetParticlePosition& getPosition, * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. * \param ex_type,ey_type,ez_type IndexType of the electric field * \param bx_type,by_type,bz_type IndexType of the magnetic field - * \param dx_arr 3D cell spacing - * \param xyzmin_arr Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry * \param nox order of the particle shape function @@ -1666,8 +1609,8 @@ void doGatherShapeN (const amrex::ParticleReal xp, const amrex::IndexType bx_type, const amrex::IndexType by_type, const amrex::IndexType bz_type, - const amrex::GpuArray& dx_arr, - const amrex::GpuArray& xyzmin_arr, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes, const int nox, @@ -1678,44 +1621,44 @@ void doGatherShapeN (const amrex::ParticleReal xp, doGatherShapeN<1,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeN<2,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeN<3,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeN<4,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } else { if (nox == 1) { doGatherShapeN<1,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeN<2,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeN<3,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeN<4,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } } @@ -1732,8 +1675,8 @@ void doGatherShapeN (const amrex::ParticleReal xp, * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. * \param ex_type,ey_type,ez_type IndexType of the electric field * \param bx_type,by_type,bz_type IndexType of the magnetic field - * \param dx_arr 3D cell spacing - * \param xyzmin_arr Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry * \param nox order of the particle shape function @@ -1766,8 +1709,8 @@ void doGatherShapeNImplicit ( const amrex::IndexType bx_type, const amrex::IndexType by_type, const amrex::IndexType bz_type, - const amrex::GpuArray& dx_arr, - const amrex::GpuArray& xyzmin_arr, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes, const int nox, @@ -1779,25 +1722,25 @@ void doGatherShapeNImplicit ( Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeNEsirkepovStencilImplicit<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeNEsirkepovStencilImplicit<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeNEsirkepovStencilImplicit<4>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } else if (depos_type==3) { // CurrentDepositionAlgo::Villasenor @@ -1806,25 +1749,25 @@ void doGatherShapeNImplicit ( Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherPicnicShapeN<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherPicnicShapeN<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherPicnicShapeN<4>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } else if (depos_type==1) { // CurrentDepositionAlgo::Direct @@ -1832,22 +1775,22 @@ void doGatherShapeNImplicit ( doGatherShapeN<1,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeN<2,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeN<3,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeN<4,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } } diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index aa9f04be224..1f15d5210f5 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -93,8 +93,8 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, int lev, int gather_lev, amrex::Real dt, ScaleFields /*scaleFields*/, DtType a_dt_type) { - // Get cell size on gather_lev - const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + // Get inverse cell size on gather_lev + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(gather_lev,0)); // Get box from which field is gathered. // If not gathering from the finest level, the box is coarsened. @@ -142,7 +142,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); const Dim3 lo = lbound(box); @@ -150,9 +150,6 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -201,7 +198,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, doGatherShapeN(x, y, z, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e2be4f948ca..89ae435882b 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -2558,7 +2558,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, if (do_not_push) { return; } - const std::array& dx = WarpX::CellSize(std::max(lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev,0)); #ifdef AMREX_USE_OMP #pragma omp parallel @@ -2590,7 +2590,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const Dim3 lo = lbound(box); @@ -2598,9 +2598,6 @@ PhysicalParticleContainer::PushP (int lev, Real dt, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab.array(); amrex::Array4 const& ey_arr = eyfab.array(); amrex::Array4 const& ez_arr = ezfab.array(); @@ -2657,7 +2654,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); } @@ -2755,7 +2752,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, if (np_to_push == 0) { return; } // Get cell size on gather_lev - const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(gather_lev,0)); // Get box from which field is gathered. // If not gathering from the finest level, the box is coarsened. @@ -2783,7 +2780,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); const Dim3 lo = lbound(box); @@ -2791,9 +2788,6 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -2908,7 +2902,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); } @@ -3008,7 +3002,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, if (np_to_push == 0) { return; } // Get cell size on gather_lev - const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(gather_lev,0)); // Get box from which field is gathered. // If not gathering from the finest level, the box is coarsened. @@ -3035,7 +3029,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); const Dim3 lo = lbound(box); @@ -3043,9 +3037,6 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -3157,9 +3148,9 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, amrex::ParticleReal dxp, dxp_save; amrex::ParticleReal dyp, dyp_save; amrex::ParticleReal dzp, dzp_save; - auto idxg2 = static_cast(1._rt/(dx[0]*dx[0])); - auto idyg2 = static_cast(1._rt/(dx[1]*dx[1])); - auto idzg2 = static_cast(1._rt/(dx[2]*dx[2])); + auto idxg2 = static_cast(dinv.x*dinv.x); + auto idyg2 = static_cast(dinv.y*dinv.y); + auto idzg2 = static_cast(dinv.z*dinv.z); amrex::ParticleReal step_norm = 1._prt; for (int iter=0; iter& dx = WarpX::CellSize(std::max(lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev,0)); #ifdef AMREX_USE_OMP #pragma omp parallel @@ -369,7 +369,6 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const Dim3 lo = lbound(box); @@ -377,8 +376,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); amrex::Array4 const& ex_arr = exfab.array(); amrex::Array4 const& ey_arr = eyfab.array(); @@ -445,7 +443,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 4800d9e209f..bdce18b7b2b 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -396,7 +396,8 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, amrex::numParticlesOutOfRange(pti, range) == 0, "Particles shape does not fit within tile (CPU) or guard cells (GPU) used for current deposition"); - const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(depos_lev,0)); + const amrex::ParticleReal q = this->charge; WARPX_PROFILE_VAR_NS("WarpXParticleContainer::DepositCurrent::Sorting", blp_sort); @@ -467,7 +468,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, // Note that this includes guard cells since it is after tilebox.ngrow const Dim3 lo = lbound(tilebox); // Take into account Galilean shift - const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, 0.5_rt*dt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, depos_lev, 0.5_rt*dt); if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { @@ -544,28 +545,28 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doDepositionSharedShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 2){ doDepositionSharedShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 3){ doDepositionSharedShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 4){ doDepositionSharedShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } @@ -580,25 +581,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doEsirkepovDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { @@ -625,7 +626,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doChargeConservingDepositionShapeNImplicit<2>( @@ -633,7 +634,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doChargeConservingDepositionShapeNImplicit<3>( @@ -641,7 +642,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doChargeConservingDepositionShapeNImplicit<4>( @@ -649,7 +650,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } @@ -678,7 +679,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVillasenorDepositionShapeNImplicit<2>( @@ -686,7 +687,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVillasenorDepositionShapeNImplicit<3>( @@ -694,7 +695,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVillasenorDepositionShapeNImplicit<4>( @@ -702,7 +703,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } @@ -717,25 +718,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doVayDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVayDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVayDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVayDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else { // Direct deposition @@ -744,25 +745,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { @@ -775,7 +776,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeNImplicit<2>( @@ -783,7 +784,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeNImplicit<3>( @@ -791,7 +792,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeNImplicit<4>( @@ -799,7 +800,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } @@ -931,7 +932,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, "Particles shape does not fit within tile (CPU) or guard cells (GPU) used for charge deposition"); amrex::ignore_unused(range); // In case the assertion isn't compiled - const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); const Real q = this->charge; WARPX_PROFILE_VAR_NS("WarpXParticleContainer::DepositCharge::Sorting", blp_sort); @@ -981,12 +981,13 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // Take into account Galilean shift const Real dt = warpx.getdt(lev); const amrex::Real time_shift_delta = (icomp == 0 ? 0.0_rt : dt); - const std::array& xyzmin = WarpX::LowerCorner( - tilebox, depos_lev, time_shift_delta); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); // Indices of the lower bound const Dim3 lo = lbound(tilebox); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(depos_lev,0)); + // HACK - sort particles by bin here. WARPX_PROFILE_VAR_START(blp_sort); amrex::DenseBins bins; @@ -1088,25 +1089,25 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, if (WarpX::nox == 1){ doChargeDepositionSharedShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 2){ doChargeDepositionSharedShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 3){ doChargeDepositionSharedShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 4){ doChargeDepositionSharedShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); @@ -1125,7 +1126,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // note: this is smaller than rho->nGrowVect() for PSATD const amrex::IntVect& ng_rho = warpx.get_ng_depos_rho(); - const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); amrex::IntVect ref_ratio; if (lev == depos_lev) { ref_ratio = IntVect(AMREX_D_DECL(1, 1, 1 )); @@ -1150,7 +1150,8 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // Take into account Galilean shift const amrex::Real dt = warpx.getdt(lev); const amrex::Real time_shift_delta = (icomp == 0 ? 0.0_rt : dt); - const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(depos_lev,0)); AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noy); AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noz); @@ -1158,7 +1159,7 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, ablastr::particles::deposit_charge( pti, wp, this->charge, ion_lev, rho, local_rho[thread_num], - WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, + WarpX::noz, dinv, xyzmin, WarpX::n_rz_azimuthal_modes, ng_rho, depos_lev, ref_ratio, offset, np_to_deposit, icomp, nc); diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index d2691685c53..2ef4ee55d6e 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -232,8 +232,8 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) for(amrex::MFIter mfi(mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.tilebox(); // Get box lower and upper physical z bound, and dz - const amrex::Real zmin_box = WarpX::LowerCorner(bx, lev, 0._rt)[2]; - const amrex::Real zmax_box = WarpX::UpperCorner(bx, lev, 0._rt)[2]; + const amrex::Real zmin_box = WarpX::LowerCorner(bx, lev, 0._rt).z; + const amrex::Real zmax_box = WarpX::UpperCorner(bx, lev, 0._rt).z; const amrex::Real dz = WarpX::CellSize(lev)[2]; // Get box lower index in the z direction #if defined(WARPX_DIM_3D) diff --git a/Source/WarpX.H b/Source/WarpX.H index a0a1379d9e2..768f2d486dc 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -895,6 +895,7 @@ public: amrex::Vector& output_geom ) const; static std::array CellSize (int lev); + static amrex::XDim3 InvCellSize (int lev); static amrex::RealBox getRealBox(const amrex::Box& bx, int lev); /** @@ -905,7 +906,7 @@ public: * (when v_galilean is not zero) * \return An array of the position coordinates */ - static std::array LowerCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); + static amrex::XDim3 LowerCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); /** * \brief Return the upper corner of the box in real units. * \param bx The box @@ -914,7 +915,7 @@ public: * (when v_galilean is not zero) * \return An array of the position coordinates */ - static std::array UpperCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); + static amrex::XDim3 UpperCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); static amrex::IntVect RefRatio (int lev); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index ef81aef4482..d1f9eb16f47 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2846,6 +2846,13 @@ WarpX::CellSize (int lev) #endif } +amrex::XDim3 +WarpX::InvCellSize (int lev) +{ + std::array dx = WarpX::CellSize(lev); + return {1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; +} + amrex::RealBox WarpX::getRealBox(const Box& bx, int lev) { @@ -2854,13 +2861,13 @@ WarpX::getRealBox(const Box& bx, int lev) return( grid_box ); } -std::array +amrex::XDim3 WarpX::LowerCorner(const Box& bx, const int lev, const amrex::Real time_shift_delta) { auto & warpx = GetInstance(); const RealBox grid_box = getRealBox( bx, lev ); - const Real* xyzmin = grid_box.lo(); + const Real* grid_min = grid_box.lo(); const amrex::Real cur_time = warpx.gett_new(lev); const amrex::Real time_shift = (cur_time + time_shift_delta - warpx.time_of_last_gal_shift); @@ -2869,23 +2876,23 @@ WarpX::LowerCorner(const Box& bx, const int lev, const amrex::Real time_shift_de warpx.m_v_galilean[2]*time_shift }; #if defined(WARPX_DIM_3D) - return { xyzmin[0] + galilean_shift[0], xyzmin[1] + galilean_shift[1], xyzmin[2] + galilean_shift[2] }; + return { grid_min[0] + galilean_shift[0], grid_min[1] + galilean_shift[1], grid_min[2] + galilean_shift[2] }; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - return { xyzmin[0] + galilean_shift[0], std::numeric_limits::lowest(), xyzmin[1] + galilean_shift[2] }; + return { grid_min[0] + galilean_shift[0], std::numeric_limits::lowest(), grid_min[1] + galilean_shift[2] }; #elif defined(WARPX_DIM_1D_Z) - return { std::numeric_limits::lowest(), std::numeric_limits::lowest(), xyzmin[0] + galilean_shift[2] }; + return { std::numeric_limits::lowest(), std::numeric_limits::lowest(), grid_min[0] + galilean_shift[2] }; #endif } -std::array +amrex::XDim3 WarpX::UpperCorner(const Box& bx, const int lev, const amrex::Real time_shift_delta) { auto & warpx = GetInstance(); const RealBox grid_box = getRealBox( bx, lev ); - const Real* xyzmax = grid_box.hi(); + const Real* grid_max = grid_box.hi(); const amrex::Real cur_time = warpx.gett_new(lev); const amrex::Real time_shift = (cur_time + time_shift_delta - warpx.time_of_last_gal_shift); @@ -2894,13 +2901,13 @@ WarpX::UpperCorner(const Box& bx, const int lev, const amrex::Real time_shift_de warpx.m_v_galilean[2]*time_shift }; #if defined(WARPX_DIM_3D) - return { xyzmax[0] + galilean_shift[0], xyzmax[1] + galilean_shift[1], xyzmax[2] + galilean_shift[2] }; + return { grid_max[0] + galilean_shift[0], grid_max[1] + galilean_shift[1], grid_max[2] + galilean_shift[2] }; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - return { xyzmax[0] + galilean_shift[0], std::numeric_limits::max(), xyzmax[1] + galilean_shift[1] }; + return { grid_max[0] + galilean_shift[0], std::numeric_limits::max(), grid_max[1] + galilean_shift[1] }; #elif defined(WARPX_DIM_1D_Z) - return { std::numeric_limits::max(), std::numeric_limits::max(), xyzmax[0] + galilean_shift[0] }; + return { std::numeric_limits::max(), std::numeric_limits::max(), grid_max[0] + galilean_shift[0] }; #endif } diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index ff3741a7a43..75e3bca170d 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -34,7 +34,7 @@ namespace ablastr::particles * \param rho MultiFab of the charge density * \param local_rho temporary FArrayBox for deposition with OpenMP * \param particle_shape shape factor in each direction - * \param dx cell spacing at level lev + * \param dinv cell spacing inverses at level lev * \param xyzmin lo corner of the current tile in physical coordinates. * \param n_rz_azimuthal_modes number of azimuthal modes in use, irrelevant outside RZ geometry (default: 0) * \param num_rho_deposition_guards number of ghost cells to use for rho (default: rho.nGrowVect()) @@ -54,8 +54,8 @@ deposit_charge (typename T_PC::ParIterType& pti, amrex::MultiFab* rho, amrex::FArrayBox& local_rho, int const particle_shape, - std::array const & dx, - std::array const & xyzmin, + const amrex::XDim3 dinv, + const amrex::XDim3 xyzmin, int const n_rz_azimuthal_modes = 0, std::optional num_rho_deposition_guards = std::nullopt, std::optional depos_lev = std::nullopt, @@ -175,19 +175,19 @@ deposit_charge (typename T_PC::ParIterType& pti, if (nox == 1){ doChargeDepositionShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } else if (nox == 2){ doChargeDepositionShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } else if (nox == 3){ doChargeDepositionShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } else if (nox == 4){ doChargeDepositionShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd); From 8a94987a33c329431c9db848e7dcf35ce2b7a224 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 19 Jun 2024 00:20:20 +0200 Subject: [PATCH 127/190] Fix typo (#5001) --- Docs/source/usage/parameters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index a3ca703d15e..ae1353159dd 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -88,7 +88,7 @@ Overall simulation parameters * ``theta_implicit_em``: Use a fully implicit electromagnetic solver with a time-biasing parameter theta bound between 0.5 and 1.0. Exact energy conservation is achieved using theta = 0.5. Maximal damping of high-k modes is obtained using theta = 1.0. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed (PS) JNFK. The algorithm itself is numerical stable for large time steps. That is, it does not require time steps that resolve the plasma period or the CFL condition for light waves. However, the practicality of using a large time step depends on the nonlinear solver. Note that the Picard solver is for demonstration only. It is inefficient and will most like not converge when - :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relavent physics. + :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relevant physics. The Picard method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. The PS-JFNK method is described in `Angus et al., An implicit particle code with exact energy and charge conservation for electromagnetic studies of dense plasmas `__ . (The version implemented in WarpX is an updated version that includes the relativistic gamma factor for the particles.) Also see `Chen et al., An energy- and charge-conserving, implicit, electrostatic particle-in-cell algorithm. `__ . Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. From 34794926c373add1f8839efb1ce5919b6641874f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Jun 2024 17:15:38 -0700 Subject: [PATCH 128/190] Doc: Update WarpX Ack Text (#5004) Generalize text post ECP. (Various funding sources from many people.) --- Docs/source/acknowledge_us.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/acknowledge_us.rst b/Docs/source/acknowledge_us.rst index 4c7a2d8137c..bffbb80a797 100644 --- a/Docs/source/acknowledge_us.rst +++ b/Docs/source/acknowledge_us.rst @@ -23,14 +23,14 @@ Please add the following sentence to your publications, it helps contributors ke **Plain text:** - This research used the open-source particle-in-cell code WarpX https://github.com/ECP-WarpX/WarpX, primarily funded by the US DOE Exascale Computing Project. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors. + This research used the open-source particle-in-cell code WarpX https://github.com/ECP-WarpX/WarpX. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors. **LaTeX:** .. code-block:: latex \usepackage{hyperref} - This research used the open-source particle-in-cell code WarpX \url{https://github.com/ECP-WarpX/WarpX}, primarily funded by the US DOE Exascale Computing Project. + This research used the open-source particle-in-cell code WarpX \url{https://github.com/ECP-WarpX/WarpX}. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors. From bdda0bd0c68cb4687c85acc76e4aa2921dcc8b1a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Jun 2024 18:41:11 -0700 Subject: [PATCH 129/190] Doc: Resonant excitation of plasma waves in a plasma channel (#4998) * Doc: Resonant excitation of plasma waves in a plasma channel WarpX used in the paper by A. J. Ross et al., PRR 6 (2024) * Nguyen, B et a. IPAC24 * Sort by Pub Year * Update PASC24 in other sections --- Docs/source/acknowledge_us.rst | 4 ++-- Docs/source/highlights.rst | 31 +++++++++++++++++++++---------- Docs/source/refs.bib | 25 ++++++++++++++----------- 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/Docs/source/acknowledge_us.rst b/Docs/source/acknowledge_us.rst index bffbb80a797..b01ebf00a39 100644 --- a/Docs/source/acknowledge_us.rst +++ b/Docs/source/acknowledge_us.rst @@ -55,8 +55,8 @@ If your project uses a specific algorithm or component, please consider citing t - Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint __` + Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024. + `DOI:10.1145/3659914.3659937 `__ - Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index b243f62cd97..7baec74d606 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -14,7 +14,17 @@ Plasma-Based Acceleration Scientific works in laser-plasma and beam-plasma acceleration. -#. Peng, H. and Huang, T. W. and Jiang, K. and Li, R. and Wu, C. N. and Yu, M. Y. and Riconda, C. and Weber, S. and Zhou, C. T. and Ruan, S. C. +#. Ross AJ, Chappell J, van de Wetering JJ, Cowley J, Archer E, Bourgeois N, Corner L, Emerson DR, Feder L, Gu XJ, Jakobsson O, Jones H, Picksley A, Reid L, Wang W, Walczak R, Hooker SM. + **Resonant excitation of plasma waves in a plasma channel**. + Phys. Rev. Research **6**, L022001, 2024 + `DOI:10.1103/PhysRevResearch.6.L022001 `__ + +#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024. + `DOI:10.1145/3659914.3659937 `__ + +#. Peng H, Huang TW, Jiang K, Li R, Wu CN, Yu MY, Riconda C, Weber S, Zhou CT, Ruan SC. **Coherent Subcycle Optical Shock from a Superluminal Plasma Wake**. Phys. Rev. Lett. **131**, 145003, 2023 `DOI:10.1103/PhysRevLett.131.145003 `__ @@ -24,11 +34,6 @@ Scientific works in laser-plasma and beam-plasma acceleration. Phys. Rev. Research **5**, 033112, 2023 `DOI:10.1103/PhysRevResearch.5.033112 `__ -#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. - **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint `__ - #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. @@ -103,14 +108,20 @@ Scientific works in particle and beam modeling. #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint `__ + Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024. + `DOI:10.1145/3659914.3659937 `__ + +#. Nguyen B, Formenti A, Lehe R, Vay J-L, Gessner S, and Fedeli L. + **Comparison of WarpX and GUINEA-PIG for electron positron collisions**. + 15th International Particle Accelerator Conference (IPAC'24), WEPC84, 2024. + `preprint `__, + `DOI:10.18429/JACoW-IPAC2024-WEPC84 `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. - 14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023. + 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. `preprint `__, - `DOI:10.18429/JACoW-IPAC-23-WEPA101 `__ + `DOI:10.18429/JACoW-IPAC2023-WEPA101 `__ #. Tan W H, Piot P, Myers A, Zhang W, Rheaume T, Jambunathan R, Huebl A, Lehe R, Vay J-L. **Simulation studies of drive-beam instability in a dielectric wakefield accelerator**. diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 84b34b17351..130e0ce4da7 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -205,17 +205,20 @@ @article{Roedel2010 year = {2010} } -@misc{SandbergPASC24, -address = {Zuerich, Switzerland}, -author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Andrew Myers and Ji Qiang and Jean-Luc Vay and Axel Huebl}, -booktitle = {Proc. of PASC24}, -note = {accepted}, -series = {PASC'24 - Platform for Advanced Scientific Computing}, -title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, -venue = {Zuerich, Switzerland}, -year = {2024}, -doi = {10.48550/arXiv.2402.17248}, -url = {https://arxiv.org/abs/2402.17248} +@inproceedings{SandbergPASC24, +author = {Sandberg, Ryan and Lehe, Remi and Mitchell, Chad and Garten, Marco and Myers, Andrew and Qiang, Ji and Vay, Jean-Luc and Huebl, Axel}, +title = {{Synthesizing Particle-In-Cell Simulations through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, +series = {PASC '24}, +booktitle = {Proceedings of the Platform for Advanced Scientific Computing Conference}, +location = {Zuerich, Switzerland}, +publisher = {Association for Computing Machinery}, +address = {New York, NY, USA}, +articleno = {23}, +numpages = {11}, +isbn = {9798400706394}, +doi = {10.1145/3659914.3659937}, +note = {{PASC24 Best Paper Award}}, +year = {2024} } @inproceedings{SandbergIPAC23, From 8fe9c24e332acd6af8d2f68f572af66f6666636b Mon Sep 17 00:00:00 2001 From: Revathi Jambunathan <41089244+RevathiJambunathan@users.noreply.github.com> Date: Tue, 18 Jun 2024 19:57:19 -0700 Subject: [PATCH 130/190] Warning instead of error for Background MCC (#4991) * add warning instead of error * Apply suggestions from code review Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --------- Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> --- .../BackgroundMCC/BackgroundMCCCollision.cpp | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 1eb89ff4c0f..80ce13744fd 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -234,9 +234,12 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult // dt has to be small enough that a linear expansion of the collision // probability is sufficiently accurately, otherwise the MCC results // will be very heavily affected by small changes in the timestep - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(coll_n < 0.1_prt, - "dt is too large to ensure accurate MCC results" - ); + if (coll_n > 0.1_prt) { + ablastr::warn_manager::WMRecordWarning("BackgroundMCC Collisions", + "dt is too large to ensure accurate MCC results , coll_n: " + + std::to_string(coll_n) + " is > 0.1 and collision probability is = " + + std::to_string(m_total_collision_prob) + "\n"); + } if (ionization_flag) { // calculate maximum collision frequency for ionization @@ -246,9 +249,12 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult auto coll_n_ioniz = m_nu_max_ioniz * dt; m_total_collision_prob_ioniz = 1.0_prt - std::exp(-coll_n_ioniz); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(coll_n_ioniz < 0.1_prt, - "dt is too large to ensure accurate MCC results" - ); + if (coll_n_ioniz > 0.1_prt) { + ablastr::warn_manager::WMRecordWarning("BackgroundMCC Collisions", + "dt is too large to ensure accurate MCC ionization , coll_n_ionization: " + + std::to_string(coll_n_ioniz) + " is > 0.1 and ionization probability is = " + + std::to_string(m_total_collision_prob_ioniz) + "\n"); + } // if an ionization process is included the secondary species mass // is taken as the background mass From 6dd5e9261a15860ad46ff3e6dba4340e9fae9e2a Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Jun 2024 21:14:46 -0700 Subject: [PATCH 131/190] AMReX/pyAMReX/PICSAR: Weekly Update (#4995) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 3a03ea01c0c..71775879896 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.06 && cd - + cd ../amrex && git checkout --detach 1f038e767011e20100af6ab0db02c69cf7ebe55c && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 05ed74fe1b1..f1fb5f4b0db 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.06 +branch = 1f038e767011e20100af6ab0db02c69cf7ebe55c [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 41db9a15bdc..f00fb083cbc 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.06 +branch = 1f038e767011e20100af6ab0db02c69cf7ebe55c [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index eda45dc9f77..30fe67bc018 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.06" +set(WarpX_amrex_branch "1f038e767011e20100af6ab0db02c69cf7ebe55c" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 18cab89e347..1016158fc4a 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.06" +set(WarpX_pyamrex_branch "25e097f4ca10e0c5ad506041e56b7b2181d2f34a" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index f397d31048e..d590a710f4a 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.06 && cd - +cd amrex && git checkout --detach 1f038e767011e20100af6ab0db02c69cf7ebe55c && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 1077dbc00f4ca6be589dee8b13ba8bd41940fe01 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Jun 2024 23:57:54 -0700 Subject: [PATCH 132/190] Fix: Perlmutter heFFTe CPU (#5006) --- Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index a5a7c28b85e..e9ac393cd32 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -115,7 +115,6 @@ then cd $HOME/src/heffte git fetch --prune git checkout v2.4.0 - git pull cd - else git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte From 03dc9b76cfe17e7cbd1d66983ccfad41e0e8c810 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Thu, 20 Jun 2024 13:49:06 -0700 Subject: [PATCH 133/190] Fix ccache for macos (#5007) Need to ignore time macros. --- .github/workflows/macos.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 21d5ae04faf..00ac8f06b5d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -44,7 +44,7 @@ jobs: - name: CCache Cache uses: actions/cache@v4 with: - path: /Users/runner/Library/Caches/ccache + path: ~/Library/Caches/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | ccache-${{ github.workflow }}-${{ github.job }}-git- @@ -53,7 +53,7 @@ jobs: export CCACHE_COMPRESS=1 export CCACHE_COMPRESSLEVEL=10 export CCACHE_MAXSIZE=100M - export CCACHE_DEPEND=1 + export CCACHE_SLOPPINESS=time_macros ccache -z source py-venv/bin/activate @@ -76,6 +76,7 @@ jobs: cmake --build build_sp -j 3 cmake --build build_sp --target pip_install + du -hs ~/Library/Caches/ccache ccache -s - name: run pywarpx From 3e5112f13f5b6edf50f11cdb8dc46a599fdd7d1d Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 24 Jun 2024 10:41:40 -0700 Subject: [PATCH 134/190] BLAS++: v2024.05.31+ (#5012) Update our HankelTransform logic calling into BLAS++ by using the latest APIs for `blas::Queue`. Pass the active AMReX GPU stream. --- .../SpectralSolver/SpectralHankelTransform/HankelTransform.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp index 9f4cac71736..7fd419b54f8 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp @@ -36,7 +36,8 @@ HankelTransform::HankelTransform (int const hankel_order, // SYCL note: we need to double check AMReX device ID conventions and // BLAS++ device ID conventions are the same int const device_id = amrex::Gpu::Device::deviceId(); - m_queue = std::make_unique( device_id, 0 ); + blas::Queue::stream_t stream_id = amrex::Gpu::gpuStream(); + m_queue = std::make_unique( device_id, stream_id ); #endif amrex::Vector alphas; From 64ab258e93198b87114fb9c92d435809f79608a9 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 24 Jun 2024 16:05:44 -0700 Subject: [PATCH 135/190] Doc: Latest Commit of BLAS++/LAPACK++ (#5013) For BLAS++ and LAPACK++, switch the documented HPC dependency install logic from using the latest commit on `master` to the latest release `2024.05.31`. Using tags instead of living branches increases robustness and reproducibility with regards to future changes. --- .../adastra-cines/adastra_warpx.profile.example | 8 ++++---- .../machines/adastra-cines/install_dependencies.sh | 14 ++++++-------- .../cori-nersc/haswell_warpx.profile.example | 4 ++-- .../machines/cori-nersc/knl_warpx.profile.example | 4 ++-- .../crusher-olcf/crusher_warpx.profile.example | 8 ++++---- .../machines/crusher-olcf/install_dependencies.sh | 14 ++++++-------- .../frontier-olcf/frontier_warpx.profile.example | 8 ++++---- .../machines/frontier-olcf/install_dependencies.sh | 14 ++++++-------- .../greatlakes_v100_warpx.profile.example | 8 ++++---- .../greatlakes-umich/install_v100_dependencies.sh | 14 ++++++-------- .../hpc3-uci/hpc3_gpu_warpx.profile.example | 8 ++++---- .../machines/hpc3-uci/install_gpu_dependencies.sh | 14 ++++++-------- .../lassen-llnl/install_v100_dependencies.sh | 14 ++++++-------- .../lassen-llnl/lassen_v100_warpx.profile.example | 8 ++++---- .../lassen_v100_warpx_toss3.profile.example | 8 ++++---- .../lawrencium_warpx.profile.example | 4 ++-- .../leonardo-cineca/install_gpu_dependencies.sh | 14 ++++++-------- .../leonardo_gpu_warpx.profile.example | 8 ++++---- Tools/machines/lumi-csc/install_dependencies.sh | 14 ++++++-------- Tools/machines/lumi-csc/lumi_warpx.profile.example | 8 ++++---- .../perlmutter-nersc/install_cpu_dependencies.sh | 14 ++++++-------- .../perlmutter-nersc/install_gpu_dependencies.sh | 14 ++++++-------- .../perlmutter_cpu_warpx.profile.example | 8 ++++---- .../perlmutter_gpu_warpx.profile.example | 8 ++++---- .../polaris-alcf/install_gpu_dependencies.sh | 14 ++++++-------- .../polaris-alcf/polaris_gpu_warpx.profile.example | 8 ++++---- Tools/machines/quartz-llnl/install_dependencies.sh | 14 ++++++-------- .../quartz-llnl/quartz_warpx.profile.example | 8 ++++---- .../summit-olcf/install_gpu_dependencies.sh | 14 ++++++-------- .../summit-olcf/summit_warpx.profile.example | 8 ++++---- 30 files changed, 140 insertions(+), 166 deletions(-) diff --git a/Tools/machines/adastra-cines/adastra_warpx.profile.example b/Tools/machines/adastra-cines/adastra_warpx.profile.example index 0d55e869d6a..3cba4346421 100644 --- a/Tools/machines/adastra-cines/adastra_warpx.profile.example +++ b/Tools/machines/adastra-cines/adastra_warpx.profile.example @@ -11,10 +11,10 @@ module load CCE-GPU-3.0.0 module load amd-mixed/5.2.3 # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.83.0-mpi-python3 diff --git a/Tools/machines/adastra-cines/install_dependencies.sh b/Tools/machines/adastra-cines/install_dependencies.sh index b48bf144c2a..242f5fc664c 100755 --- a/Tools/machines/adastra-cines/install_dependencies.sh +++ b/Tools/machines/adastra-cines/install_dependencies.sh @@ -38,14 +38,13 @@ if [ -d $SHAREDHOMEDIR/src/blaspp ] then cd $SHAREDHOMEDIR/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $SHAREDHOMEDIR/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $SHAREDHOMEDIR/src/blaspp fi rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -CXX=$(which CC) cmake -S $SHAREDHOMEDIR/src/blaspp -B $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $SHAREDHOMEDIR/src/blaspp -B $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build --target install --parallel 16 rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build @@ -54,14 +53,13 @@ if [ -d $SHAREDHOMEDIR/src/lapackpp ] then cd $SHAREDHOMEDIR/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $SHAREDHOMEDIR/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $SHAREDHOMEDIR/src/lapackpp fi rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $SHAREDHOMEDIR/src/lapackpp -B $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $SHAREDHOMEDIR/src/lapackpp -B $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build --target install --parallel 16 rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build diff --git a/Tools/machines/cori-nersc/haswell_warpx.profile.example b/Tools/machines/cori-nersc/haswell_warpx.profile.example index b0ac225cc72..f636b1ab555 100644 --- a/Tools/machines/cori-nersc/haswell_warpx.profile.example +++ b/Tools/machines/cori-nersc/haswell_warpx.profile.example @@ -8,8 +8,8 @@ module load cray-python/3.9.7.1 export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH export CMAKE_PREFIX_PATH=$HOME/sw/haswell/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/haswell/adios2-2.7.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/haswell/blaspp-master-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/haswell/lapackpp-master-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/haswell/blaspp-2024.05.31-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/haswell/lapackpp-2024.05.31-install:$CMAKE_PREFIX_PATH if [ -d "$HOME/sw/haswell/venvs/haswell_warpx" ] then diff --git a/Tools/machines/cori-nersc/knl_warpx.profile.example b/Tools/machines/cori-nersc/knl_warpx.profile.example index 093c7f62783..fdd7e14d594 100644 --- a/Tools/machines/cori-nersc/knl_warpx.profile.example +++ b/Tools/machines/cori-nersc/knl_warpx.profile.example @@ -9,8 +9,8 @@ module load cray-python/3.9.7.1 export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH export CMAKE_PREFIX_PATH=$HOME/sw/knl/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/knl/adios2-2.7.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/knl/blaspp-master-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/knl/lapackpp-master-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/knl/blaspp-2024.05.31-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/knl/lapackpp-2024.05.31-install:$CMAKE_PREFIX_PATH if [ -d "$HOME/sw/knl/venvs/knl_warpx" ] then diff --git a/Tools/machines/crusher-olcf/crusher_warpx.profile.example b/Tools/machines/crusher-olcf/crusher_warpx.profile.example index ffc70a128e2..7b05546b3f3 100644 --- a/Tools/machines/crusher-olcf/crusher_warpx.profile.example +++ b/Tools/machines/crusher-olcf/crusher_warpx.profile.example @@ -21,10 +21,10 @@ module load ninja module load nano # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.79.0-cxx17 diff --git a/Tools/machines/crusher-olcf/install_dependencies.sh b/Tools/machines/crusher-olcf/install_dependencies.sh index ab5deb6b0a6..6e9f97eddae 100755 --- a/Tools/machines/crusher-olcf/install_dependencies.sh +++ b/Tools/machines/crusher-olcf/install_dependencies.sh @@ -49,14 +49,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-crusher-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-crusher-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-crusher-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-crusher-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-crusher-gpu-build @@ -65,14 +64,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-crusher-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-crusher-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-crusher-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-crusher-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-crusher-gpu-build diff --git a/Tools/machines/frontier-olcf/frontier_warpx.profile.example b/Tools/machines/frontier-olcf/frontier_warpx.profile.example index 2f86d047d00..50e12e99a38 100644 --- a/Tools/machines/frontier-olcf/frontier_warpx.profile.example +++ b/Tools/machines/frontier-olcf/frontier_warpx.profile.example @@ -20,10 +20,10 @@ module load ninja module load nano # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.79.0-cxx17 diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 98c30cfca8f..503ab4525e9 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -49,14 +49,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-frontier-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-frontier-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-frontier-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-frontier-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-frontier-gpu-build @@ -65,14 +64,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-frontier-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-frontier-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-frontier-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-frontier-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-frontier-gpu-build diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example index 98476eecfab..c08255e7962 100644 --- a/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example +++ b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example @@ -22,13 +22,13 @@ module load phdf5/1.12.1 SW_DIR="${HOME}/sw/greatlakes/v100" export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc2-2.14.4:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.0:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${SW_DIR}/c-blosc2-2.14.4/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.0/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=${SW_DIR}/adios2-2.10.0/bin:${PATH} diff --git a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh index 9361fd6c64b..30faec52421 100755 --- a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh +++ b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh @@ -80,14 +80,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-v100-build -cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-v100-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-v100-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-v100-build --target install --parallel 8 rm -rf ${build_dir}/blaspp-v100-build @@ -96,14 +95,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-v100-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-v100-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-v100-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-v100-build --target install --parallel 8 rm -rf ${build_dir}/lapackpp-v100-build diff --git a/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example b/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example index 9f41cc7e337..017613f9d60 100644 --- a/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example +++ b/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example @@ -19,13 +19,13 @@ module load OpenBLAS/0.3.21 module load hdf5/1.13.1/gcc.11.2.0-openmpi.4.1.2 export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh index 51d4bcfe2c5..b585c2702b6 100755 --- a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh +++ b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh @@ -79,14 +79,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/blaspp-pm-gpu-build @@ -95,14 +94,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/lapackpp-pm-gpu-build diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh index fe8285b7501..6368010ab7b 100755 --- a/Tools/machines/lassen-llnl/install_v100_dependencies.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies.sh @@ -82,13 +82,12 @@ if [ -d ${SRC_DIR}/blaspp ] then cd ${SRC_DIR}/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi -cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 # LAPACK++ (for PSATD+RZ) @@ -96,13 +95,12 @@ if [ -d ${SRC_DIR}/lapackpp ] then cd ${SRC_DIR}/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example index 4d4efbf917d..80755ece308 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example @@ -22,10 +22,10 @@ export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} export PATH=${SW_DIR}/adios2-2.8.3/bin:$PATH # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings module load python/3.8.2 diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example index abad909943c..6a7067f596e 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example @@ -22,10 +22,10 @@ export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings module load python/3.8.2 diff --git a/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example b/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example index 472d0785bb2..8db2b44b8a7 100644 --- a/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example +++ b/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example @@ -18,8 +18,8 @@ module load lapack/3.8.0-gcc export CMAKE_PREFIX_PATH=$HOME/sw/v100/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/v100/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/v100/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/v100/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/v100/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/v100/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export PATH=$HOME/sw/v100/adios2-2.8.3/bin:$PATH diff --git a/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh b/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh index bbaf0ab8464..4d89e30cd29 100644 --- a/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh +++ b/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh @@ -48,14 +48,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-gpu-build -CXX=$(which g++) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which g++) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-gpu-build @@ -65,14 +64,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-gpu-build diff --git a/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example b/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example index cffe565f9d7..dd8e79ffb37 100644 --- a/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example +++ b/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example @@ -16,13 +16,13 @@ module load boost/1.80.0--openmpi--4.1.4--gcc--11.3.0 module load openblas/0.3.21--gcc--11.3.0 export CMAKE_PREFIX_PATH=/leonardo/prod/spack/03/install/0.19/linux-rhel8-icelake/gcc-11.3.0/c-blosc-1.21.1-aifmix6v5lwxgt7rigwoebalrgbcnv26:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/adios2-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=/leonardo/prod/spack/03/install/0.19/linux-rhel8-icelake/gcc-11.3.0/c-blosc-1.21.1-aifmix6v5lwxgt7rigwoebalrgbcnv26/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$HOME/sw/adios2-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$HOME/sw/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$HOME/sw/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=$HOME/sw/adios2-master/bin:$PATH diff --git a/Tools/machines/lumi-csc/install_dependencies.sh b/Tools/machines/lumi-csc/install_dependencies.sh index 814e134bad4..2fd31b79bce 100755 --- a/Tools/machines/lumi-csc/install_dependencies.sh +++ b/Tools/machines/lumi-csc/install_dependencies.sh @@ -43,11 +43,10 @@ if [ -d ${SRC_DIR}/blaspp ] then cd ${SRC_DIR}/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi rm -rf ${build_dir}/blaspp-lumi-gpu-build CXX=$(which CC) \ @@ -56,7 +55,7 @@ cmake -S ${SRC_DIR}/blaspp \ -Duse_openmp=OFF \ -Dgpu_backend=hip \ -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-lumi-gpu-build --target install --parallel 16 rm -rf ${build_dir}/blaspp-lumi-gpu-build @@ -65,11 +64,10 @@ if [ -d ${SRC_DIR}/lapackpp ] then cd ${SRC_DIR}/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi rm -rf ${build_dir}/lapackpp-lumi-gpu-build CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" \ @@ -78,7 +76,7 @@ cmake -S ${SRC_DIR}/lapackpp \ -DCMAKE_CXX_STANDARD=17 \ -Dbuild_tests=OFF \ -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-lumi-gpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-lumi-gpu-build diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example index 33aff1946c2..13fb6b1d81e 100644 --- a/Tools/machines/lumi-csc/lumi_warpx.profile.example +++ b/Tools/machines/lumi-csc/lumi_warpx.profile.example @@ -11,10 +11,10 @@ module load nano # optional: for PSATD in RZ geometry support SW_DIR="${HOME}/sw/lumi/gpu" -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load Boost/1.82.0-cpeCray-23.09 diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index e9ac393cd32..437300b8303 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -82,14 +82,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-cpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-pm-cpu-build --target install --parallel 16 rm -rf ${build_dir}/blaspp-pm-cpu-build @@ -98,14 +97,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-cpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-cpu-build diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index 125c63104be..ac95ad9f3a0 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -82,14 +82,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-pm-gpu-build --target install --parallel 16 rm -rf ${build_dir}/blaspp-pm-gpu-build @@ -98,14 +97,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-gpu-build diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 3fea9b3aa39..94d598abf5b 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -17,14 +17,14 @@ export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/ module load cray-hdf5-parallel/1.12.2.9 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 9e1465d6c02..da1d55964d1 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -21,14 +21,14 @@ export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/ module load cray-hdf5-parallel/1.12.2.9 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/polaris-alcf/install_gpu_dependencies.sh b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh index dbc66b2ffff..18f94e6fd3b 100755 --- a/Tools/machines/polaris-alcf/install_gpu_dependencies.sh +++ b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh @@ -65,14 +65,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-pm-gpu-build @@ -81,14 +80,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-pm-gpu-build diff --git a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example index d7635188141..333434c1b97 100644 --- a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example +++ b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example @@ -19,13 +19,13 @@ module load cmake/3.23.2 module load cray-hdf5-parallel/1.12.2.3 export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/quartz-llnl/install_dependencies.sh b/Tools/machines/quartz-llnl/install_dependencies.sh index 2920b00663b..cfb01769384 100755 --- a/Tools/machines/quartz-llnl/install_dependencies.sh +++ b/Tools/machines/quartz-llnl/install_dependencies.sh @@ -67,13 +67,12 @@ if [ -d ${HOME}/src/blaspp ] then cd ${HOME}/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${HOME}/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${HOME}/src/blaspp fi -cmake -S ${HOME}/src/blaspp -B ${build_dir}/blaspp-quartz-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S ${HOME}/src/blaspp -B ${build_dir}/blaspp-quartz-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-quartz-build --target install --parallel 6 # LAPACK++ (for PSATD+RZ) @@ -81,13 +80,12 @@ if [ -d ${HOME}/src/lapackpp ] then cd ${HOME}/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${HOME}/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${HOME}/src/lapackpp fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${HOME}/src/lapackpp -B ${build_dir}/lapackpp-quartz-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${HOME}/src/lapackpp -B ${build_dir}/lapackpp-quartz-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-quartz-build --target install --parallel 6 diff --git a/Tools/machines/quartz-llnl/quartz_warpx.profile.example b/Tools/machines/quartz-llnl/quartz_warpx.profile.example index a3646bfd557..f296a0738ff 100644 --- a/Tools/machines/quartz-llnl/quartz_warpx.profile.example +++ b/Tools/machines/quartz-llnl/quartz_warpx.profile.example @@ -21,10 +21,10 @@ export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings module load python/3.9.12 diff --git a/Tools/machines/summit-olcf/install_gpu_dependencies.sh b/Tools/machines/summit-olcf/install_gpu_dependencies.sh index 433b7ab0580..042e34538d0 100755 --- a/Tools/machines/summit-olcf/install_gpu_dependencies.sh +++ b/Tools/machines/summit-olcf/install_gpu_dependencies.sh @@ -63,13 +63,12 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi -cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-summit-build -Duse_openmp=ON -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-summit-build -Duse_openmp=ON -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-summit-build --target install --parallel 10 # LAPACK++ (for PSATD+RZ) @@ -77,13 +76,12 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi -cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-summit-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-summit-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-summit-build --target install --parallel 10 # remove build temporary directory diff --git a/Tools/machines/summit-olcf/summit_warpx.profile.example b/Tools/machines/summit-olcf/summit_warpx.profile.example index e41521b5815..d70ee9eb42a 100644 --- a/Tools/machines/summit-olcf/summit_warpx.profile.example +++ b/Tools/machines/summit-olcf/summit_warpx.profile.example @@ -18,10 +18,10 @@ module load ccache module load ninja # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.76.0 From 1565175d304917101be165877c26c887251e1b81 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 24 Jun 2024 18:26:06 -0700 Subject: [PATCH 136/190] AMReX/pyAMReX/PICSAR: Weekly Update (#5014) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 71775879896..7e1cf7ae63f 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 1f038e767011e20100af6ab0db02c69cf7ebe55c && cd - + cd ../amrex && git checkout --detach 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index f1fb5f4b0db..e4e5d7b8678 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 1f038e767011e20100af6ab0db02c69cf7ebe55c +branch = 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index f00fb083cbc..a18f39b89c1 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 1f038e767011e20100af6ab0db02c69cf7ebe55c +branch = 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 30fe67bc018..632b9edd4f4 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "1f038e767011e20100af6ab0db02c69cf7ebe55c" +set(WarpX_amrex_branch "259db7cfb99e7d1d2ab4bec9b1587fdf624a138a" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 1016158fc4a..90aee6cb930 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "25e097f4ca10e0c5ad506041e56b7b2181d2f34a" +set(WarpX_pyamrex_branch "fd7126cb907be7e3c8570930b79209db5cd7c492" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index d590a710f4a..6ac22466ef0 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 1f038e767011e20100af6ab0db02c69cf7ebe55c && cd - +cd amrex && git checkout --detach 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 77c04659700bad186aacd326469d3cb8cf7541e9 Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Fri, 28 Jun 2024 23:42:40 -0700 Subject: [PATCH 137/190] obvious bug fix in ElasticCollisionPerez (#5021) * obvious bug fix in ElasticCollisionPerez * updating benchmark json files for collisionISO and collisionXYZ --- .../Checksum/benchmarks_json/collisionISO.json | 14 +++++++------- .../Checksum/benchmarks_json/collisionXYZ.json | 4 ++-- .../Coulomb/ElasticCollisionPerez.H | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json index 96a2f5cbdac..5146837be6f 100644 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ b/Regression/Checksum/benchmarks_json/collisionISO.json @@ -11,12 +11,12 @@ "jz": 0.0 }, "electron": { - "particle_momentum_x": 3.5856774369040155e-19, - "particle_momentum_y": 3.5832349891760786e-19, - "particle_momentum_z": 3.5757013442445486e-19, - "particle_position_x": 1.0241562532152744, - "particle_position_y": 1.0238555904782611, - "particle_position_z": 1.0240053504978093, + "particle_momentum_x": 3.579130200773753e-19, + "particle_momentum_y": 3.5788119408700804e-19, + "particle_momentum_z": 3.584163522201744e-19, + "particle_position_x": 1.024188253213835, + "particle_position_y": 1.0238795904737117, + "particle_position_z": 1.02399735048655, "particle_weight": 714240000000.0 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index c87bbd98e42..984dd1d2761 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,7 +6,7 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 358778.6506903592, - "T_ion": 341562.3085776466 + "T_electron": 351375.5409141184, + "T_ion": 349005.3985888339 } } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index e407ae1603b..af4e6ba38c9 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -58,7 +58,7 @@ void ElasticCollisionPerez ( const T_index NI1 = I1e - I1s; const T_index NI2 = I2e - I2s; const T_index max_N = amrex::max(NI1,NI2); - const T_index min_N = amrex::max(NI1,NI2); + const T_index min_N = amrex::min(NI1,NI2); T_PR * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; T_PR * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; From dcdf4f7d31313850925650a1878fe965ba09611f Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 2 Jul 2024 18:59:33 +0200 Subject: [PATCH 138/190] Document how to define EB with STL file (#5023) --- Docs/source/usage/parameters.rst | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index ae1353159dd..afa570cca02 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -581,11 +581,26 @@ Additional PML parameters Embedded Boundary Conditions ---------------------------- -* ``warpx.eb_implicit_function`` (`string`) - A function of `x`, `y`, `z` that defines the surface of the embedded - boundary. That surface lies where the function value is 0 ; - the physics simulation area is where the function value is negative ; - the interior of the embeddded boundary is where the function value is positive. +In WarpX, the embedded boundary can be defined in either of two ways: + + - **From an analytical function:** + In that case, you will need to set the following parameter in the input file. + + * ``warpx.eb_implicit_function`` (`string`) + A function of `x`, `y`, `z` that defines the surface of the embedded + boundary. That surface lies where the function value is 0 ; + the physics simulation area is where the function value is negative ; + the interior of the embeddded boundary is where the function value is positive. + + - **From an STL file:** + In that case, you will need to set the following parameters in the input file. + + * ``eb2.stl_file`` (`string`) + The path to an STL file. In addition, you also need to set ``eb2.geom_type = stl``, + in order for the file to be read by WarpX. + +Whether the embedded boundary is defined with an analytical function or an STL file, you can +additionally define the electric potential at the embedded boundary with an analytical function: * ``warpx.eb_potential(x,y,z,t)`` (`string`) Gives the value of the electric potential at the surface of the embedded boundary, From 2d845dcf967ae0a593b88bb4d12911d402725981 Mon Sep 17 00:00:00 2001 From: David Grote Date: Tue, 2 Jul 2024 09:59:52 -0700 Subject: [PATCH 139/190] Add random seed to Examples/Tests/collision/inputs_3d (#5022) * Add random seed to Examples/Tests/collision/inputs_3d * Update benchmark values * Update benchmark again --- Examples/Tests/collision/inputs_3d | 1 + Regression/Checksum/benchmarks_json/collisionXYZ.json | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Examples/Tests/collision/inputs_3d b/Examples/Tests/collision/inputs_3d index ed413ba2776..11d6b150dd2 100644 --- a/Examples/Tests/collision/inputs_3d +++ b/Examples/Tests/collision/inputs_3d @@ -22,6 +22,7 @@ boundary.field_hi = periodic periodic periodic warpx.serialize_initial_conditions = 1 warpx.verbose = 1 warpx.const_dt = 1.224744871e-07 +warpx.random_seed = 2034958209 # Do not evolve the E and B fields algo.maxwell_solver = none diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 984dd1d2761..7436302a99e 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,7 +6,7 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 351375.5409141184, - "T_ion": 349005.3985888339 + "T_electron": 351570.64632548566, + "T_ion": 350085.3998917431 } } From ca34cbc10e70ed8f25cfe2f0f4566fd45f7e06d7 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 3 Jul 2024 15:50:37 +0200 Subject: [PATCH 140/190] Simplify VelocityProperties.cpp (#5016) * simplify VelocityProperties.cpp * fix issues --- Source/Initialization/VelocityProperties.cpp | 23 ++++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/Source/Initialization/VelocityProperties.cpp b/Source/Initialization/VelocityProperties.cpp index 14b48c79524..b6986970e31 100644 --- a/Source/Initialization/VelocityProperties.cpp +++ b/Source/Initialization/VelocityProperties.cpp @@ -18,26 +18,25 @@ VelocityProperties::VelocityProperties (const amrex::ParmParse& pp, std::string std::string vel_dir_s = "x"; utils::parser::query(pp, source_name, "bulk_vel_dir", vel_dir_s); - if(vel_dir_s[0] == '-'){ - m_sign_dir = -1; - } - else { - m_sign_dir = 1; + + if(vel_dir_s.empty()){ + WARPX_ABORT_WITH_MESSAGE("'.bulk_vel_dir input ' can't be empty."); } - if ((vel_dir_s == "x" || vel_dir_s[1] == 'x') || - (vel_dir_s == "X" || vel_dir_s[1] == 'X')){ + m_sign_dir = (vel_dir_s[0] == '-') ? -1 : 1; + + const auto dir = std::tolower(vel_dir_s.back()); + + if (dir == 'x'){ m_dir = 0; } - else if ((vel_dir_s == "y" || vel_dir_s[1] == 'y') || - (vel_dir_s == "Y" || vel_dir_s[1] == 'Y')){ + else if (dir == 'y'){ m_dir = 1; } - else if ((vel_dir_s == "z" || vel_dir_s[1] == 'z') || - (vel_dir_s == "Z" || vel_dir_s[1] == 'Z')) { + else if (dir == 'z'){ m_dir = 2; } - else { + else{ WARPX_ABORT_WITH_MESSAGE( "Cannot interpret .bulk_vel_dir input '" + vel_dir_s + "'. Please enter +/- x, y, or z with no whitespace between the sign and"+ From fbe9d6aa9036fa214d90e578f39e8c5cab4522fa Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Wed, 3 Jul 2024 18:47:22 +0200 Subject: [PATCH 141/190] Fix gatherParticlesFromEmbeddedBoundaries (#5011) * touch particle tiles before OMP loop * touch tiles before OMP loop --- Source/Particles/ParticleBoundaryBuffer.cpp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 784c27816da..25068b2e65c 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -390,7 +390,15 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC buffer[i].AddRealComp("ny", false); buffer[i].AddRealComp("nz", false); } + auto& species_buffer = buffer[i]; + for (int lev = 0; lev < pc.numLevels(); ++lev){ + for(PIter pti(pc, lev); pti.isValid(); ++pti){ + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } + } + for (int lev = 0; lev < pc.numLevels(); ++lev) { const auto& plevel = pc.GetParticles(lev); @@ -474,7 +482,15 @@ void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries ( buffer[i].AddRealComp("nz", false); } + auto& species_buffer = buffer[i]; + for (int lev = 0; lev < pc.numLevels(); ++lev){ + for(PIter pti(pc, lev); pti.isValid(); ++pti){ + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } + } + for (int lev = 0; lev < pc.numLevels(); ++lev) { const auto& plevel = pc.GetParticles(lev); From a1980cf989c94354e2194d9b6d832c83093c7ff0 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 3 Jul 2024 23:03:07 +0200 Subject: [PATCH 142/190] Update default behavior for gathering with direct deposition (#5024) * Update default behavior for gathering with direct deposition * Modify condition under which to use galerkin interpolation * Update condition * Update benchmarks --------- Co-authored-by: Edoardo Zoni --- Docs/source/usage/parameters.rst | 2 +- .../Langmuir_multi_2d_MR_psatd.json | 44 +++++------ .../Langmuir_multi_2d_psatd.json | 28 +++---- .../Langmuir_multi_2d_psatd_multiJ.json | 32 ++++---- .../Langmuir_multi_psatd_div_cleaning.json | 46 ++++++------ .../Langmuir_multi_psatd_multiJ.json | 42 +++++------ .../benchmarks_json/Python_Langmuir.json | 16 ++-- .../benchmarks_json/Python_Langmuir_2d.json | 14 ++-- .../benchmarks_json/Python_gaussian_beam.json | 48 ++++++------ .../benchmarks_json/restart_psatd.json | 74 +++++++++---------- .../restart_psatd_time_avg.json | 66 ++++++++--------- Source/WarpX.cpp | 12 ++- 12 files changed, 216 insertions(+), 208 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index afa570cca02..4dc888df3d0 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2446,7 +2446,7 @@ Grid types (collocated, staggered, hybrid) For example, :math:`E_z` is gathered using ``algo.particle_shape`` along :math:`(x,y)` and ``algo.particle_shape - 1`` along :math:`z`. See equations (21)-(23) of :cite:t:`param-Godfrey2013` and associated references for details. - Default: ``interpolation.galerkin_scheme = 0`` with collocated grids and/or momentum-conserving field gathering, ``interpolation.galerkin_scheme = 1`` otherwise. + Default: ``interpolation.galerkin_scheme = 0`` with collocated grids, or momentum-conserving field gathering, or when ``algo.current_deposition = direct`` ; ``interpolation.galerkin_scheme = 1`` otherwise. .. warning:: diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json index 378b7691898..07c990b3997 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json @@ -1,40 +1,40 @@ { - "electrons": { - "particle_momentum_x": 4.2387000553111037e-20, - "particle_momentum_y": 0.0, - "particle_momentum_z": 4.238700039223283e-20, - "particle_position_x": 0.6553599659707846, - "particle_position_y": 0.6553599659699976, - "particle_weight": 3200000000000000.5 - }, "lev=0": { "Bx": 0.0, - "By": 46.86760020196344, + "By": 46.65494014556924, "Bz": 0.0, - "Ex": 7589330186947.288, + "Ex": 7596470155558.35, "Ey": 0.0, - "Ez": 7589330198843.742, - "jx": 7283102663077210.0, + "Ez": 7596470167219.471, + "jx": 7277974695617658.0, "jy": 0.0, - "jz": 7283102630279985.0 + "jz": 7277974664068728.0 }, "lev=1": { "Bx": 0.0, - "By": 369.2209081024592, + "By": 371.2497742366917, "Bz": 0.0, - "Ex": 7593801203430.788, + "Ex": 7596619292118.433, "Ey": 0.0, - "Ez": 7593800988021.711, - "jx": 6597471633118175.0, + "Ez": 7596619076137.7295, + "jx": 6592788899529852.0, "jy": 0.0, - "jz": 6597471447306984.0 + "jz": 6592788734672843.0 + }, + "electrons": { + "particle_momentum_x": 4.2314133085678375e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 4.2314132920088513e-20, + "particle_position_x": 0.6553599899254421, + "particle_position_y": 0.6553599899246253, + "particle_weight": 3200000000000000.5 }, "positrons": { - "particle_momentum_x": 4.238670783273887e-20, + "particle_momentum_x": 4.2314444192249906e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 4.238670762294589e-20, - "particle_position_x": 0.6553596917214832, - "particle_position_y": 0.6553596917221793, + "particle_momentum_z": 4.231444401040395e-20, + "particle_position_x": 0.655359770013864, + "particle_position_y": 0.6553597700144888, "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json index 85a09296e38..c59a09cc616 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json @@ -1,27 +1,27 @@ { - "electrons": { - "particle_momentum_x": 5.663705661977914e-20, - "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705661977915e-20, - "particle_position_x": 0.65536, - "particle_position_y": 0.6553599999999999, - "particle_weight": 3200000000000000.5 - }, "lev=0": { - "Ex": 3771082811318.298, + "Ex": 3751590727260.198, "Ey": 0.0, - "Ez": 3771082811318.301, - "jx": 1.009365330460902e+16, + "Ez": 3751590727260.1885, + "jx": 1.0100623838390416e+16, "jy": 0.0, - "jz": 1.009365330460902e+16, + "jz": 1.0100623838390416e+16, "part_per_cell": 131072.0 }, "positrons": { - "particle_momentum_x": 5.663705661977916e-20, + "particle_momentum_x": 5.668407775012392e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705661977916e-20, + "particle_momentum_z": 5.668407775012392e-20, "particle_position_x": 0.65536, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 + }, + "electrons": { + "particle_momentum_x": 5.668407775012391e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.668407775012391e-20, + "particle_position_x": 0.65536, + "particle_position_y": 0.6553600000000002, + "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json index c342d9dcaeb..ddf519913bd 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json @@ -1,29 +1,29 @@ { - "electrons": { - "particle_momentum_x": 5.663705657675969e-20, - "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705657675969e-20, - "particle_position_x": 0.65536, - "particle_position_y": 0.65536, - "particle_weight": 3200000000000000.5 - }, "lev=0": { "Bx": 0.0, - "By": 3.4892704618136277, + "By": 5.710236645367475, "Bz": 0.0, - "Ex": 3771082786646.7104, + "Ex": 3751590785149.1333, "Ey": 0.0, - "Ez": 3771082786646.702, - "jx": 1.0093631772735916e+16, + "Ez": 3751590785149.124, + "jx": 1.0100602372860838e+16, "jy": 0.0, - "jz": 1.0093631772735912e+16 + "jz": 1.0100602372860836e+16 }, "positrons": { - "particle_momentum_x": 5.663705657675971e-20, + "particle_momentum_x": 5.668407759026453e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705657675969e-20, - "particle_position_x": 0.65536, + "particle_momentum_z": 5.668407759026454e-20, + "particle_position_x": 0.6553599999999999, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 + }, + "electrons": { + "particle_momentum_x": 5.668407759026452e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.668407759026452e-20, + "particle_position_x": 0.65536, + "particle_position_y": 0.6553600000000002, + "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json index c9339aeb1aa..7098b83dbac 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json @@ -1,30 +1,30 @@ { - "electrons": { - "particle_momentum_x": 9.520485222844096e-20, - "particle_position_x": 2.6214400000233367, - "particle_position_y": 2.6214400000233367, - "particle_position_z": 2.6214400000000007, - "particle_weight": 128000000000.00002 - }, "lev=0": { - "Bx": 78.90261993791012, - "By": 78.90261993794903, - "Bz": 78.9041393998435, - "Ex": 84580428018688.97, - "Ey": 84580428018688.73, - "Ez": 84580441923246.2, - "F": 685.6927382678912, - "divE": 7.94184911020468e+19, - "jx": 6.007457695701311e+16, - "jy": 6.007457695701312e+16, - "jz": 6.007453636158103e+16, + "Bx": 86.03228226677659, + "By": 86.03228226679013, + "Bz": 86.033798606458, + "Ex": 84659309211473.42, + "Ey": 84659309211473.42, + "Ez": 84659323064827.52, + "F": 700.2020613746595, + "divE": 7.94984970149747e+19, + "jx": 5.999100369888243e+16, + "jy": 5.999100369888247e+16, + "jz": 5.999096310379822e+16, "part_per_cell": 524288.0, - "rho": 705198439.2596256 + "rho": 705877712.4251474 }, "positrons": { - "particle_momentum_z": 9.520479438980041e-20, - "particle_position_x": 2.6214400000233367, - "particle_position_y": 2.6214400000233358, + "particle_momentum_z": 9.508931687286131e-20, + "particle_position_x": 2.621440000034161, + "particle_position_y": 2.6214400000341618, "particle_position_z": 2.62144 + }, + "electrons": { + "particle_momentum_x": 9.508937471997448e-20, + "particle_position_x": 2.6214400000341604, + "particle_position_y": 2.621440000034161, + "particle_position_z": 2.6214399999999993, + "particle_weight": 128000000000.00002 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json index 487d4a89828..e1785dcce4d 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json @@ -1,28 +1,28 @@ { - "electrons": { - "particle_momentum_x": 9.633869745818886e-20, - "particle_position_x": 2.621440000001177, - "particle_position_y": 2.6214400000011784, - "particle_position_z": 2.6214400000000007, - "particle_weight": 128000000000.00002 - }, "lev=0": { - "Bx": 80.96035538655111, - "By": 80.96035538657691, - "Bz": 80.96271445263956, - "Ex": 84777489275096.88, - "Ey": 84777489275096.88, - "Ez": 84777485856239.4, - "jx": 6.08447015360442e+16, - "jy": 6.084470153604425e+16, - "jz": 6.084470085554113e+16, + "Bx": 88.73949682623977, + "By": 88.73949682627456, + "Bz": 88.74185838578241, + "Ex": 84851140116479.64, + "Ey": 84851140116479.62, + "Ez": 84851136696893.72, + "jx": 6.075773528102969e+16, + "jy": 6.0757735281029736e+16, + "jz": 6.075773460167053e+16, "part_per_cell": 524288.0, - "rho": 703546536.8089281 + "rho": 704298346.5173542 }, "positrons": { - "particle_momentum_z": 9.63386961193585e-20, - "particle_position_x": 2.621440000001177, - "particle_position_y": 2.621440000001179, - "particle_position_z": 2.6214400000000007 + "particle_momentum_z": 9.621748455552108e-20, + "particle_position_x": 2.621440000001309, + "particle_position_y": 2.6214400000013076, + "particle_position_z": 2.6214399999999998 + }, + "electrons": { + "particle_momentum_x": 9.621748589257923e-20, + "particle_position_x": 2.621440000001309, + "particle_position_y": 2.6214400000013076, + "particle_position_z": 2.6214399999999998, + "particle_weight": 128000000000.00002 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir.json b/Regression/Checksum/benchmarks_json/Python_Langmuir.json index 950d7f9194a..46867790cb4 100644 --- a/Regression/Checksum/benchmarks_json/Python_Langmuir.json +++ b/Regression/Checksum/benchmarks_json/Python_Langmuir.json @@ -1,13 +1,13 @@ { + "lev=0": { + "Ex": 3047094224871324.5, + "jx": 3.294250379355652e+18 + }, "electrons": { - "particle_momentum_x": 1.7507165884671642e-17, - "particle_position_x": 10.350911572498216, - "particle_position_y": 10.485760000000003, - "particle_position_z": 10.485760000000003, + "particle_momentum_x": 1.740964347107983e-17, + "particle_position_x": 10.351045542291539, + "particle_position_y": 10.485759999999997, + "particle_position_z": 10.48576, "particle_weight": 320000000000.00006 - }, - "lev=0": { - "Ex": 3091324128949712.5, - "jx": 3.3015975627803766e+18 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json b/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json index 2d1ca2f23c5..d152ff41998 100644 --- a/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json @@ -1,12 +1,12 @@ { + "lev=0": { + "Ex": 53556162492188.62, + "jx": 5.722548841990386e+16 + }, "electrons": { - "particle_momentum_x": 1.0449418710231497e-19, - "particle_position_x": 0.0831205701527011, - "particle_position_y": 0.08192, + "particle_momentum_x": 1.0581442023547547e-19, + "particle_position_x": 0.0831038802568349, + "particle_position_y": 0.08191999999999999, "particle_weight": 8000000000000001.0 - }, - "lev=0": { - "Ex": 54311177004701.516, - "jx": 5.639670983525456e+16 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json b/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json index 2129e8af565..2b4297e3627 100644 --- a/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json +++ b/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json @@ -1,32 +1,32 @@ { - "electrons": { - "particle_momentum_x": 6.808479576836945e-20, - "particle_momentum_y": 6.876934757849776e-20, - "particle_momentum_z": 6.802215755177324e-20, - "particle_position_x": 6339.279719574188, - "particle_position_y": 6400.519522256783, - "particle_position_z": 6332.552838182088, - "particle_weight": 4999999954710.04 - }, "lev=0": { - "Bx": 2.18760546171463e-05, - "By": 1.777427672926153e-05, - "Bz": 1.798431779205324e-05, - "Ex": 262735.3418998487, - "Ey": 264285.9081716973, - "Ez": 262283.0576561827, - "jx": 930.8363498458478, - "jy": 940.577756171662, - "jz": 929.9701516370898, + "Bx": 2.1885709623428054e-05, + "By": 1.7781538740987698e-05, + "Bz": 1.798374460438725e-05, + "Ex": 262814.04898035893, + "Ey": 264360.66474815976, + "Ez": 262360.8533027486, + "jx": 931.6306925490601, + "jy": 941.3279496407436, + "jz": 930.7478072804985, "part_per_cell": 65536.0 }, + "electrons": { + "particle_momentum_x": 6.823819887231586e-20, + "particle_momentum_y": 6.891558962132863e-20, + "particle_momentum_z": 6.817050175691089e-20, + "particle_position_x": 6339.113170504729, + "particle_position_y": 6400.3586668158905, + "particle_position_z": 6332.387112999683, + "particle_weight": 4999999954710.04 + }, "protons": { - "particle_momentum_x": 3.177356695273816e-21, - "particle_momentum_y": 3.165603704977051e-21, - "particle_momentum_z": 3.157786057110636e-21, - "particle_position_x": 6550.923797287247, - "particle_position_y": 6537.195882086984, - "particle_position_z": 6523.379262220709, + "particle_momentum_x": 2.9968929267374238e-21, + "particle_momentum_y": 2.984482330834465e-21, + "particle_momentum_z": 2.9781449297861096e-21, + "particle_position_x": 6550.923883255297, + "particle_position_y": 6537.195968516041, + "particle_position_z": 6523.379347813018, "particle_weight": 4999999954710.04 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd.json b/Regression/Checksum/benchmarks_json/restart_psatd.json index d22fb5b57e7..fd31e7057f4 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd.json @@ -1,15 +1,33 @@ { "lev=0": { - "Bx": 116102.13010406512, - "By": 245467.5412026496, - "Bz": 65305.35287114741, - "Ex": 118565481099326.73, - "Ey": 18640100573642.96, - "Ez": 51777969821120.38, - "jx": 2.1177246490642464e+16, - "jy": 351544815339900.3, - "jz": 4573466562652713.0, - "rho": 22294533.587530266 + "Bx": 116097.15119490701, + "By": 245135.43138545044, + "Bz": 65087.42714316293, + "Ex": 118461420042835.34, + "Ey": 18622845483678.824, + "Ez": 51793144992361.125, + "jx": 2.1162465965376624e+16, + "jy": 349487178458066.6, + "jz": 4563460427657682.0, + "rho": 22252333.818822004 + }, + "driverback": { + "particle_momentum_x": 4.813131349021332e+21, + "particle_momentum_y": 5.16548074090123e+21, + "particle_momentum_z": 3.005830430844926e+25, + "particle_position_x": 0.001649481123084974, + "particle_position_y": 0.0016172218745428432, + "particle_position_z": 0.4899808854005956, + "particle_weight": 6241509074.460762 + }, + "plasma_e": { + "particle_momentum_x": 2.084066965693142e-19, + "particle_momentum_y": 1.330336553422521e-20, + "particle_momentum_z": 2.6335518561017005e-17, + "particle_position_x": 0.4991724057788647, + "particle_position_y": 0.49919697481183745, + "particle_position_z": 0.4562433914656611, + "particle_weight": 33067341227104.625 }, "beam": { "particle_momentum_x": 4.178482505909375e-19, @@ -20,6 +38,15 @@ "particle_position_z": 1.901942694291968, "particle_weight": 3120754537.230381 }, + "plasma_p": { + "particle_momentum_x": 2.097709570232301e-19, + "particle_momentum_y": 3.402465268604418e-21, + "particle_momentum_z": 4.82960000875208e-14, + "particle_position_x": 0.49912500313484137, + "particle_position_y": 0.4991249983460674, + "particle_position_z": 0.4563845471150555, + "particle_weight": 33067341227104.625 + }, "driver": { "particle_momentum_x": 4.700436405078562e+21, "particle_momentum_y": 4.6785862113093076e+21, @@ -28,32 +55,5 @@ "particle_position_y": 0.0016212373149699414, "particle_position_z": 0.30417779498653386, "particle_weight": 6241509074.460762 - }, - "driverback": { - "particle_momentum_x": 4.813131349021332e+21, - "particle_momentum_y": 5.16548074090123e+21, - "particle_momentum_z": 3.005830430844926e+25, - "particle_position_x": 0.001649481123084974, - "particle_position_y": 0.0016172218745428432, - "particle_position_z": 0.4899808854005956, - "particle_weight": 6241509074.460762 - }, - "plasma_e": { - "particle_momentum_x": 2.0912364079892767e-19, - "particle_momentum_y": 1.3510804924903876e-20, - "particle_momentum_z": 2.6336070297356917e-17, - "particle_position_x": 0.49917333523960167, - "particle_position_y": 0.49919828074154127, - "particle_position_z": 0.45624167532314724, - "particle_weight": 33067341227104.625 - }, - "plasma_p": { - "particle_momentum_x": 2.1053025446547977e-19, - "particle_momentum_y": 3.3272838305044046e-21, - "particle_momentum_z": 4.829599953646669e-14, - "particle_position_x": 0.49912500297599893, - "particle_position_y": 0.49912499825258866, - "particle_position_z": 0.45638454709795806, - "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json index 50971f0d45e..cfd031d6e33 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json @@ -1,24 +1,15 @@ { "lev=0": { - "Bx": 115622.1878941125, - "By": 243403.66715242947, - "Bz": 63602.66529020351, - "Ex": 117118934931289.56, - "Ey": 18448377440588.27, - "Ez": 50821967818817.24, - "jx": 2.1044522674691452e+16, - "jy": 329314843111847.94, - "jz": 4524787623275627.0, - "rho": 22128883.53068064 - }, - "driverback": { - "particle_momentum_x": 4.813131349021332e+21, - "particle_momentum_y": 5.16548074090123e+21, - "particle_momentum_z": 3.005830430844926e+25, - "particle_position_x": 0.001649481123084974, - "particle_position_y": 0.0016172218745428432, - "particle_position_z": 0.4899808854005956, - "particle_weight": 6241509074.460762 + "Bx": 115610.21497814235, + "By": 243019.564489834, + "Bz": 63397.3416561916, + "Ex": 117012138466922.22, + "Ey": 18429606525820.12, + "Ez": 50835997822445.34, + "jx": 2.1028213761477244e+16, + "jy": 326844980054897.7, + "jz": 4514643615865121.0, + "rho": 22084564.546177678 }, "beam": { "particle_momentum_x": 4.178482505909375e-19, @@ -29,24 +20,33 @@ "particle_position_z": 1.901942694291968, "particle_weight": 3120754537.230381 }, - "plasma_e": { - "particle_momentum_x": 1.9905288525315664e-19, - "particle_momentum_y": 1.2685401564810352e-20, - "particle_momentum_z": 2.6334746597885943e-17, - "particle_position_x": 0.4991702885212871, - "particle_position_y": 0.4991949978513417, - "particle_position_z": 0.4562491611716817, + "plasma_p": { + "particle_momentum_x": 1.9959818202810056e-19, + "particle_momentum_y": 2.9455107653405726e-21, + "particle_momentum_z": 4.82960013792663e-14, + "particle_position_x": 0.4991250027534606, + "particle_position_y": 0.49912499804419774, + "particle_position_z": 0.4563845471451087, "particle_weight": 33067341227104.625 }, - "plasma_p": { - "particle_momentum_x": 2.002897995377179e-19, - "particle_momentum_y": 2.87423099731012e-21, - "particle_momentum_z": 4.8296000849128737e-14, - "particle_position_x": 0.49912500259846493, - "particle_position_y": 0.4991249979476612, - "particle_position_z": 0.45638454712861487, + "plasma_e": { + "particle_momentum_x": 1.9841122555955512e-19, + "particle_momentum_y": 1.2475587913571367e-20, + "particle_momentum_z": 2.6334235362474606e-17, + "particle_position_x": 0.49916947686056645, + "particle_position_y": 0.49919374997004357, + "particle_position_z": 0.4562507922838501, "particle_weight": 33067341227104.625 }, + "driverback": { + "particle_momentum_x": 4.813131349021332e+21, + "particle_momentum_y": 5.16548074090123e+21, + "particle_momentum_z": 3.005830430844926e+25, + "particle_position_x": 0.001649481123084974, + "particle_position_y": 0.0016172218745428432, + "particle_position_z": 0.4899808854005956, + "particle_weight": 6241509074.460762 + }, "driver": { "particle_momentum_x": 4.700436405078562e+21, "particle_momentum_y": 4.6785862113093076e+21, diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index d1f9eb16f47..5e3380cc1a6 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1284,8 +1284,16 @@ WarpX::ReadParameters () } } - // Use same shape factors in all directions, for gathering - if (field_gathering_algo == GatheringAlgo::MomentumConserving) { galerkin_interpolation = false; } + // Use same shape factors in all directions + // - with momentum-conserving field gathering + if (field_gathering_algo == GatheringAlgo::MomentumConserving) {galerkin_interpolation = false;} + // - with direct current deposition and the EM solver + if( electromagnetic_solver_id != ElectromagneticSolverAlgo::None && + electromagnetic_solver_id != ElectromagneticSolverAlgo::HybridPIC ) { + if (current_deposition_algo == CurrentDepositionAlgo::Direct) { + galerkin_interpolation = false; + } + } { const ParmParse pp_interpolation("interpolation"); From fd2248ace32d2022fb89d6d04bacc5410428ae50 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 4 Jul 2024 07:18:54 +0200 Subject: [PATCH 143/190] Release 24.07 (#5025) * AMReX: 24.07 * pyAMReX: 24.07 * WarpX: 24.07 --- .github/workflows/cuda.yml | 2 +- CMakeLists.txt | 2 +- Docs/source/conf.py | 4 ++-- Python/setup.py | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 4 ++-- cmake/dependencies/pyAMReX.cmake | 4 ++-- run_test.sh | 2 +- setup.py | 2 +- 10 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 7e1cf7ae63f..190294bb0e1 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a && cd - + cd ../amrex && git checkout --detach 24.07 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/CMakeLists.txt b/CMakeLists.txt index 5e2d1ebba9c..23427a00ec8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.06) +project(WarpX VERSION 24.07) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 18ca8370e80..7997b1e338c 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.06' +version = u'24.07' # The full version, including alpha/beta/rc tags. -release = u'24.06' +release = u'24.07' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Python/setup.py b/Python/setup.py index 9f9c8d7d736..e4adbb6c458 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,7 +54,7 @@ package_data = {} setup(name = 'pywarpx', - version = '24.06', + version = '24.07', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index e4e5d7b8678..0003dcff134 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a +branch = 24.07 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index a18f39b89c1..aa7c74cf21c 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a +branch = 24.07 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 632b9edd4f4..0b90b09df30 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.06 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.07 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "259db7cfb99e7d1d2ab4bec9b1587fdf624a138a" +set(WarpX_amrex_branch "24.07" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 90aee6cb930..5c17681d4b6 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.06 CONFIG REQUIRED) + find_package(pyAMReX 24.07 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "fd7126cb907be7e3c8570930b79209db5cd7c492" +set(WarpX_pyamrex_branch "24.07" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 6ac22466ef0..37264bef243 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 259db7cfb99e7d1d2ab4bec9b1587fdf624a138a && cd - +cd amrex && git checkout --detach 24.07 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets diff --git a/setup.py b/setup.py index 6ad6fd81960..f05d5a71899 100644 --- a/setup.py +++ b/setup.py @@ -280,7 +280,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.06', + version = '24.07', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.', From f9f3db5534836b5ddc53b10310007522bf84e9cd Mon Sep 17 00:00:00 2001 From: Steve Richardson Date: Fri, 5 Jul 2024 12:56:46 -0400 Subject: [PATCH 144/190] Fix typo in parameters.rst (#5030) --- Docs/source/usage/parameters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 4dc888df3d0..313c5f0d740 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -86,7 +86,7 @@ Overall simulation parameters * ``explicit``: Use an explicit solver, such as the standard FDTD or PSATD - * ``theta_implicit_em``: Use a fully implicit electromagnetic solver with a time-biasing parameter theta bound between 0.5 and 1.0. Exact energy conservation is achieved using theta = 0.5. Maximal damping of high-k modes is obtained using theta = 1.0. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed (PS) JNFK. + * ``theta_implicit_em``: Use a fully implicit electromagnetic solver with a time-biasing parameter theta bound between 0.5 and 1.0. Exact energy conservation is achieved using theta = 0.5. Maximal damping of high-k modes is obtained using theta = 1.0. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed (PS) JFNK. The algorithm itself is numerical stable for large time steps. That is, it does not require time steps that resolve the plasma period or the CFL condition for light waves. However, the practicality of using a large time step depends on the nonlinear solver. Note that the Picard solver is for demonstration only. It is inefficient and will most like not converge when :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relevant physics. The Picard method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. From bf77d6d6c5ef75e61c13c3ab9a3014cd18a04b35 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Sat, 6 Jul 2024 02:33:06 +0200 Subject: [PATCH 145/190] replace std::is_same::value with std::is_same_v (#5028) --- Source/Diagnostics/ReducedDiags/FieldReduction.H | 6 +++--- Source/Initialization/WarpXInitData.cpp | 4 ++-- .../Collision/BinaryCollision/BinaryCollision.H | 2 +- Source/Particles/LaserParticleContainer.cpp | 2 +- Source/Utils/Parser/ParserUtils.H | 12 ++++++------ Source/ablastr/fields/PoissonSolver.H | 2 +- Source/ablastr/fields/VectorPoissonSolver.H | 2 +- Tools/QedTablesUtils/Source/QedTableGenerator.cpp | 4 ++-- Tools/QedTablesUtils/Source/QedTableReader.cpp | 4 ++-- 9 files changed, 19 insertions(+), 19 deletions(-) diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index a15d6489e2c..f467499ad56 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -212,15 +212,15 @@ public: amrex::Real reduce_value = amrex::get<0>(reduce_data.value()); // MPI reduce - if (std::is_same::value) + if (std::is_same_v) { amrex::ParallelDescriptor::ReduceRealMax(reduce_value); } - if (std::is_same::value) + if (std::is_same_v) { amrex::ParallelDescriptor::ReduceRealMin(reduce_value); } - if (std::is_same::value) + if (std::is_same_v) { amrex::ParallelDescriptor::ReduceRealSum(reduce_value); // If reduction operation is a sum, multiply the value by the cell volume so that the diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 68b38153f8d..03393debeae 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -155,13 +155,13 @@ WarpX::PrintMainPICparameters () amrex::Print() << "-------------------------------------------------------------------------------\n"; // print warpx build information - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { amrex::Print() << "Precision: | SINGLE" << "\n"; } else { amrex::Print() << "Precision: | DOUBLE" << "\n"; } - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { amrex::Print() << "Particle precision: | SINGLE" << "\n"; } else { diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index b3d39a9c581..a386cb3012e 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -108,7 +108,7 @@ public: } m_have_product_species = !m_product_species.empty(); - if ((std::is_same::value) && (m_have_product_species)) { + if ((std::is_same_v) && (m_have_product_species)) { WARPX_ABORT_WITH_MESSAGE( "Binary collision " + collision_name + " does not produce species. Thus, `product_species` should not be specified in the input script." ); } diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index 0693a13c1f9..bd266ab368a 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -719,7 +719,7 @@ LaserParticleContainer::ComputeSpacing (int lev, Real& Sx, Real& Sy) const #if !defined(WARPX_DIM_RZ) constexpr float small_float_coeff = 1.e-25f; constexpr double small_double_coeff = 1.e-50; - constexpr Real small_coeff = std::is_same::value ? + constexpr Real small_coeff = std::is_same_v ? static_cast(small_float_coeff) : static_cast(small_double_coeff); const auto eps = static_cast(dx[0]*small_coeff); diff --git a/Source/Utils/Parser/ParserUtils.H b/Source/Utils/Parser/ParserUtils.H index dbc4845d1bf..96937abdbbe 100644 --- a/Source/Utils/Parser/ParserUtils.H +++ b/Source/Utils/Parser/ParserUtils.H @@ -147,7 +147,7 @@ namespace utils::parser auto parser = makeParser(str_val, {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); @@ -174,7 +174,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -217,7 +217,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -250,7 +250,7 @@ namespace utils::parser Store_parserString(a_pp, str, str_val); auto parser = makeParser(str_val, {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -270,7 +270,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -308,7 +308,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index e5340e825c9..fed76b2987b 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -315,7 +315,7 @@ computePhi (amrex::Vector const & rho, } // Run additional operations, such as calculation of the E field for embedded boundaries - if constexpr (!std::is_same::value) { + if constexpr (!std::is_same_v) { if (post_phi_calculation.has_value()) { post_phi_calculation.value()(mlmg, lev); } diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index d49335723d8..3ef96c30c84 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -241,7 +241,7 @@ computeVectorPotential ( amrex::Vector > co curr[lev][adim]->mult(-1._rt/ablastr::constant::SI::mu0); } // Loop over adim // Run additional operations, such as calculation of the B fields for embedded boundaries - if constexpr (!std::is_same::value) { + if constexpr (!std::is_same_v) { if (post_A_calculation.has_value()) { post_A_calculation.value()(mlmg, lev); } diff --git a/Tools/QedTablesUtils/Source/QedTableGenerator.cpp b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp index dfbde1df050..1ea62b5c6ed 100644 --- a/Tools/QedTablesUtils/Source/QedTableGenerator.cpp +++ b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp @@ -106,7 +106,7 @@ template void GenerateTableBW (const ParsedArgs& args, const string& outfile_name) { cout << " Generating BW table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; if (!Contains(args, "--dndt_chi_min") || !Contains(args, "--dndt_chi_max") || !Contains(args, "--dndt_how_many") || !Contains(args, "--pair_chi_min") || @@ -173,7 +173,7 @@ template void GenerateTableQS (const ParsedArgs& args, const string& outfile_name) { cout << " Generating QS table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; if (!Contains(args, "--dndt_chi_min") || !Contains(args, "--dndt_chi_max") || !Contains(args, "--dndt_how_many") || !Contains(args, "--em_chi_min") || diff --git a/Tools/QedTablesUtils/Source/QedTableReader.cpp b/Tools/QedTablesUtils/Source/QedTableReader.cpp index ae689a87643..ba9d58775f2 100644 --- a/Tools/QedTablesUtils/Source/QedTableReader.cpp +++ b/Tools/QedTablesUtils/Source/QedTableReader.cpp @@ -117,7 +117,7 @@ template void ReadTableBW (const string& input_file, const string& outfile_name) { cout << " Reading BW table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; auto ifs = ifstream(input_file, std::ios::binary); auto raw_data = vector(istreambuf_iterator(ifs), {}); @@ -161,7 +161,7 @@ void ReadTableQS ( const string& input_file, const string& outfile_name) { cout << " Reading QS table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; auto ifs = ifstream(input_file, std::ios::binary); auto raw_data = vector(istreambuf_iterator(ifs), {}); From 0c333653ce35d25cace06754b03518e3566b0139 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 8 Jul 2024 07:35:20 -0700 Subject: [PATCH 146/190] Remove debug build type from Langmuir 2D fluid test (#5031) While working on another PR, I noticed that the CI test `Langmuir_fluid_2D` was compiled in debug mode. The test was added originally in #3991 and it might be that the debug build type was left over unintentionally. In general, I think we avoid running CI tests in debug mode, in order to keep the runtime of the tests as low as possible (the current runtime of this test in debug mode is around 80 seconds). However, we might have changed policy in the last months and I might not be up-to-date, so feel free to let me know and close this PR without merging if it is not relevant. --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index aa7c74cf21c..e33be6b42ba 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -1173,7 +1173,7 @@ inputFile = Examples/Tests/langmuir_fluids/inputs_2d runtime_params = dim = 2 addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -DCMAKE_BUILD_TYPE=Debug +cmakeSetupOpts = -DWarpX_DIMS=2 restartTest = 0 useMPI = 1 numprocs = 2 From ce9f4af505c5b02d66573c3163dfbc9d338fecc3 Mon Sep 17 00:00:00 2001 From: Olga Shapoval <30510597+oshapoval@users.noreply.github.com> Date: Tue, 9 Jul 2024 14:59:26 -0700 Subject: [PATCH 147/190] Add ability to load external particle fields from file (#4997) * Update documentation * Allocate fields * Always parse filename * Add option "FromFile" in external particles fields * Create two separate examples * Update documentation * Allocate aux array * Update aux to take into account copy * Add external fields from file to auxiliary data * Update tests * Remove debugging Print statements * If averaged PSATD is used, we copy Efield_avg_fp to Efield_aux, otherwise copy Efield_fp. * Do not access cells that have invalid data when summing external fields * Update path of checkpoint files * Update comments * Update number of components * Fix clang tidy error * Make the code compatible with momentum-conserving gather * Add Python interface and tests * Use PICMI version that defined LoadAppliedField * Update file names * Update PICMI standard version * Allocate dedicated MultiFabs for external particle fields * Some clean-up * Performance optimization: do not use external fields in gather kernel * Allow external particle fields and external grid fields to be simultaneously defined * External fields are incompatible with the moving window * Support load balancing * Update Source/Initialization/WarpXInitData.cpp * Update Source/Initialization/WarpXInitData.cpp --------- Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 19 ++- ...s_3d.py => PICMI_inputs_3d_grid_fields.py} | 4 +- .../PICMI_inputs_3d_particle_fields.py | 140 ++++++++++++++++++ .../{inputs_rz => inputs_rz_grid_fields} | 0 .../inputs_rz_particle_fields | 65 ++++++++ Python/pywarpx/picmi.py | 7 + ...ldRZ.json => LoadExternalFieldRZGrid.json} | 0 .../LoadExternalFieldRZParticles.json | 22 +++ ...on => Python_LoadExternalGridField3D.json} | 0 .../Python_LoadExternalParticleField3D.json | 22 +++ Regression/WarpX-tests.ini | 51 ++++++- Source/Initialization/ExternalField.cpp | 1 - Source/Initialization/WarpXInitData.cpp | 50 +++++++ Source/Parallelization/WarpXComm.cpp | 44 +++++- Source/Parallelization/WarpXRegrid.cpp | 6 + Source/Particles/Gather/GetExternalFields.cpp | 12 ++ Source/WarpX.H | 2 + Source/WarpX.cpp | 92 ++++++++---- requirements.txt | 2 +- 19 files changed, 497 insertions(+), 42 deletions(-) rename Examples/Tests/LoadExternalField/{PICMI_inputs_3d.py => PICMI_inputs_3d_grid_fields.py} (96%) create mode 100644 Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py rename Examples/Tests/LoadExternalField/{inputs_rz => inputs_rz_grid_fields} (100%) create mode 100644 Examples/Tests/LoadExternalField/inputs_rz_particle_fields rename Regression/Checksum/benchmarks_json/{LoadExternalFieldRZ.json => LoadExternalFieldRZGrid.json} (100%) create mode 100644 Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json rename Regression/Checksum/benchmarks_json/{Python_LoadExternalField3D.json => Python_LoadExternalGridField3D.json} (100%) create mode 100644 Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 313c5f0d740..d6420440935 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1747,8 +1747,8 @@ are applied to the grid directly. In particular, these fields can be seen in the One can refer to input files in ``Examples/Tests/LoadExternalField`` for more information. Regarding how to prepare the openPMD data file, one can refer to the `openPMD-example-datasets `__. - Note that if both `B_ext_grid_init_style` and `E_ext_grid_init_style` are set to - `read_from_file`, the openPMD file specified by `warpx.read_fields_from_path` + Note that if both ``B_ext_grid_init_style`` and ``E_ext_grid_init_style`` are set to + ``read_from_file``, the openPMD file specified by ``warpx.read_fields_from_path`` should contain both B and E external fields data. * ``warpx.E_external_grid`` & ``warpx.B_external_grid`` (list of `3 floats`) @@ -1804,6 +1804,21 @@ are applied to the particles directly, at each timestep. As a results, these fie Note that the position is defined in Cartesian coordinates, as a function of (x,y,z), even for RZ. + * ``read_from_file``: load the external field from an openPMD file. + An additional parameter, indicating the path of an openPMD data file, ``particles.read_fields_from_path`` + must be specified, from which the external E field data can be loaded into WarpX. + One can refer to input files in ``Examples/Tests/LoadExternalField`` for more information. + Regarding how to prepare the openPMD data file, one can refer to + the `openPMD-example-datasets `__. + Note that if both ``B_ext_particle_init_style`` and ``E_ext_particle_init_style`` are set to + ``read_from_file``, the openPMD file specified by ``particles.read_fields_from_path`` + should contain both B and E external fields data. + + .. note:: + + When using ``read_from_file``, the fields loaded from the file will be interpolated + to the resolution of the grid used for the simulation. + * ``repeated_plasma_lens``: apply a series of plasma lenses. The properties of the lenses are defined in the lab frame by the input parameters: diff --git a/Examples/Tests/LoadExternalField/PICMI_inputs_3d.py b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py similarity index 96% rename from Examples/Tests/LoadExternalField/PICMI_inputs_3d.py rename to Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py index 849d4d2be08..d128d9c10e0 100644 --- a/Examples/Tests/LoadExternalField/PICMI_inputs_3d.py +++ b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py @@ -93,7 +93,7 @@ species=[ions], data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], write_dir='.', - warpx_file_prefix='Python_LoadExternalField3D_plt' + warpx_file_prefix='Python_LoadExternalGridField3D_plt' ) field_diag = picmi.FieldDiagnostic( name='diag1', @@ -101,7 +101,7 @@ period=300, data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], write_dir='.', - warpx_file_prefix='Python_LoadExternalField3D_plt' + warpx_file_prefix='Python_LoadExternalGridField3D_plt' ) ################################# diff --git a/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py new file mode 100644 index 00000000000..7bf7c5a084c --- /dev/null +++ b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# +# --- Input file for loading initial field from openPMD file. + +from pywarpx import picmi + +constants = picmi.constants + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_steps = 300 + +max_grid_size = 40 +nx = max_grid_size +ny = max_grid_size +nz = max_grid_size + +xmin = -1 +xmax = 1 +ymin = xmin +ymax = xmax +zmin = 0 +zmax = 5 + +number_per_cell = 200 + +################################# +############ NUMERICS ########### +################################# + +verbose = 1 +dt = 4.4e-7 +use_filter = 0 + +# Order of particle shape factors +particle_shape = 1 + +################################# +############ PLASMA ############# +################################# + +ion_dist = picmi.ParticleListDistribution( + x=0.0, + y=0.2, + z=2.5, + ux=9.5e-05*constants.c, + uy=0.0*constants.c, + uz=1.34e-4*constants.c, + weight=1.0 + ) + +ions = picmi.Species( + particle_type='H', + name='proton', charge='q_e',mass="m_p", + warpx_do_not_deposit=1, + initial_distribution=ion_dist + ) + +################################# +######## INITIAL FIELD ########## +################################# + +applied_field = picmi.LoadAppliedField( + read_fields_from_path="../../../../openPMD-example-datasets/example-femm-3d.h5", + load_E=False + ) + +################################# +###### GRID AND SOLVER ########## +################################# + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + warpx_max_grid_size=max_grid_size, + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=['dirichlet', 'dirichlet', 'dirichlet'], + upper_boundary_conditions=['dirichlet', 'dirichlet', 'dirichlet'], + lower_boundary_conditions_particles=['absorbing', 'absorbing', 'absorbing'], + upper_boundary_conditions_particles=['absorbing', 'absorbing', 'absorbing'] + ) +solver = picmi.ElectrostaticSolver(grid=grid) + +################################# +######### DIAGNOSTICS ########### +################################# + +particle_diag = picmi.ParticleDiagnostic( + name='diag1', + period=300, + species=[ions], + data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], + write_dir='.', + warpx_file_prefix='Python_LoadExternalParticleField3D_plt' + ) +field_diag = picmi.FieldDiagnostic( + name='diag1', + grid=grid, + period=300, + data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], + write_dir='.', + warpx_file_prefix='Python_LoadExternalParticleField3D_plt' + ) + +################################# +####### SIMULATION SETUP ######## +################################# + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=verbose, + warpx_serialize_initial_conditions=False, + warpx_grid_type='collocated', + warpx_do_dynamic_scheduling=False, + warpx_use_filter=use_filter, + time_step_size=dt, + particle_shape=particle_shape + ) + +sim.add_applied_field(applied_field) + +sim.add_species( + ions, + layout=picmi.PseudoRandomLayout( + n_macroparticles_per_cell=number_per_cell, grid=grid + ) + ) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(particle_diag) + +################################# +##### SIMULATION EXECUTION ###### +################################# + +#sim.write_input_file('PICMI_inputs_3d') +sim.step(max_steps) diff --git a/Examples/Tests/LoadExternalField/inputs_rz b/Examples/Tests/LoadExternalField/inputs_rz_grid_fields similarity index 100% rename from Examples/Tests/LoadExternalField/inputs_rz rename to Examples/Tests/LoadExternalField/inputs_rz_grid_fields diff --git a/Examples/Tests/LoadExternalField/inputs_rz_particle_fields b/Examples/Tests/LoadExternalField/inputs_rz_particle_fields new file mode 100644 index 00000000000..b76d4cb7efc --- /dev/null +++ b/Examples/Tests/LoadExternalField/inputs_rz_particle_fields @@ -0,0 +1,65 @@ +warpx.serialize_initial_conditions = 0 +warpx.do_dynamic_scheduling = 0 +particles.do_tiling = 0 + +particles.B_ext_particle_init_style = "read_from_file" +particles.read_fields_from_path = "../../../../openPMD-example-datasets/example-femm-thetaMode.h5" + +warpx.grid_type = collocated +warpx.do_electrostatic = labframe + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 300 +amr.n_cell = 40 40 +warpx.numprocs = 1 1 +amr.max_level = 0 +geometry.dims = RZ + +geometry.prob_lo = 0.0 0.0 +geometry.prob_hi = 1.0 5.0 + +################################# +###### Boundary Condition ####### +################################# +boundary.field_lo = none pec +boundary.field_hi = pec pec +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = 4.40917904849092e-7 +warpx.use_filter = 0 + +# Order of particle shape factors +algo.particle_shape = 1 + +################################# +############ PLASMA ############# +################################# +particles.species_names = proton +proton.injection_style = "SingleParticle" +proton.single_particle_pos = 0.0 0.2 2.5 +proton.single_particle_u = 9.506735958279367e-05 0.0 0.00013435537232359165 +proton.single_particle_weight = 1.0 +proton.do_not_deposit = 1 +proton.mass = m_p +proton.charge = q_e + +# Diagnostics +diagnostics.diags_names = diag1 chk +diag1.intervals = 300 +diag1.diag_type = Full + +chk.intervals = 150 +chk.diag_type = Full +chk.format = checkpoint diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 0048cb1e51b..42c0313a6e9 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1486,6 +1486,13 @@ def applied_field_initialize_inputs(self): expression = pywarpx.my_constants.mangle_expression(expression, self.mangle_dict) pywarpx.warpx.__setattr__(f'B{sdir}_external_grid_function(x,y,z)', expression) +class LoadAppliedField(picmistandard.PICMI_LoadAppliedField): + def applied_field_initialize_inputs(self): + pywarpx.particles.read_fields_from_path = self.read_fields_from_path + if self.load_E: + pywarpx.particles.E_ext_particle_init_style = 'read_from_file' + if self.load_B: + pywarpx.particles.B_ext_particle_init_style = 'read_from_file' class ConstantAppliedField(picmistandard.PICMI_ConstantAppliedField): def applied_field_initialize_inputs(self): diff --git a/Regression/Checksum/benchmarks_json/LoadExternalFieldRZ.json b/Regression/Checksum/benchmarks_json/LoadExternalFieldRZGrid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LoadExternalFieldRZ.json rename to Regression/Checksum/benchmarks_json/LoadExternalFieldRZGrid.json diff --git a/Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json b/Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json new file mode 100644 index 00000000000..dc4ffa3beb2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json @@ -0,0 +1,22 @@ +{ + "lev=0": { + "Br": 0.6380326770459374, + "Bt": 0.0, + "Bz": 4.850698699951099, + "Er": 0.0, + "Et": 0.0, + "Ez": 0.0, + "jr": 0.0, + "jt": 0.0, + "jz": 0.0 + }, + "proton": { + "particle_momentum_x": 1.2945379007125463e-23, + "particle_momentum_y": 7.901629565307941e-23, + "particle_momentum_z": 2.0004592432172922e-23, + "particle_position_x": 0.12402004585603355, + "particle_position_y": 4.363249204658946, + "particle_theta": 0.22200801011337173, + "particle_weight": 1.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LoadExternalField3D.json b/Regression/Checksum/benchmarks_json/Python_LoadExternalGridField3D.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_LoadExternalField3D.json rename to Regression/Checksum/benchmarks_json/Python_LoadExternalGridField3D.json diff --git a/Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json b/Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json new file mode 100644 index 00000000000..a34b2f136a6 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json @@ -0,0 +1,22 @@ +{ + "lev=0": { + "Bx": 29.8448958859583, + "By": 29.844895885958305, + "Bz": 197.55124310523018, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "proton": { + "particle_momentum_x": 1.2118953253556959e-23, + "particle_momentum_y": 7.822512598009088e-23, + "particle_momentum_z": 2.2761898274347412e-23, + "particle_position_x": 0.12238072371983327, + "particle_position_y": 0.009653941154598592, + "particle_position_z": 4.317544769595657, + "particle_weight": 1.0 + } +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index e33be6b42ba..3d1dbfe3b0f 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -2266,10 +2266,29 @@ doVis = 0 compareParticles = 0 analysisRoutine = Examples/Tests/resampling/analysis_leveling_thinning.py -[LoadExternalFieldRZ] +[LoadExternalFieldRZGrid] buildDir = . -inputFile = Examples/Tests/LoadExternalField/inputs_rz -runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZ_chk chk.file_min_digits=5 +inputFile = Examples/Tests/LoadExternalField/inputs_rz_grid_fields +runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZGrid_chk chk.file_min_digits=5 +dim = 2 +addToCompileString = USE_RZ=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON +restartTest = 1 +restartFileNum = 150 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = proton +analysisRoutine = Examples/Tests/LoadExternalField/analysis_rz.py + +[LoadExternalFieldRZParticles] +buildDir = . +inputFile = Examples/Tests/LoadExternalField/inputs_rz_particle_fields +runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZParticles_chk chk.file_min_digits=5 dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON @@ -3448,11 +3467,31 @@ compileTest = 0 doVis = 0 analysisRoutine = Examples/analysis_default_openpmd_regression.py -[Python_LoadExternalField3D] +[Python_LoadExternalGridField3D] buildDir = . -inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d.py +inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py runtime_params = -customRunCmd = python3 PICMI_inputs_3d.py +customRunCmd = python3 PICMI_inputs_3d_grid_fields.py +dim = 3 +addToCompileString = USE_PYTHON_MAIN=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +compileTest = 0 +doVis = 0 +compareParticles = 1 +particleTypes = proton +analysisRoutine = Examples/Tests/LoadExternalField/analysis_3d.py + +[Python_LoadExternalParticleField3D] +buildDir = . +inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py +runtime_params = +customRunCmd = python3 PICMI_inputs_3d_particle_fields.py dim = 3 addToCompileString = USE_PYTHON_MAIN=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON diff --git a/Source/Initialization/ExternalField.cpp b/Source/Initialization/ExternalField.cpp index e7a272e2e95..69120c0ae3c 100644 --- a/Source/Initialization/ExternalField.cpp +++ b/Source/Initialization/ExternalField.cpp @@ -172,7 +172,6 @@ ExternalFieldParams::ExternalFieldParams(const amrex::ParmParse& pp_warpx) // // External fields from file // - if (E_ext_grid_type == ExternalFieldType::read_from_file || B_ext_grid_type == ExternalFieldType::read_from_file){ const std::string read_fields_from_path="./"; diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 03393debeae..9a4c46b1c10 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -1359,6 +1359,21 @@ void WarpX::CheckKnownIssues() void WarpX::LoadExternalFieldsFromFile (int const lev) { + // External fields from file are currently not compatible with the moving window + // In order to support the moving window, the MultiFab containing the external + // fields should be updated every time the window moves. + if ( (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) || + (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) || + (mypc->m_B_ext_particle_s == "read_from_file") || + (mypc->m_E_ext_particle_s == "read_from_file") ) { + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + WarpX::do_moving_window == 0, + "External fields from file are not compatible with the moving window." ); + } + + // External grid fields + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, @@ -1383,6 +1398,41 @@ WarpX::LoadExternalFieldsFromFile (int const lev) ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); +#endif + } + + // External particle fields + + if (mypc->m_B_ext_particle_s == "read_from_file") { + std::string external_fields_path; + const amrex::ParmParse pp_particles("particles"); + pp_particles.get("read_fields_from_path", external_fields_path ); +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][0].get(), "B", "r"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][1].get(), "B", "t"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][2].get(), "B", "z"); +#else + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][0].get(), "B", "x"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][1].get(), "B", "y"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][2].get(), "B", "z"); +#endif + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + std::string external_fields_path; + const amrex::ParmParse pp_particles("particles"); + pp_particles.get("read_fields_from_path", external_fields_path ); +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][0].get(), "E", "r"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][1].get(), "E", "t"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][2].get(), "E", "z"); +#else + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][0].get(), "E", "x"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][1].get(), "E", "y"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][2].get(), "E", "z"); #endif } } diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index dcf5fbe5621..e7df489236e 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -18,6 +18,7 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpXComm_K.H" #include "WarpXSumGuardCells.H" +#include "Particles/MultiParticleContainer.H" #include #include @@ -59,6 +60,21 @@ WarpX::UpdateAuxilaryData () } else { UpdateAuxilaryDataStagToNodal(); } + + // When loading particle fields from file: add the external fields: + for (int lev = 0; lev <= finest_level; ++lev) { + if (mypc->m_E_ext_particle_s == "read_from_file") { + amrex::MultiFab::Add(*Efield_aux[lev][0], *E_external_particle_field[lev][0], 0, 0, E_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[lev][1], *E_external_particle_field[lev][1], 0, 0, E_external_particle_field[lev][1]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[lev][2], *E_external_particle_field[lev][2], 0, 0, E_external_particle_field[lev][2]->nComp(), guard_cells.ng_FieldGather); + } + if (mypc->m_B_ext_particle_s == "read_from_file") { + amrex::MultiFab::Add(*Bfield_aux[lev][0], *B_external_particle_field[lev][0], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[lev][1], *B_external_particle_field[lev][1], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[lev][2], *B_external_particle_field[lev][2], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + } + } + } void @@ -293,6 +309,30 @@ WarpX::UpdateAuxilaryDataStagToNodal () void WarpX::UpdateAuxilaryDataSameType () { + // Update aux field, including guard cells, up to ng_FieldGather + const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + + // Level 0: Copy from fine to aux + // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the + // same MultiFab object. MultiFab::Copy operation automatically detects this and does nothing in this case. + if (WarpX::fft_do_time_averaging) + { + MultiFab::Copy(*Efield_aux[0][0], *Efield_avg_fp[0][0], 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *Efield_avg_fp[0][1], 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *Efield_avg_fp[0][2], 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][0], *Bfield_avg_fp[0][0], 0, 0, Bfield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][1], *Bfield_avg_fp[0][1], 0, 0, Bfield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][2], *Bfield_avg_fp[0][2], 0, 0, Bfield_aux[0][2]->nComp(), ng_src); + } + else + { + MultiFab::Copy(*Efield_aux[0][0], *Efield_fp[0][0], 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *Efield_fp[0][1], 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *Efield_fp[0][2], 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][0], *Bfield_fp[0][0], 0, 0, Bfield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][1], *Bfield_fp[0][1], 0, 0, Bfield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][2], *Bfield_fp[0][2], 0, 0, Bfield_aux[0][2]->nComp(), ng_src); + } for (int lev = 1; lev <= finest_level; ++lev) { const amrex::Periodicity& crse_period = Geom(lev-1).periodicity(); @@ -308,8 +348,6 @@ WarpX::UpdateAuxilaryDataSameType () dBy.setVal(0.0); dBz.setVal(0.0); - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs @@ -379,8 +417,6 @@ WarpX::UpdateAuxilaryDataSameType () dEy.setVal(0.0); dEz.setVal(0.0); - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; // Copy Efield_aux to the dE MultiFabs, using up to ng_src (=ng_FieldGather) guard // cells from Efield_aux and filling up to ng (=nGrow) guard cells in the dE MultiFabs ablastr::utils::communication::ParallelCopy(dEx, *Efield_aux[lev - 1][0], 0, 0, diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index 52586a5c88c..d33bd17ccdd 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -192,6 +192,12 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { RemakeMultiFab(Efield_fp_external[lev][idim], true); } + if (mypc->m_B_ext_particle_s == "read_from_file") { + RemakeMultiFab(B_external_particle_field[lev][idim], true); + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + RemakeMultiFab(E_external_particle_field[lev][idim], true); + } RemakeMultiFab(current_fp[lev][idim], false); RemakeMultiFab(current_store[lev][idim], false); if (current_deposition_algo == CurrentDepositionAlgo::Vay) { diff --git a/Source/Particles/Gather/GetExternalFields.cpp b/Source/Particles/Gather/GetExternalFields.cpp index 8e3c679aa3a..bb55f79f394 100644 --- a/Source/Particles/Gather/GetExternalFields.cpp +++ b/Source/Particles/Gather/GetExternalFields.cpp @@ -83,6 +83,18 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset m_repeated_plasma_lens_strengths_B = mypc.d_repeated_plasma_lens_strengths_B.data(); } + // When the external particle fields are read from file, + // the external fields are not added directly inside the gather kernel. + // (Hence of `None`, which ensures that the gather kernel is compiled without support + // for external fields.) Instead, the external fields are added to the MultiFab + // Efield_aux and Bfield_aux before the particles gather from these MultiFab. + if (mypc.m_E_ext_particle_s == "read_from_file") { + m_Etype = ExternalFieldInitType::None; + } + if (mypc.m_B_ext_particle_s == "read_from_file") { + m_Btype = ExternalFieldInitType::None; + } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_Etype != Unknown, "Unknown E_ext_particle_init_style"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_Btype != Unknown, "Unknown B_ext_particle_init_style"); diff --git a/Source/WarpX.H b/Source/WarpX.H index 768f2d486dc..3372dd869cc 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1512,6 +1512,8 @@ private: // Same as Bfield_fp/Efield_fp for reading external field data amrex::Vector, 3 > > Efield_fp_external; amrex::Vector, 3 > > Bfield_fp_external; + amrex::Vector, 3 > > E_external_particle_field; + amrex::Vector, 3 > > B_external_particle_field; //! EB: Lengths of the mesh edges amrex::Vector, 3 > > m_edge_lengths; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 5e3380cc1a6..4dfebe90c67 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -343,6 +343,8 @@ WarpX::WarpX () // Same as Bfield_fp/Efield_fp for reading external field data Bfield_fp_external.resize(1); Efield_fp_external.resize(1); + B_external_particle_field.resize(1); + E_external_particle_field.resize(1); m_edge_lengths.resize(nlevs_max); m_face_areas.resize(nlevs_max); @@ -2315,18 +2317,6 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(current_fp[lev][1], amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[y]", 0.0_rt); AllocInitMultiFab(current_fp[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[z]", 0.0_rt); - // Match external field MultiFabs to fine patch - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); - } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { - AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); - } - if (do_current_centering) { amrex::BoxArray const& nodal_ba = amrex::convert(ba, amrex::IntVect::TheNodeVector()); @@ -2558,23 +2548,35 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Efield_aux[lev][1], nba, dm, ncomps, ngEB, lev, "Efield_aux[y]", 0.0_rt); AllocInitMultiFab(Efield_aux[lev][2], nba, dm, ncomps, ngEB, lev, "Efield_aux[z]", 0.0_rt); } else if (lev == 0) { - if (!WarpX::fft_do_time_averaging) { - // In this case, the aux grid is simply an alias of the fp grid - AliasInitMultiFab(Efield_aux[lev][0], *Efield_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *Efield_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *Efield_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); - - AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); - } else { - AliasInitMultiFab(Efield_aux[lev][0], *Efield_avg_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *Efield_avg_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *Efield_avg_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); - + if (WarpX::fft_do_time_averaging) { AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_avg_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_avg_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_avg_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); + + AliasInitMultiFab(Efield_aux[lev][0], *Efield_avg_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][1], *Efield_avg_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][2], *Efield_avg_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + } else { + if (mypc->m_B_ext_particle_s == "read_from_file") { + AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]"); + AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]"); + AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]"); + } else { + // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) + AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); + AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); + AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]"); + AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]"); + AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); + } else { + // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) + AliasInitMultiFab(Efield_aux[lev][0], *Efield_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][1], *Efield_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][2], *Efield_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + } } } else { AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]"); @@ -2586,6 +2588,44 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); } + // The external fields that are read from file + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + // These fields will be added directly to the grid, i.e. to fp, and need to match the index type + AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bfield_fp[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); + AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, Bfield_fp[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); + AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bfield_fp[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); + } + if (mypc->m_B_ext_particle_s == "read_from_file") { + // These fields will be added to the fields that the particles see, and need to match the index type + AllocInitMultiFab(B_external_particle_field[lev][0], amrex::convert(ba, Bfield_aux[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "B_external_particle_field[x]", 0.0_rt); + AllocInitMultiFab(B_external_particle_field[lev][1], amrex::convert(ba, Bfield_aux[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "B_external_particle_field[y]", 0.0_rt); + AllocInitMultiFab(B_external_particle_field[lev][2], amrex::convert(ba, Bfield_aux[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "B_external_particle_field[z]", 0.0_rt); + } + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { + // These fields will be added directly to the grid, i.e. to fp, and need to match the index type + AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Efield_fp[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); + AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Efield_fp[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); + AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Efield_fp[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + // These fields will be added to the fields that the particles see, and need to match the index type + AllocInitMultiFab(E_external_particle_field[lev][0], amrex::convert(ba, Efield_aux[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "E_external_particle_field[x]", 0.0_rt); + AllocInitMultiFab(E_external_particle_field[lev][1], amrex::convert(ba, Efield_aux[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "E_external_particle_field[y]", 0.0_rt); + AllocInitMultiFab(E_external_particle_field[lev][2], amrex::convert(ba, Efield_aux[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "E_external_particle_field[z]", 0.0_rt); + } + // // The coarse patch // diff --git a/requirements.txt b/requirements.txt index 60f14e1282d..bb0d8ebb704 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ periodictable~=1.5 # PICMI # note: don't forget to update the version in Docs/requirements.txt, too -picmistandard==0.28.0 +picmistandard==0.29.0 # for development against an unreleased PICMI version, use: #picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python From bca3e9af7228a9d3c928fbf67d443759882d2436 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 10 Jul 2024 20:09:54 +0200 Subject: [PATCH 148/190] Reset collision checksums and add particle data (#5042) * Turn on particle comparison * New value for checksum --- Regression/Checksum/benchmarks_json/collisionXYZ.json | 4 ++-- Regression/WarpX-tests.ini | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 7436302a99e..417e8a0b144 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,7 +6,7 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 351570.64632548566, - "T_ion": 350085.3998917431 + "T_electron": 353384.4616461907, + "T_ion": 348277.2774202612 } } diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 3d1dbfe3b0f..fdb4d14a75f 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -256,7 +256,7 @@ useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 -compareParticles = 0 +compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_rz.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -274,7 +274,7 @@ useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 -compareParticles = 0 +compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_3d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -292,7 +292,7 @@ useOMP = 1 numthreads = 1 compileTest = 0 doVis = 0 -compareParticles = 0 +compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_2d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py From 45278d772468779bad0b05c9b515a2c7f3482d9d Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:03:58 -0700 Subject: [PATCH 149/190] CI: fix dimensionality of test `Python_ohms_law_solver_EM_modes_rz` (#5037) --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index fdb4d14a75f..82e628a2bd4 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3571,7 +3571,7 @@ buildDir = . inputFile = Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py runtime_params = warpx.abort_on_warning_threshold = medium customRunCmd = python3 PICMI_inputs_rz.py --test -dim = 1 +dim = 2 addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_APP=OFF -DWarpX_QED=OFF -DWarpX_PYTHON=ON target = pip_install From a0ce802dd7fc323477800aa55af63f390dcf5f32 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 10 Jul 2024 23:10:01 +0200 Subject: [PATCH 150/190] Specify the unit of the potential, in the documentation (#5035) --- Docs/source/usage/parameters.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index d6420440935..172a9cc7cb8 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -500,7 +500,7 @@ Domain Boundary Conditions * ``open``: For the electrostatic Poisson solver based on a Integrated Green Function method. * ``boundary.potential_lo_x/y/z`` and ``boundary.potential_hi_x/y/z`` (default `0`) - Gives the value of the electric potential at the boundaries, for ``pec`` boundaries. With electrostatic solvers + Gives the value of the electric potential, in Volts, at the boundaries, for ``pec`` boundaries. With electrostatic solvers (i.e., with ``warpx.do_electrostatic = ...``), this is used in order to compute the potential in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver), setting these variables will trigger an electrostatic solve at ``t=0``, to compute the initial @@ -603,7 +603,7 @@ Whether the embedded boundary is defined with an analytical function or an STL f additionally define the electric potential at the embedded boundary with an analytical function: * ``warpx.eb_potential(x,y,z,t)`` (`string`) - Gives the value of the electric potential at the surface of the embedded boundary, + Gives the value of the electric potential, in Volts, at the surface of the embedded boundary, as a function of `x`, `y`, `z` and `t`. With electrostatic solvers (i.e., with ``warpx.do_electrostatic = ...``), this is used in order to compute the potential in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver), From 5f6715148bd46f160851a36aa7c1f9d293799590 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 11 Jul 2024 05:23:03 +0200 Subject: [PATCH 151/190] Docs: correct formatting of `gaussian_parse_momentum_function` (#5038) --- Docs/source/usage/parameters.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 172a9cc7cb8..4a8c4be0b5f 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -1156,8 +1156,8 @@ Particle initialization * ``gaussian_parse_momentum_function``: Gaussian momentum distribution where the mean and the standard deviation are given by functions of position in the input file. Both are assumed to be non-relativistic. The mean is the normalized momentum, :math:`u_m = \gamma v_m/c`. - The standard deviation is normalized, :math:`u_th = v_th/c`. - For example, this might be `u_th = sqrt(T*q_e/mass)/clight` given the temperature (in eV) and mass. + The standard deviation is normalized, :math:`u_{th} = v_{th}/c`. + For example, this might be ``u_th = sqrt(T*q_e/mass)/clight`` given the temperature (in eV) and mass. It requires the following arguments: * ``.momentum_function_ux_m(x,y,z)``: mean :math:`u_{x}` From ade19e5f10451f86c8a5f6817ffa6b56194e701b Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 10 Jul 2024 21:10:03 -0700 Subject: [PATCH 152/190] Remove work-around in regression testing script (#5034) --- run_test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/run_test.sh b/run_test.sh index 37264bef243..2ca880833b0 100755 --- a/run_test.sh +++ b/run_test.sh @@ -80,8 +80,6 @@ cd - # Clone the AMReX regression test utility git clone https://github.com/AMReX-Codes/regression_testing.git -# FIXME: https://github.com/AMReX-Codes/regression_testing/issues/136 -cd regression_testing && git checkout 93ddfb11456f47d6555c39388ba1a4ead61fbf4e && cd - # Prepare regression tests mkdir -p rt-WarpX/WarpX-benchmarks From 892f65b30c448e71737acd9f970e7b1f59d7d596 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 10 Jul 2024 21:11:34 -0700 Subject: [PATCH 153/190] CI: restore input for test `Langmuir_multi_nodal` (#5039) --- Regression/WarpX-tests.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 82e628a2bd4..a4cf87a53dd 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -1509,7 +1509,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_nodal] buildDir = . -inputFile = +inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = warpx.grid_type=collocated algo.current_deposition=direct dim = 3 addToCompileString = From f5ae2ec3267c07b15a5611b4c4cd08aac88d7c7d Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Thu, 11 Jul 2024 05:45:46 -0700 Subject: [PATCH 154/190] fixed ComputeTemperature routine called from Coulomb collisions to work for particles of varying weight. (#5040) --- .../BinaryCollision/Coulomb/ComputeTemperature.H | 15 +++++++++------ .../Coulomb/ElasticCollisionPerez.H | 4 ++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H index 04440288ba3..6d0b48615a4 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H @@ -14,6 +14,7 @@ template AMREX_GPU_HOST_DEVICE T_R ComputeTemperature ( T_index const Is, T_index const Ie, T_index const * AMREX_RESTRICT I, + T_R const * AMREX_RESTRICT w, T_R const * AMREX_RESTRICT ux, T_R const * AMREX_RESTRICT uy, T_R const * AMREX_RESTRICT uz, T_R const m ) { @@ -26,6 +27,7 @@ T_R ComputeTemperature ( T_R vx = T_R(0.0); T_R vy = T_R(0.0); T_R vz = T_R(0.0); T_R vs = T_R(0.0); T_R gm = T_R(0.0); T_R us = T_R(0.0); + T_R wtot = T_R(0.0); for (int i = Is; i < static_cast(Ie); ++i) { @@ -33,14 +35,15 @@ T_R ComputeTemperature ( uy[ I[i] ] * uy[ I[i] ] + uz[ I[i] ] * uz[ I[i] ] ); gm = std::sqrt( T_R(1.0) + us*inv_c2 ); - vx += ux[ I[i] ] / gm; - vy += uy[ I[i] ] / gm; - vz += uz[ I[i] ] / gm; - vs += us / gm / gm; + wtot += w[ I[i] ]; + vx += w[ I[i] ] * ux[ I[i] ] / gm; + vy += w[ I[i] ] * uy[ I[i] ] / gm; + vz += w[ I[i] ] * uz[ I[i] ] / gm; + vs += w[ I[i] ] * us / gm / gm; } - vx = vx / N; vy = vy / N; - vz = vz / N; vs = vs / N; + vx = vx / wtot; vy = vy / wtot; + vz = vz / wtot; vs = vs / wtot; return m/T_R(3.0)*(vs-(vx*vx+vy*vy+vz*vz)); } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index af4e6ba38c9..a782fac5f6e 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -74,12 +74,12 @@ void ElasticCollisionPerez ( T_PR T1t; T_PR T2t; if ( T1 <= T_PR(0.0) && L <= T_PR(0.0) ) { - T1t = ComputeTemperature(I1s,I1e,I1,u1x,u1y,u1z,m1); + T1t = ComputeTemperature(I1s,I1e,I1,w1,u1x,u1y,u1z,m1); } else { T1t = T1; } if ( T2 <= T_PR(0.0) && L <= T_PR(0.0) ) { - T2t = ComputeTemperature(I2s,I2e,I2,u2x,u2y,u2z,m2); + T2t = ComputeTemperature(I2s,I2e,I2,w2,u2x,u2y,u2z,m2); } else { T2t = T2; } From a305fc6e5c031f2e1285f25300c73a86d96a6404 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 11 Jul 2024 17:36:38 +0200 Subject: [PATCH 155/190] Use latest PICMI version in documentation (#5044) --- Docs/requirements.txt | 2 +- Python/setup.py | 2 +- Tools/machines/karolina-it4i/install_dependencies.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Docs/requirements.txt b/Docs/requirements.txt index b16bf161144..1ce1ee6c1a0 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -13,7 +13,7 @@ openpmd-viewer # for checksumAPI # PICMI API docs # note: keep in sync with version in ../requirements.txt -picmistandard==0.28.0 +picmistandard==0.29.0 # for development against an unreleased PICMI version, use: # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python diff --git a/Python/setup.py b/Python/setup.py index e4adbb6c458..31f35eeceac 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -59,7 +59,7 @@ package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", package_data = package_data, - install_requires = ['numpy', 'picmistandard==0.28.0', 'periodictable'], + install_requires = ['numpy', 'picmistandard==0.29.0', 'periodictable'], python_requires = '>=3.8', zip_safe=False ) diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh index 0435b5e2926..cba455f3d29 100755 --- a/Tools/machines/karolina-it4i/install_dependencies.sh +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -53,7 +53,7 @@ python -m pip install --user --upgrade matplotlib #python -m pip install --user --upgrade yt # install or update WarpX dependencies -python -m pip install --user --upgrade picmistandard==0.28.0 +python -m pip install --user --upgrade picmistandard==0.29.0 python -m pip install --user --upgrade lasy # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) From 63bedffff42a11b073cca3d708f4910dff60a997 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:28:03 -0700 Subject: [PATCH 156/190] CI: remove unused params, check particles in `collisionXYZ` (#5043) * CI: remove unused params, check particles in `collisionXYZ` * Update benchmark --------- Co-authored-by: Remi Lehe --- .../Tests/collision/analysis_collision_3d.py | 2 +- .../benchmarks_json/collisionXYZ.json | 20 +- Regression/WarpX-tests.ini | 866 ------------------ run_test.sh | 4 +- 4 files changed, 22 insertions(+), 870 deletions(-) diff --git a/Examples/Tests/collision/analysis_collision_3d.py b/Examples/Tests/collision/analysis_collision_3d.py index ee0c81d8f75..86a434caab2 100755 --- a/Examples/Tests/collision/analysis_collision_3d.py +++ b/Examples/Tests/collision/analysis_collision_3d.py @@ -111,4 +111,4 @@ dim, species_name) test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 417e8a0b144..5ec1f21c32a 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -8,5 +8,23 @@ "Ez": 0.0, "T_electron": 353384.4616461907, "T_ion": 348277.2774202612 + }, + "electron": { + "particle_momentum_x": 8.378820415249081e-19, + "particle_momentum_y": 8.238482720534644e-19, + "particle_momentum_z": 8.189221038447925e-19, + "particle_position_x": 21268719.554914076, + "particle_position_y": 21282437.437205918, + "particle_position_z": 21231947.47401523, + "particle_weight": 7.168263344048695e+28 + }, + "ion": { + "particle_momentum_x": 2.013855005436386e-18, + "particle_momentum_y": 1.8231305152698165e-18, + "particle_momentum_z": 1.8194149390229157e-18, + "particle_position_x": 21217737.157987915, + "particle_position_y": 21263096.30516862, + "particle_position_z": 21261343.069983635, + "particle_weight": 7.168263344048695e+28 } -} +} \ No newline at end of file diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index a4cf87a53dd..6ee9820aa28 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -81,10 +81,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [averaged_galilean_2d_psatd_hybrid] @@ -99,10 +95,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [averaged_galilean_3d_psatd] @@ -117,10 +109,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [averaged_galilean_3d_psatd_hybrid] @@ -135,10 +123,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [background_mcc] @@ -153,10 +137,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons he_ions analysisRoutine = Examples/analysis_default_regression.py [background_mcc_dp_psp] @@ -171,10 +151,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons he_ions analysisRoutine = Examples/analysis_default_regression.py [bilinear_filter] @@ -189,8 +165,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/single_particle/analysis_bilinear_filter.py [BTD_rz] @@ -205,8 +179,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/btd_rz/analysis_BTD_laser_antenna.py [collider_diagnostics] @@ -221,8 +193,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py [collisionISO] @@ -237,8 +207,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/collision/analysis_collision_3d_isotropization.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -254,9 +222,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_rz.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -272,9 +237,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_3d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -290,9 +252,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/collision/analysis_collision_2d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -308,10 +267,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions beam analysisRoutine = Examples/analysis_default_regression.py [Deuterium_Deuterium_Fusion_3D] @@ -326,8 +281,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [Deuterium_Deuterium_Fusion_3D_intraspecies] @@ -342,8 +295,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py [Deuterium_Tritium_Fusion_3D] @@ -358,8 +309,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [Deuterium_Tritium_Fusion_RZ] @@ -374,8 +323,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [dirichletbc] @@ -390,9 +337,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py [divb_cleaning_3d] @@ -407,9 +351,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/divb_cleaning/analysis.py [dive_cleaning_2d] @@ -423,9 +364,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/dive_cleaning/analysis.py analysisOutputImage = Comparison.png @@ -441,9 +379,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/dive_cleaning/analysis.py analysisOutputImage = Comparison.png @@ -460,9 +395,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereLabFrame_MR_emass_10] @@ -477,9 +409,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereEB] @@ -494,9 +423,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis.py [ElectrostaticSphereEB_mixedBCs] @@ -511,9 +437,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [EmbeddedBoundaryDiffraction] @@ -528,9 +451,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 outputFile = EmbeddedBoundaryDiffraction_plt analysisRoutine = Examples/Tests/embedded_boundary_diffraction/analysis_fields.py @@ -546,9 +466,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereEB_RZ_MR] @@ -563,9 +480,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereLabFrame] @@ -580,9 +494,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereRZ] @@ -597,9 +508,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereRelNodal] @@ -614,9 +522,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [embedded_boundary_cube] @@ -631,9 +536,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields.py [embedded_boundary_cube_2d] @@ -648,9 +550,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py [embedded_boundary_cube_macroscopic] @@ -665,9 +564,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields.py [embedded_boundary_python_API] @@ -684,9 +580,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_python_api/analysis.py [embedded_boundary_rotated_cube] @@ -701,9 +594,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py [embedded_boundary_rotated_cube_2d] @@ -718,9 +608,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py [embedded_circle] @@ -735,10 +622,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ar_ions analysisRoutine = Examples/Tests/embedded_circle/analysis.py [FieldProbe] @@ -753,9 +636,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/field_probe/analysis_field_probe.py [FluxInjection] @@ -770,10 +650,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/flux_injection/analysis_flux_injection_rz.py [FluxInjection3D] @@ -788,9 +664,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/flux_injection/analysis_flux_injection_3d.py [galilean_2d_psatd] @@ -805,10 +678,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_2d_psatd_current_correction] @@ -823,10 +692,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_2d_psatd_current_correction_psb] @@ -841,10 +706,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_2d_psatd_hybrid] @@ -859,10 +720,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions beam analysisRoutine = Examples/analysis_default_regression.py [galilean_3d_psatd] @@ -877,10 +734,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_3d_psatd_current_correction] @@ -895,10 +748,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_3d_psatd_current_correction_psb] @@ -913,10 +762,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_rz_psatd] @@ -931,10 +776,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_rz_psatd_current_correction_psb] @@ -949,10 +790,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_rz_psatd_current_correction] @@ -967,10 +804,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [hard_edged_plasma_lens] @@ -985,10 +818,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [hard_edged_quadrupoles] @@ -1003,10 +832,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py [hard_edged_quadrupoles_boosted] @@ -1021,10 +846,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py [hard_edged_quadrupoles_moving] @@ -1039,10 +860,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py [initial_distribution] @@ -1057,9 +874,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/initial_distribution/analysis_distribution.py aux1File = Tools/PostProcessing/read_raw_data.py @@ -1075,8 +889,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/ionization/analysis_ionization.py [ionization_lab] @@ -1091,8 +903,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/ionization/analysis_ionization.py [ion_stopping] @@ -1107,9 +917,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ion_stopping/analysis_ion_stopping.py [Langmuir_multi] @@ -1124,10 +931,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1143,9 +946,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_1d.py analysisOutputImage = langmuir_fluid_multi_1d_analysis.png @@ -1161,9 +961,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_rz.py analysisOutputImage = langmuir_fluid_rz_analysis.png @@ -1179,9 +976,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_2d.py analysisOutputImage = langmuir_fluid_multi_2d_analysis.png @@ -1197,9 +991,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_3d.py analysisOutputImage = langmuir_fluid_multi_analysis.png @@ -1215,10 +1006,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_1d.py analysisOutputImage = langmuir_multi_1d_analysis.png @@ -1234,10 +1021,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_MR.png @@ -1253,10 +1036,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_MR.png @@ -1272,10 +1051,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_MR_momentum_conserving.png @@ -1291,10 +1066,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_MR_psatd.png @@ -1310,10 +1081,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1329,10 +1096,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1348,10 +1111,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1367,10 +1126,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1386,10 +1141,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1405,10 +1156,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_psatd_multiJ.png @@ -1424,10 +1171,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = Langmuir_multi_2d_psatd_multiJ_nodal.png @@ -1443,10 +1186,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1462,10 +1201,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1481,10 +1216,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1500,10 +1231,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py analysisOutputImage = langmuir_multi_2d_analysis.png @@ -1519,10 +1246,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1538,10 +1261,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1557,10 +1276,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1576,10 +1291,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1595,10 +1306,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1614,10 +1321,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1633,10 +1336,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = Langmuir_multi_psatd_multiJ.png @@ -1652,10 +1351,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = Langmuir_multi_psatd_multiJ_nodal.png @@ -1671,10 +1366,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1690,10 +1381,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1709,10 +1396,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1728,10 +1411,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1747,10 +1426,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py analysisOutputImage = Langmuir_multi_rz_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -1767,10 +1442,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py analysisOutputImage = Langmuir_multi_rz_psatd_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -1787,10 +1458,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py analysisOutputImage = Langmuir_multi_rz_psatd_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -1807,10 +1474,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py analysisOutputImage = Langmuir_multi_rz_psatd_multiJ_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -1827,10 +1490,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py analysisOutputImage = langmuir_multi_analysis.png @@ -1846,8 +1505,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [LaserAcceleration] @@ -1862,10 +1519,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons outputFile = LaserAcceleration_plt analysisRoutine = Examples/analysis_default_openpmd_regression.py @@ -1881,10 +1534,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [LaserAcceleration_1d_fluid] @@ -1899,10 +1548,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py [LaserAcceleration_1d_fluid_boosted] @@ -1917,10 +1562,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py [LaserAccelerationBoost] @@ -1935,8 +1576,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [LaserAcceleration_BTD] @@ -1951,10 +1590,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/boosted_diags/analysis.py [LaserAccelerationMR] @@ -1969,10 +1604,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [LaserAccelerationRZ] @@ -1987,10 +1618,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [LaserAccelerationRZ_opmd] @@ -2005,10 +1632,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons beam outputFile = LaserAccelerationRZ_opmd_plt analysisRoutine = Examples/Tests/openpmd_rz/analysis_openpmd_rz.py @@ -2024,10 +1647,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons outputFile = LaserAcceleration_single_precision_comms_plt analysisRoutine = Examples/analysis_default_openpmd_regression.py @@ -2043,9 +1662,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/laser_injection/analysis_laser.py analysisOutputImage = laser_analysis.png @@ -2061,9 +1677,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/laser_injection/analysis_1d.py [LaserInjection_2d] @@ -2078,9 +1691,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/laser_injection/analysis_2d.py [LaserInjectionFromBINARYFile] @@ -2097,10 +1707,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile] buildDir = . @@ -2116,10 +1724,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_1d] buildDir = . @@ -2135,10 +1741,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_1d_boost] buildDir = . @@ -2154,10 +1758,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_2d] buildDir = . @@ -2173,10 +1775,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_RZ] buildDir = . @@ -2192,10 +1792,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromRZLASYFile] buildDir = . @@ -2211,10 +1809,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserIonAcc2d] buildDir = . @@ -2229,8 +1825,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserOnFine] @@ -2245,8 +1839,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [leveling_thinning] @@ -2261,9 +1853,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/resampling/analysis_leveling_thinning.py [LoadExternalFieldRZGrid] @@ -2279,10 +1868,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = proton analysisRoutine = Examples/Tests/LoadExternalField/analysis_rz.py [LoadExternalFieldRZParticles] @@ -2298,10 +1883,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = proton analysisRoutine = Examples/Tests/LoadExternalField/analysis_rz.py [magnetostatic_eb_3d] @@ -2316,9 +1897,6 @@ useMPI = 1 numprocs = 1 useOMP = 2 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [Maxwell_Hybrid_QED_solver] @@ -2333,8 +1911,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/maxwell_hybrid_qed/analysis_Maxwell_QED_Hybrid.py [momentum-conserving-gather] @@ -2349,10 +1925,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam driver plasma_e analysisRoutine = Examples/analysis_default_regression.py [multi_J_rz_psatd] @@ -2367,10 +1939,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = driver plasma_e plasma_p analysisRoutine = Examples/analysis_default_regression.py [nci_corrector] @@ -2385,9 +1953,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -doComparison = 0 analysisRoutine = Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py [nci_correctorMR] @@ -2402,9 +1967,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -doComparison = 0 analysisRoutine = Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py [parabolic_channel_initialization_2d_single_precision] @@ -2418,9 +1980,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/initial_plasma_profile/analysis.py @@ -2436,10 +1995,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/particle_boundary_process/analysis_absorption.py [particle_boundaries_3d] @@ -2454,9 +2009,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/boundaries/analysis.py [particle_fields_diags] @@ -2472,9 +2024,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_fields_diags/analysis_particle_diags.py [particle_fields_diags_single_precision] @@ -2490,9 +2039,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_fields_diags/analysis_particle_diags_single.py [particle_pusher] @@ -2507,9 +2053,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_pusher/analysis_pusher.py [particle_scrape] @@ -2524,10 +2067,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/particle_boundary_scrape/analysis_scrape.py [particles_in_pml] @@ -2542,9 +2081,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [particles_in_pml_2d] @@ -2559,9 +2095,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [particles_in_pml_2d_MR] @@ -2576,9 +2109,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [particles_in_pml_3d_MR] @@ -2593,9 +2123,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [PEC_field] @@ -2610,9 +2137,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/pec/analysis_pec.py [PEC_field_mr] @@ -2627,9 +2151,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/pec/analysis_pec_mr.py [PEC_particle] @@ -2644,10 +2165,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron proton analysisRoutine = Examples/analysis_default_regression.py [Performance_works_1_uniform_rest_32ppc] @@ -2662,10 +2179,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_2_uniform_rest_1ppc] @@ -2680,10 +2193,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Performance_works_3_uniform_drift_4ppc] @@ -2698,10 +2207,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_4_labdiags_2ppc] @@ -2716,10 +2221,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_5_loadimbalance] @@ -2734,10 +2235,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_6_output_2ppc] @@ -2752,10 +2249,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [photon_pusher] @@ -2770,9 +2263,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/photon_pusher/analysis_photon_pusher.py [PlasmaAccelerationBoost2d] @@ -2787,8 +2277,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationBoost3d] @@ -2803,8 +2291,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationBoost3d_hybrid] @@ -2818,8 +2304,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationMR] @@ -2834,10 +2318,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam driver plasma_e analysisRoutine = Examples/analysis_default_regression.py [Plasma_lens] @@ -2852,10 +2332,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [Plasma_lens_boosted] @@ -2870,10 +2346,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [Plasma_lens_short] @@ -2888,10 +2360,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [PlasmaMirror] @@ -2906,8 +2374,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [pml_psatd_dive_divb_cleaning] @@ -2922,8 +2388,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [pml_psatd_rz] @@ -2938,8 +2402,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_psatd_rz.py [pml_x_ckc] @@ -2954,8 +2416,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_ckc.py [pml_x_galilean] @@ -2970,8 +2430,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py [pml_x_psatd] @@ -2987,8 +2445,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py [pml_x_yee] @@ -3004,8 +2460,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py [pml_x_yee_eb] @@ -3021,8 +2475,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py [Proton_Boron_Fusion_2D] @@ -3037,8 +2489,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py [Proton_Boron_Fusion_3D] @@ -3053,8 +2503,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py [Python_background_mcc] @@ -3071,9 +2519,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_2d.py [Python_background_mcc_1d] @@ -3090,9 +2535,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py [Python_background_mcc_1d_tridiag] @@ -3109,9 +2551,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py [Python_collisionXZ] @@ -3128,9 +2567,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/collision/analysis_collision_2d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -3148,9 +2584,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py [Python_dsmc_1d] @@ -3167,9 +2600,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py [Python_ElectrostaticSphereEB] @@ -3186,9 +2616,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis.py [Python_gaussian_beam] @@ -3205,10 +2632,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_gaussian_beam_no_field_output] @@ -3225,10 +2648,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Python_gaussian_beam_opmd] @@ -3245,10 +2664,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Python_gaussian_beam_opmd_no_field_output] @@ -3265,10 +2680,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Python_id_cpu_read] @@ -3285,10 +2696,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons [Python_ionization] buildDir = . @@ -3304,8 +2711,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/ionization/analysis_ionization.py [Python_Langmuir] @@ -3322,10 +2727,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_Langmuir_2d] @@ -3342,10 +2743,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_Langmuir_rz_multimode] @@ -3362,10 +2759,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons protons analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAcceleration] @@ -3382,10 +2775,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAcceleration_1d] @@ -3402,10 +2791,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAccelerationMR] @@ -3422,10 +2807,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAccelerationRZ] @@ -3442,10 +2823,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [Python_LaserIonAcc2d] @@ -3463,8 +2840,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_openpmd_regression.py [Python_LoadExternalGridField3D] @@ -3481,10 +2856,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = proton analysisRoutine = Examples/Tests/LoadExternalField/analysis_3d.py [Python_LoadExternalParticleField3D] @@ -3501,10 +2872,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = proton analysisRoutine = Examples/Tests/LoadExternalField/analysis_3d.py [Python_magnetostatic_eb_3d] @@ -3521,10 +2888,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 2 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_magnetostatic_eb_rz] @@ -3541,10 +2904,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 2 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons analysisRoutine = Examples/Tests/magnetostatic_eb/analysis_rz.py [Python_ohms_law_solver_EM_modes_1d] @@ -3561,9 +2920,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_EM_modes/analysis.py [Python_ohms_law_solver_EM_modes_rz] @@ -3580,9 +2936,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_EM_modes/analysis_rz.py [Python_ohms_law_solver_ion_beam_1d] @@ -3599,9 +2952,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_ion_beam_instability/analysis.py [Python_ohms_law_solver_landau_damping_2d] @@ -3618,9 +2968,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py [Python_ohms_law_solver_magnetic_reconnection_2d] @@ -3637,9 +2984,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py [Python_particle_attr_access] @@ -3656,8 +3000,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/particle_data_python/analysis.py [Python_particle_attr_access_unique] @@ -3674,8 +3016,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/particle_data_python/analysis.py [Python_particle_reflection] @@ -3692,8 +3032,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/particle_boundary_process/analysis_reflection.py [Python_particle_scrape] @@ -3710,10 +3048,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/particle_boundary_scrape/analysis_scrape.py # TODO: Enable in pyAMReX, then enable lines in PICMI_inputs_2d.py again @@ -3732,10 +3066,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons # TODO: comment in again once enabled #analysisRoutine = Examples/Tests/pass_mpi_communicator/analysis.py @@ -3753,10 +3083,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam analysisRoutine = Examples/analysis_default_regression.py [Python_PlasmaAcceleration1d] @@ -3773,10 +3099,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam analysisRoutine = Examples/analysis_default_regression.py [Python_PlasmaAccelerationMR] @@ -3793,10 +3115,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam analysisRoutine = Examples/analysis_default_regression.py [Python_plasma_lens] @@ -3813,10 +3131,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [Python_prev_positions] @@ -3833,9 +3147,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/analysis_default_regression.py [Python_reduced_diags_loadbalancecosts_timers] @@ -3852,9 +3163,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [Python_restart_eb] @@ -3872,10 +3180,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/restart/analysis_restart.py [Python_restart_runtime_components] @@ -3893,10 +3197,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons [Python_wrappers] buildDir = . @@ -3912,9 +3212,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [qed_breit_wheeler_2d] @@ -3930,9 +3227,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_yt.py [qed_breit_wheeler_2d_opmd] @@ -3948,9 +3242,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 outputFile = qed_breit_wheeler_2d_opmd_plt analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_opmd.py @@ -3967,9 +3258,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_yt.py [qed_breit_wheeler_3d_opmd] @@ -3985,9 +3273,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 outputFile = qed_breit_wheeler_3d_opmd_plt analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_opmd.py @@ -4003,9 +3288,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/quantum_synchrotron/analysis.py [qed_quantum_sync_3d] @@ -4020,9 +3302,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/quantum_synchrotron/analysis.py [qed_schwinger1] @@ -4037,8 +3316,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [qed_schwinger2] @@ -4053,8 +3330,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [qed_schwinger3] @@ -4069,8 +3344,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [qed_schwinger4] @@ -4085,8 +3358,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [radiation_reaction] @@ -4101,9 +3372,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py [reduced_diags] @@ -4119,9 +3387,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags.py [reduced_diags_loadbalancecosts_heuristic] @@ -4136,9 +3401,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [reduced_diags_loadbalancecosts_timers] @@ -4153,9 +3415,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [reduced_diags_loadbalancecosts_timers_psatd] @@ -4170,9 +3429,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [reduced_diags_single_precision] @@ -4188,9 +3444,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_single.py [RefinedInjection] @@ -4205,10 +3458,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py [relativistic_space_charge_initialization] @@ -4222,9 +3471,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/relativistic_space_charge_initialization/analysis.py analysisOutputImage = Comparison.png @@ -4241,8 +3487,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/repelling_particles/analysis_repelling.py [resample_velocity_coincidence_thinning] @@ -4257,9 +3501,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [resample_velocity_coincidence_thinning_cartesian] @@ -4274,9 +3515,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [restart] @@ -4292,10 +3530,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = beam analysisRoutine = Examples/Tests/restart/analysis_restart.py [restart_psatd] @@ -4311,10 +3545,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = beam analysisRoutine = Examples/Tests/restart/analysis_restart.py [restart_psatd_time_avg] @@ -4330,10 +3560,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = beam analysisRoutine = Examples/Tests/restart/analysis_restart.py [RigidInjection_BTD] @@ -4348,10 +3574,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py [RigidInjection_lab] @@ -4366,9 +3588,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/rigid_injection/analysis_rigid_injection_LabFrame.py [scraping] @@ -4383,9 +3602,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/scraping/analysis_rz.py [scraping_filter] @@ -4400,9 +3616,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/scraping/analysis_rz_filter.py [silver_mueller_1d] @@ -4417,8 +3630,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [silver_mueller_2d_x] @@ -4433,8 +3644,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [silver_mueller_2d_z] @@ -4449,8 +3658,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [silver_mueller_rz_z] @@ -4465,8 +3672,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [space_charge_initialization] @@ -4480,9 +3685,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py analysisOutputImage = Comparison.png @@ -4498,9 +3700,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py analysisOutputImage = Comparison.png @@ -4517,9 +3716,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [Uniform_2d] @@ -4534,8 +3730,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [uniform_plasma_restart] @@ -4551,10 +3745,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons analysisRoutine = Examples/Tests/restart/analysis_restart.py [uniform_plasma_multiJ] @@ -4569,10 +3759,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_multiJ.py [VayDeposition2D] @@ -4587,10 +3773,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron ion analysisRoutine = Examples/Tests/vay_deposition/analysis.py [VayDeposition3D] @@ -4605,10 +3787,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron ion analysisRoutine = Examples/Tests/vay_deposition/analysis.py [NodalElectrostaticSolver] @@ -4623,9 +3801,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/nodal_electrostatic/analysis_3d.py [BeamBeamCollision] @@ -4640,8 +3815,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 outputFile = BeamBeamCollision_plt analysisRoutine = Examples/analysis_default_openpmd_regression.py @@ -4659,10 +3832,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons protons outputFile = spacecraft_charging_plt analysisRoutine = Examples/Physics_applications/spacecraft_charging/analysis.py analysisOutputImage = min_phi_analysis.png @@ -4679,10 +3848,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons outputFile = Point_of_contact_EB_3d_plt analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py @@ -4698,10 +3863,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons outputFile = Point_of_contact_EB_rz_plt analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py @@ -4717,9 +3878,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py [ThetaImplicitJFNK_VandB_2d] @@ -4734,9 +3892,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py [SemiImplicitPicard_1d] @@ -4751,9 +3906,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py [EnergyConservingThermalPlasma] @@ -4767,9 +3919,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/energy_conserving_thermal_plasma/analysis.py [focusing_gaussian_beam] @@ -4784,10 +3933,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/gaussian_beam/analysis_focusing_beam.py [particle_boundary_interaction] @@ -4804,10 +3949,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons outputFile = particle_boundary_interaction_plt analysisRoutine = Examples/Tests/particle_boundary_interaction/analysis.py @@ -4823,9 +3964,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py [openbc_poisson_solver] @@ -4840,8 +3978,4 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py diff --git a/run_test.sh b/run_test.sh index 2ca880833b0..21e84c7030d 100755 --- a/run_test.sh +++ b/run_test.sh @@ -94,10 +94,10 @@ cd ../../regression_testing/ echo "cd $PWD" # run only tests specified in variable tests_arg (single test or multiple tests) if [[ ! -z "${tests_arg}" ]]; then - python3 regtest.py ../rt-WarpX/ci-tests.ini --no_update all "${tests_run}" + python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all "${tests_run}" # run all tests (variables tests_arg and tests_run are empty) else - python3 regtest.py ../rt-WarpX/ci-tests.ini --no_update all + python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all fi # clean up python virtual environment From dc087bf18b1e3d2367acdf86ea44ee3fc41d1755 Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Thu, 11 Jul 2024 21:52:40 -0700 Subject: [PATCH 157/190] shuffling the full list of particles for intra-species binary collisions (#5045) * shuffling the full list of particles for intra-species binary collisions * fixing merge issue with collision json files. * moved checksum to before error assert for intraspecies DD fusion regression test analysis script. * updating benchmark values for DD_fusion_3D_intraspecies. * adding fixed random seed to DD_3D_intraspecies fusion regression test. * putting checksum back after assert. * adjusting json. * updating json. --- ...sis_deuterium_deuterium_3d_intraspecies.py | 4 +- ...inputs_deuterium_deuterium_3d_intraspecies | 2 + ...rium_Deuterium_Fusion_3D_intraspecies.json | 48 +++++++++---------- .../benchmarks_json/collisionISO.json | 12 ++--- .../benchmarks_json/collisionXYZ.json | 30 ++++++------ .../BinaryCollision/BinaryCollision.H | 4 +- 6 files changed, 51 insertions(+), 49 deletions(-) diff --git a/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py b/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py index 70178fe97a6..858df0b26a6 100755 --- a/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py +++ b/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py @@ -47,7 +47,9 @@ sigma_th = 2.603e-18 error = np.abs(sigma-sigma_th)/sigma_th -tolerance = 1e-2 +tolerance = 2e-2 +print('error = ', error) +print('tolerance = ', tolerance) assert error < tolerance # Compare checksums with benchmark diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies b/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies index 985cc2d7612..0030ca9b945 100644 --- a/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies +++ b/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies @@ -8,6 +8,8 @@ # plasma simulations. Journal of Computational Physics, 388, pp.439-453. # DOI: https://doi.org/10.1016/j.jcp.2019.03.020 +warpx.random_seed = 1034958209 + # algo algo.particle_shape = 3 diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json index 59818f62826..45e29874cda 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json @@ -1,34 +1,34 @@ { "lev=0": { "rho": 0.0, - "rho_deuterium": 8203144355.71195, - "rho_helium3": 10.368009592276463 + "rho_deuterium": 8203144355.767546, + "rho_helium3": 10.312416024595505 }, "neutron": { - "particle_momentum_x": 2.2543499835759282e-15, - "particle_momentum_y": 2.2526527390783875e-15, - "particle_momentum_z": 2.2619641737859965e-15, - "particle_position_x": 61.961041864660686, - "particle_position_y": 61.78141653674165, - "particle_position_z": 61.741022731492514, - "particle_weight": 505562702.7678892 + "particle_momentum_x": 2.260586487910896e-15, + "particle_momentum_y": 2.2564723094408887e-15, + "particle_momentum_z": 2.263164930227645e-15, + "particle_position_x": 62.03084683414219, + "particle_position_y": 61.95011296352698, + "particle_position_z": 62.02404756675538, + "particle_weight": 502851860.91505826 }, "deuterium": { - "particle_momentum_x": 1.3370046499332103e-14, - "particle_momentum_y": 1.3364310231320824e-14, - "particle_momentum_z": 1.3372728873714894e-14, - "particle_position_x": 2560.1613417364665, - "particle_position_y": 2560.082464065988, - "particle_position_z": 2560.0018477161034, - "particle_weight": 7.999999989888742e+17 + "particle_momentum_x": 1.3380470537895298e-14, + "particle_momentum_y": 1.3367685546044215e-14, + "particle_momentum_z": 1.3372922616599391e-14, + "particle_position_x": 2559.782513518522, + "particle_position_y": 2559.8747993471684, + "particle_position_z": 2560.2864832238383, + "particle_weight": 7.999999989942968e+17 }, "helium3": { - "particle_momentum_x": 2.2749239620327265e-15, - "particle_momentum_y": 2.268697031603961e-15, - "particle_momentum_z": 2.278045756364995e-15, - "particle_position_x": 61.961041864660686, - "particle_position_y": 61.78141653674165, - "particle_position_z": 61.741022731492514, - "particle_weight": 505562702.7678892 + "particle_momentum_x": 2.278275200450756e-15, + "particle_momentum_y": 2.2738610200497133e-15, + "particle_momentum_z": 2.2792408973056887e-15, + "particle_position_x": 62.03084683414219, + "particle_position_y": 61.95011296352698, + "particle_position_z": 62.02404756675538, + "particle_weight": 502851860.91505826 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json index 5146837be6f..a2fd6116cb8 100644 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ b/Regression/Checksum/benchmarks_json/collisionISO.json @@ -11,12 +11,12 @@ "jz": 0.0 }, "electron": { - "particle_momentum_x": 3.579130200773753e-19, - "particle_momentum_y": 3.5788119408700804e-19, - "particle_momentum_z": 3.584163522201744e-19, - "particle_position_x": 1.024188253213835, - "particle_position_y": 1.0238795904737117, - "particle_position_z": 1.02399735048655, + "particle_momentum_x": 3.579989064013309e-19, + "particle_momentum_y": 3.5822945977746767e-19, + "particle_momentum_z": 3.579753452653627e-19, + "particle_position_x": 1.0241322532163375, + "particle_position_y": 1.0238995904625479, + "particle_position_z": 1.02402135051502, "particle_weight": 714240000000.0 } } diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 5ec1f21c32a..39df1b9eee1 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,25 +6,25 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 353384.4616461907, - "T_ion": 348277.2774202612 + "T_electron": 353604.6247926339, + "T_ion": 347976.6168136309 }, "electron": { - "particle_momentum_x": 8.378820415249081e-19, - "particle_momentum_y": 8.238482720534644e-19, - "particle_momentum_z": 8.189221038447925e-19, - "particle_position_x": 21268719.554914076, - "particle_position_y": 21282437.437205918, - "particle_position_z": 21231947.47401523, + "particle_momentum_x": 8.370755929299189e-19, + "particle_momentum_y": 8.228112213603589e-19, + "particle_momentum_z": 8.204295817378347e-19, + "particle_position_x": 21284971.94721422, + "particle_position_y": 21212829.42991966, + "particle_position_z": 21214774.536558084, "particle_weight": 7.168263344048695e+28 }, "ion": { - "particle_momentum_x": 2.013855005436386e-18, - "particle_momentum_y": 1.8231305152698165e-18, - "particle_momentum_z": 1.8194149390229157e-18, - "particle_position_x": 21217737.157987915, - "particle_position_y": 21263096.30516862, - "particle_position_z": 21261343.069983635, + "particle_momentum_x": 2.0074097598289766e-18, + "particle_momentum_y": 1.8203553942782305e-18, + "particle_momentum_z": 1.823420185235695e-18, + "particle_position_x": 21227192.857240494, + "particle_position_y": 21286501.692027714, + "particle_position_z": 21245587.6706009, "particle_weight": 7.168263344048695e+28 } -} \ No newline at end of file +} diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index a386cb3012e..84e0fef8c9c 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -368,14 +368,12 @@ public: // given by the `indices_1[cell_start_1:cell_stop_1]` index_type const cell_start_1 = cell_offsets_1[i_cell]; index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; - index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; // Do not collide if there is only one particle in the cell if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } // shuffle - ShuffleFisherYates( - indices_1, cell_start_1, cell_half_1, engine ); + ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); } ); From f9748eb2201439d2f4391eff92e05bb0dfde90ea Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 15 Jul 2024 21:53:27 +0200 Subject: [PATCH 158/190] Fix Lassen: BLAS++/LAPACK++ (#5050) Update TOSS3 install scripts. --- .../lassen-llnl/install_v100_dependencies_toss3.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh index 916986ee119..9876334159b 100644 --- a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh @@ -82,13 +82,12 @@ if [ -d ${SRC_DIR}/blaspp ] then cd ${SRC_DIR}/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi -cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 # LAPACK++ (for PSATD+RZ) @@ -96,13 +95,12 @@ if [ -d ${SRC_DIR}/lapackpp ] then cd ${SRC_DIR}/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 From b6cf99d37701ce8e42952e04e7f05aa4b6c87eb9 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 15 Jul 2024 16:07:27 -0700 Subject: [PATCH 159/190] Initialize with value all `aux`, `cax` fields (#5049) --- Source/WarpX.cpp | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 4dfebe90c67..87fc5f2633d 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2579,13 +2579,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } } } else { - AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]"); - AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]"); - AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]"); + AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]", 0.0_rt); + AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]", 0.0_rt); + AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]", 0.0_rt); - AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]"); - AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]"); - AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); + AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]", 0.0_rt); + AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]", 0.0_rt); + AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]", 0.0_rt); } // The external fields that are read from file @@ -2736,22 +2736,22 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (n_field_gather_buffer > 0 || mypc->nSpeciesGatherFromMainGrid() > 0) { if (aux_is_nodal) { BoxArray const& cnba = amrex::convert(cba,IntVect::TheNodeVector()); - AllocInitMultiFab(Bfield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[x]"); - AllocInitMultiFab(Bfield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[y]"); - AllocInitMultiFab(Bfield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[z]"); - AllocInitMultiFab(Efield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Efield_cax[x]"); - AllocInitMultiFab(Efield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Efield_cax[y]"); - AllocInitMultiFab(Efield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Efield_cax[z]"); + AllocInitMultiFab(Bfield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[x]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[y]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[z]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Efield_cax[x]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Efield_cax[y]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Efield_cax[z]", 0.0_rt); } else { // Create the MultiFabs for B - AllocInitMultiFab(Bfield_cax[lev][0], amrex::convert(cba,Bx_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[x]"); - AllocInitMultiFab(Bfield_cax[lev][1], amrex::convert(cba,By_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[y]"); - AllocInitMultiFab(Bfield_cax[lev][2], amrex::convert(cba,Bz_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[z]"); + AllocInitMultiFab(Bfield_cax[lev][0], amrex::convert(cba,Bx_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[x]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][1], amrex::convert(cba,By_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[y]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][2], amrex::convert(cba,Bz_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[z]", 0.0_rt); // Create the MultiFabs for E - AllocInitMultiFab(Efield_cax[lev][0], amrex::convert(cba,Ex_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[x]"); - AllocInitMultiFab(Efield_cax[lev][1], amrex::convert(cba,Ey_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[y]"); - AllocInitMultiFab(Efield_cax[lev][2], amrex::convert(cba,Ez_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[z]"); + AllocInitMultiFab(Efield_cax[lev][0], amrex::convert(cba,Ex_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[x]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][1], amrex::convert(cba,Ey_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[y]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][2], amrex::convert(cba,Ez_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[z]", 0.0_rt); } AllocInitMultiFab(gather_buffer_masks[lev], ba, dm, ncomps, amrex::IntVect(1), lev, "gather_buffer_masks"); From c54eff69ce658c80f9db194d19f28daf7cd315b8 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 16 Jul 2024 06:39:45 +0200 Subject: [PATCH 160/190] ABLASTR FFT: Fix Build (#5051) Use the correct ABLASTR (not WarpX) CMake options and compiler defines. E.g., for ImpactX, we only control ABLASTR, not WarpX. --- Source/ablastr/fields/CMakeLists.txt | 2 +- Source/ablastr/fields/PoissonSolver.H | 8 ++++---- Source/ablastr/math/fft/CMakeLists.txt | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/ablastr/fields/CMakeLists.txt b/Source/ablastr/fields/CMakeLists.txt index 158e623ebaf..56acc678217 100644 --- a/Source/ablastr/fields/CMakeLists.txt +++ b/Source/ablastr/fields/CMakeLists.txt @@ -1,6 +1,6 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) - if(WarpX_FFT AND D EQUAL 3) + if(ABLASTR_FFT AND D EQUAL 3) target_sources(ablastr_${SD} PRIVATE IntegratedGreenFunctionSolver.cpp diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index fed76b2987b..ca262981010 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -15,7 +15,7 @@ #include #include -#if defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D) +#if defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D) #include #endif @@ -160,9 +160,9 @@ computePhi (amrex::Vector const & rho, {{ beta[0], beta[1], beta[2] }}; #endif -#if !defined(WARPX_USE_FFT) +#if !defined(ABLASTR_USE_FFT) ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, - "Must compile with -DWarpX_FFT=ON to use the FFT solver!"); + "Must compile with FFT support to use the IGF solver!"); #endif #if !defined(WARPX_DIM_3D) @@ -170,7 +170,7 @@ computePhi (amrex::Vector const & rho, "The FFT Poisson solver is currently only implemented for 3D!"); #endif -#if (defined(WARPX_USE_FFT) && defined(WARPX_DIM_3D)) +#if (defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D)) // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected if(is_solver_igf_on_lev0 && lev==0){ amrex::Array const dx_igf diff --git a/Source/ablastr/math/fft/CMakeLists.txt b/Source/ablastr/math/fft/CMakeLists.txt index 2b9fc3d37f9..913a912e1ee 100644 --- a/Source/ablastr/math/fft/CMakeLists.txt +++ b/Source/ablastr/math/fft/CMakeLists.txt @@ -1,6 +1,6 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) - if(WarpX_FFT STREQUAL ON) + if(ABLASTR_FFT STREQUAL ON) if(WarpX_COMPUTE STREQUAL CUDA) target_sources(ablastr_${SD} PRIVATE WrapCuFFT.cpp) elseif(WarpX_COMPUTE STREQUAL HIP) From 7d0f91a27ec100a148a0d0e51ceac82ea7eb9f3c Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 17 Jul 2024 01:03:36 +0200 Subject: [PATCH 161/190] AMReX/pyAMReX/PICSAR: Weekly Update (#5054) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 190294bb0e1..e6ba115925a 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 24.07 && cd - + cd ../amrex && git checkout --detach dcb9cc0383dcc71e38dee9070574e325a812f8bf && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 0003dcff134..b32ca61c709 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 24.07 +branch = dcb9cc0383dcc71e38dee9070574e325a812f8bf [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 6ee9820aa28..6d7c57f0961 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 24.07 +branch = dcb9cc0383dcc71e38dee9070574e325a812f8bf [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 0b90b09df30..8acbd974779 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "24.07" +set(WarpX_amrex_branch "dcb9cc0383dcc71e38dee9070574e325a812f8bf" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 5c17681d4b6..a2fbd0ddee8 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "24.07" +set(WarpX_pyamrex_branch "18f0026b1dd9b2aa4869c96f74e4b77262a067d0" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 21e84c7030d..c01c3fb1176 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 24.07 && cd - +cd amrex && git checkout --detach dcb9cc0383dcc71e38dee9070574e325a812f8bf && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From a13a11d5541bc245634cdc793bf997bfa12fbcc1 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 17 Jul 2024 17:49:51 +0200 Subject: [PATCH 162/190] Doc: Lassen Doc String h5py (#5052) Funny typo. --- Tools/machines/lassen-llnl/install_v100_dependencies.sh | 2 +- Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh index 6368010ab7b..48668efcc2a 100755 --- a/Tools/machines/lassen-llnl/install_v100_dependencies.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies.sh @@ -119,7 +119,7 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools -# Older version for h4py +# Older version for h5py # https://github.com/h5py/h5py/issues/2268 python3 -m pip install --upgrade "cython<3" python3 -m pip install --upgrade numpy diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh index 9876334159b..a6d5aad32fb 100644 --- a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh @@ -119,7 +119,7 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools -# Older version for h4py +# Older version for h5py # https://github.com/h5py/h5py/issues/2268 python3 -m pip install --upgrade "cython<3" python3 -m pip install --upgrade numpy From b58343a9cf31ad2e2e0c47b2c5f0b15cc122a961 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 18 Jul 2024 01:19:18 +0200 Subject: [PATCH 163/190] Fix CMake Python IPO Control (#5055) `pybind11::lto` is only defined if `CMAKE_INTERPROCEDURAL_OPTIMIZATION` is not set in `pybind11Common.cmake`. Package managers like Spack use the latter. --- CMakeLists.txt | 23 +++++++++++++++++++---- cmake/WarpXFunctions.cmake | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 23427a00ec8..9c81a057131 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -146,10 +146,19 @@ set_default_build_type("Release") # Option to enable interprocedural optimization # (also know as "link-time optimization" or "whole program optimization") -option(WarpX_IPO "Compile WarpX with interprocedural optimization (will take more time)" OFF) +set(_WarpX_IPO_DEFAULT OFF) +set(_WarpX_PYTHON_IPO_DEFAULT ON) +if(DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(_WarpX_IPO_DEFAULT ${CMAKE_INTERPROCEDURAL_OPTIMIZATION}) + set(_WarpX_PYTHON_IPO_DEFAULT ${CMAKE_INTERPROCEDURAL_OPTIMIZATION}) +endif() +option(WarpX_IPO + "Compile WarpX with interprocedural optimization (will take more time)" + ${_WarpX_IPO_DEFAULT} +) option(WarpX_PYTHON_IPO "Compile Python bindings with interprocedural optimization (IPO) / link-time optimization (LTO)" - ON + ${_WarpX_PYTHON_IPO_DEFAULT} ) set(pyWarpX_VERSION_INFO "" CACHE STRING @@ -455,7 +464,7 @@ endif() # Interprocedural optimization (IPO) / Link-Time Optimization (LTO) if(WarpX_IPO) - enable_IPO("${_ALL_TARGETS}") + warpx_enable_IPO("${_ALL_TARGETS}") endif() # link dependencies @@ -488,7 +497,13 @@ foreach(D IN LISTS WarpX_DIMS) if(WarpX_PYTHON) target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras) if(WarpX_PYTHON_IPO) - target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto) + if(DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + warpx_enable_IPO(pyWarpX_${SD}) + else() + # conditionally defined target in pybind11 + # https://github.com/pybind/pybind11/blob/v2.12.0/tools/pybind11Common.cmake#L397-L403 + target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto) + endif() endif() endif() diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index c12cedca3b2..e05b4c3afc4 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -209,7 +209,7 @@ endmacro() # Enables interprocedural optimization for a list of targets # -function(enable_IPO all_targets_list) +function(warpx_enable_IPO all_targets_list) include(CheckIPOSupported) check_ipo_supported(RESULT is_IPO_available) if(is_IPO_available) From f160cfcaf594ea0c0180bf6781dbfc5b425c50da Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Fri, 19 Jul 2024 17:51:17 +0200 Subject: [PATCH 164/190] Change `if (bool_var == 1)` into `if (bool_var)` for readability (#5060) --- Source/Diagnostics/SliceDiagnostic.cpp | 2 +- Source/Evolve/WarpXEvolve.cpp | 4 ++-- Source/Initialization/WarpXInitData.cpp | 12 ++++++------ Source/Parallelization/GuardCellManager.cpp | 6 +++--- Source/Particles/PhysicalParticleContainer.cpp | 4 ++-- Source/Utils/WarpXTagging.cpp | 2 +- Source/WarpX.cpp | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 14a2d633a0c..97af967f2be 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -153,7 +153,7 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, ablastr::utils::communication::ParallelCopy(*smf, mf, 0, 0, ncomp, nghost_vect, nghost_vect, WarpX::do_single_precision_comms); // interpolate if required on refined slice // - if (interpolate == 1 ) { + if (interpolate) { InterpolateSliceValues( *smf, interp_lo, slice_cc_nd_box, dom_geom, ncomp, nghost, slice_lo, slice_hi, SliceType, real_box); } diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 8b2cc2c33f1..7bdba64dc4e 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -142,7 +142,7 @@ WarpX::Evolve (int numsteps) OneStep_multiJ(cur_time); } // Electromagnetic case: no subcycling or no mesh refinement - else if (do_subcycling == 0 || finest_level == 0) + else if ( !do_subcycling || (finest_level == 0)) { OneStep_nosub(cur_time); // E: guard cells are up-to-date @@ -150,7 +150,7 @@ WarpX::Evolve (int numsteps) // F: guard cells are NOT up-to-date } // Electromagnetic case: subcycling with one level of mesh refinement - else if (do_subcycling == 1 && finest_level == 1) + else if (do_subcycling && (finest_level == 1)) { OneStep_sub1(cur_time); } diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 9a4c46b1c10..8af0ce23ce8 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -294,19 +294,19 @@ WarpX::PrintMainPICparameters () " | - v_comoving = (" << m_v_comoving[0] << "," << m_v_comoving[1] << "," << m_v_comoving[2] << ")\n"; } - if (WarpX::update_with_rho==1) { + if (WarpX::update_with_rho) { amrex::Print() << " | - update with rho is ON \n"; } - if (current_correction==1) { + if (current_correction) { amrex::Print() << " | - current correction is ON \n"; } - if (WarpX::do_dive_cleaning==1) { + if (WarpX::do_dive_cleaning) { amrex::Print() << " | - div(E) cleaning is ON \n"; } - if (WarpX::do_divb_cleaning==1) { + if (WarpX::do_divb_cleaning) { amrex::Print() << " | - div(B) cleaning is ON \n"; } - if (do_multi_J == 1){ + if (do_multi_J){ amrex::Print() << " | - multi-J deposition is ON \n"; amrex::Print() << " | - do_multi_J_n_depositions = " << WarpX::do_multi_J_n_depositions << "\n"; @@ -323,7 +323,7 @@ WarpX::PrintMainPICparameters () amrex::Print() << " | - rho_in_time = constant \n"; } } - if (fft_do_time_averaging == 1){ + if (fft_do_time_averaging){ amrex::Print()<<" | - time-averaged is ON \n"; } #endif // WARPX_USE_FFT diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 321be15df7e..bf010736853 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -59,9 +59,9 @@ guardCellManager::Init ( // When using subcycling, the particles on the fine level perform two pushes // before being redistributed ; therefore, we need one extra guard cell // (the particles may move by 2*c*dt) - int ngx_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; - int ngy_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; - int ngz_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; + int ngx_tmp = (max_level > 0 && do_subcycling) ? nox+1 : nox; + int ngy_tmp = (max_level > 0 && do_subcycling) ? nox+1 : nox; + int ngz_tmp = (max_level > 0 && do_subcycling) ? nox+1 : nox; const bool galilean = (v_galilean[0] != 0. || v_galilean[1] != 0. || v_galilean[2] != 0.); const bool comoving = (v_comoving[0] != 0. || v_comoving[1] != 0. || v_comoving[2] != 0.); diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 89ae435882b..3a30467c20d 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1065,7 +1065,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int static_cast(std::round((overlap_realbox.lo(dir)-problo[dir])/dx[dir])); // shifted is exact in non-moving-window direction. That's all we care. } - if (no_overlap == 1) { + if (no_overlap) { continue; // Go to the next tile } @@ -1640,7 +1640,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // shifted is exact in non-moving-window direction. That's all we care. } } - if (no_overlap == 1) { + if (no_overlap) { continue; // Go to the next tile } diff --git a/Source/Utils/WarpXTagging.cpp b/Source/Utils/WarpXTagging.cpp index f88380e2589..b6eb2fda5d2 100644 --- a/Source/Utils/WarpXTagging.cpp +++ b/Source/Utils/WarpXTagging.cpp @@ -61,7 +61,7 @@ WarpX::ErrorEst (int lev, TagBoxArray& tags, Real /*time*/, int /*ngrow*/) } else { tag_val = (pos > ftlo && pos < fthi); } - if ( tag_val == 1) { + if (tag_val) { fab(i,j,k) = TagBox::SET; } }); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 87fc5f2633d..fcd10418b23 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2457,7 +2457,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(phi_fp[lev], amrex::convert(ba, phi_nodal_flag), dm, ncomps, ngPhi, lev, "phi_fp", 0.0_rt); } - if (do_subcycling == 1 && lev == 0) + if (do_subcycling && lev == 0) { AllocInitMultiFab(current_store[lev][0], amrex::convert(ba,jx_nodal_flag),dm,ncomps,ngJ,lev, "current_store[x]"); AllocInitMultiFab(current_store[lev][1], amrex::convert(ba,jy_nodal_flag),dm,ncomps,ngJ,lev, "current_store[y]"); From 8bf0fb0d7a63248b6aa1084a5294996034a31e43 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 20 Jul 2024 00:40:53 +0200 Subject: [PATCH 165/190] Doc: Lassen Stays TOSS3, Fix SciPy (#5063) Sadly, the TOSS4 transition was not completed before the IBM contract ran out and Lassen stays with TOSS3. This simplifies the documentation again. --- Docs/source/install/hpc/lassen.rst | 161 ++++-------------- .../lassen-llnl/install_v100_dependencies.sh | 142 --------------- .../install_v100_dependencies_toss3.sh | 4 +- .../lassen_v100_warpx.profile.example | 56 ------ .../lassen_v100_warpx_toss3.profile.example | 2 +- 5 files changed, 33 insertions(+), 332 deletions(-) delete mode 100755 Tools/machines/lassen-llnl/install_v100_dependencies.sh delete mode 100644 Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index 0d885dff04d..43330dcc111 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -24,27 +24,9 @@ If you are new to this system, **please see the following resources**: Login ----- -.. tab-set:: - - .. tab-item:: TOSS4 (RHEL8) - - Lassen is currently transitioning to RHEL8. - During this transition, first SSH into lassen and then to the updated RHEL8/TOSS4 nodes. - - .. code-block:: bash - - ssh lassen.llnl.gov - ssh eatoss4 - - Approximately October/November 2023, the new software environment on these nodes will be the new default. - - .. tab-item:: TOSS3 (RHEL7) - - .. code-block:: bash - - ssh lassen.llnl.gov +.. code-block:: bash - Approximately October/November 2023, this partition will become TOSS4 (RHEL8) as well. + ssh lassen.llnl.gov .. _building-lassen-preparation: @@ -58,135 +40,54 @@ Use the following commands to download the WarpX source code: git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/lassen/src/warpx -.. tab-set:: - - .. tab-item:: TOSS4 (RHEL8) - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``. - Create it now: - - .. code-block:: bash - - cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example - :language: bash - - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: - - .. code-block:: bash - - export proj="nsldt" - - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). - - .. important:: - - Now, and as the first step on future logins to lassen, activate these environment settings: - - .. code-block:: bash +We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``. +Create it now: - source $HOME/lassen_v100_warpx.profile - - .. tab-item:: TOSS3 (RHEL7) - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``. - Create it now: - - .. code-block:: bash +.. code-block:: bash - cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile + cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example - :language: bash + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example + :language: bash - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: +Edit the 2nd line of this script, which sets the ``export proj=""`` variable. +For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``. +Enter the edit mode by typing ``i`` and edit line 2 to read: - .. code-block:: bash +.. code-block:: bash - export proj="nsldt" + export proj="nsldt" - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). +Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). - .. important:: +.. important:: - Now, and as the first step on future logins to lassen, activate these environment settings: + Now, and as the first step on future logins to lassen, activate these environment settings: - .. code-block:: bash + .. code-block:: bash - source $HOME/lassen_v100_warpx_toss3.profile + source $HOME/lassen_v100_warpx_toss3.profile Finally, since lassen does not yet provide software modules for some of our dependencies, install them once: -.. tab-set:: - - .. tab-item:: TOSS4 (RHEL8) - - .. code-block:: bash - - bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh - source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh - :language: bash - - .. dropdown:: AI/ML Dependencies (Optional) - :animate: fade-in-slide-down - - If you plan to run AI/ML workflows depending on pyTorch, run the next step as well. - This will take a while and should be skipped if not needed. - - .. code-block:: bash - - runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh - :language: bash - - For `optimas dependencies `__ (incl. scikit-learn), plan another hour of build time: - - .. code-block:: bash - - python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt - - .. tab-item:: TOSS3 (RHEL7) +.. code-block:: bash - .. code-block:: bash + bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh + source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate - bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh - source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh + :language: bash - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh - :language: bash .. _building-lassen-compilation: diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh deleted file mode 100755 index 48668efcc2a..00000000000 --- a/Tools/machines/lassen-llnl/install_v100_dependencies.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was lassen_v100_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lassen_v100_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SRC_DIR="/usr/workspace/${USER}/lassen/src" -SW_DIR="/usr/workspace/${USER}/lassen/gpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} -mkdir -p ${SRC_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# tmpfs build directory: avoids issues often seen with $HOME and is faster -build_dir=$(mktemp -d) - -# c-blosc (I/O compression) -if [ -d ${SRC_DIR}/c-blosc ] -then - cd ${SRC_DIR}/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc -fi -cmake -S ${SRC_DIR}/c-blosc -B ${build_dir}/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build ${build_dir}/c-blosc-lassen-build --target install --parallel 10 - -# HDF5 -if [ -d ${SRC_DIR}/hdf5 ] -then - cd ${SRC_DIR}/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git ${SRC_DIR}/hdf5 -fi -cmake -S ${SRC_DIR}/hdf5 -B ${build_dir}/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build ${build_dir}/hdf5-lassen-build --target install --parallel 10 - -# ADIOS2 -if [ -d ${SRC_DIR}/adios2 ] -then - cd ${SRC_DIR}/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 -fi -cmake -S ${SRC_DIR}/adios2 -B ${build_dir}/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build ${build_dir}/adios2-lassen-build --target install -j 10 - -# BLAS++ (for PSATD+RZ) -if [ -d ${SRC_DIR}/blaspp ] -then - cd ${SRC_DIR}/blaspp - git fetch --prune - git checkout v2024.05.31 - cd - -else - git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp -fi -cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 -cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 - -# LAPACK++ (for PSATD+RZ) -if [ -d ${SRC_DIR}/lapackpp ] -then - cd ${SRC_DIR}/lapackpp - git fetch --prune - git checkout v2024.05.31 - cd - -else - git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp -fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so -cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 - - -# Python ###################################################################### -# -# sometimes, the Lassen PIP Index is down -export PIP_EXTRA_INDEX_URL="https://pypi.org/simple" - -python3 -m pip install --upgrade --user virtualenv -rm -rf ${SW_DIR}/venvs/warpx-lassen -python3 -m venv ${SW_DIR}/venvs/warpx-lassen -source ${SW_DIR}/venvs/warpx-lassen/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip cache purge -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -# Older version for h5py -# https://github.com/h5py/h5py/issues/2268 -python3 -m pip install --upgrade "cython<3" -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade -Ccompile-args="-j10" scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -CC=mpicc H5PY_SETUP_REQUIRES=0 HDF5_DIR=${SW_DIR}/hdf5-1.14.1.2 HDF5_MPI=ON python3 -m pip install --upgrade h5py --no-cache-dir --no-build-isolation --no-binary h5py -MPLLOCALFREETYPE=1 python3 -m pip install --upgrade matplotlib==3.2.2 # does not try to build freetype itself -echo "matplotlib==3.2.2" > ${build_dir}/constraints.txt -python3 -m pip install --upgrade -c ${build_dir}/constraints.txt yt - -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt - -# for ML dependencies, see install_v100_ml.sh - - -# remove build temporary directory -rm -rf ${build_dir} diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh index a6d5aad32fb..1b14159cd22 100644 --- a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh @@ -119,9 +119,7 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools -# Older version for h5py -# https://github.com/h5py/h5py/issues/2268 -python3 -m pip install --upgrade "cython<3" +python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas CMAKE_PREFIX_PATH=/usr/lib64:${CMAKE_PREFIX_PATH} python3 -m pip install --upgrade -Ccompile-args="-j10" -Csetup-args=-Dblas=BLAS -Csetup-args=-Dlapack=BLAS scipy diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example deleted file mode 100644 index 80755ece308..00000000000 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example +++ /dev/null @@ -1,56 +0,0 @@ -# please set your project account -#export proj="" # edit this and comment in - -# required dependencies -module load cmake/3.23.1 -module load gcc/11.2.1 -module load cuda/12.0.0 - -# optional: for QED lookup table generation support -module load boost/1.70.0 - -# optional: for openPMD support -SRC_DIR="/usr/workspace/${USER}/lassen/src" -SW_DIR="/usr/workspace/${USER}/lassen/gpu" -export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} -export PATH=${SW_DIR}/adios2-2.8.3/bin:$PATH - -# optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH - -# optional: for Python bindings -module load python/3.8.2 - -if [ -d "${SW_DIR}/venvs/warpx-lassen" ] -then - source ${SW_DIR}/venvs/warpx-lassen/bin/activate -fi - -# optional: an alias to request an interactive node for two hours -alias getNode="bsub -G $proj -W 2:00 -nnodes 1 -Is /bin/bash" -# an alias to run a command on a batch node for up to 30min -# usage: runNode -alias runNode="bsub -q debug -P $proj -W 2:00 -nnodes 1 -I" - -# fix system defaults: do not escape $ with a \ on tab completion -shopt -s direxpand - -# optimize CUDA compilation for V100 -export AMREX_CUDA_ARCH=7.0 -export CUDAARCHS=70 - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example index 6a7067f596e..98e8d6410b3 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example @@ -28,7 +28,7 @@ export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings -module load python/3.8.2 +module load python/3.11.5 if [ -d "${SW_DIR}/venvs/warpx-lassen-toss3" ] then From c6bb95acf52b088341ee95052fa5f50ca0d55f0e Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Fri, 19 Jul 2024 18:18:41 -0700 Subject: [PATCH 166/190] =?UTF-8?q?Avoid=20N=C2=B2=20Operation=20in=20Bina?= =?UTF-8?q?ry-Pairing=20Coulomb=20Collsions=20(#5066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * moved density and temperature calc from each pair to once for each cell. * pulled out n12 calc to cell level. * fixed dV issue for cyl geom. --- .../BinaryCollision/BinaryCollision.H | 172 ++++++++++++++++++ .../Coulomb/ElasticCollisionPerez.H | 69 ++----- .../Coulomb/PairWiseCoulombCollisionFunc.H | 16 +- .../Collision/BinaryCollision/DSMC/DSMCFunc.H | 5 + .../NuclearFusion/NuclearFusionFunc.H | 5 + 5 files changed, 207 insertions(+), 60 deletions(-) diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 84e0fef8c9c..0ba49e9b09f 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -8,6 +8,7 @@ #define WARPX_PARTICLES_COLLISION_BINARYCOLLISION_H_ #include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" +#include "Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H" #include "Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" @@ -360,6 +361,21 @@ public: End of calculations only required when creating product particles */ + // create vectors to store density and temperature on cell level + amrex::Gpu::DeviceVector n1_vec; + amrex::Gpu::DeviceVector n12_vec; + amrex::Gpu::DeviceVector T1_vec; + if (binary_collision_functor.m_computeSpeciesDensities) { + n1_vec.resize(n_cells); + n12_vec.resize(n_cells); + } + if (binary_collision_functor.m_computeSpeciesTemperatures) { + T1_vec.resize(n_cells); + } + amrex::ParticleReal* AMREX_RESTRICT n1_in_each_cell = n1_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT n12_in_each_cell = n12_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT T1_in_each_cell = T1_vec.dataPtr(); + // Loop over cells amrex::ParallelForRNG( n_cells, [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept @@ -368,12 +384,60 @@ public: // given by the `indices_1[cell_start_1:cell_stop_1]` index_type const cell_start_1 = cell_offsets_1[i_cell]; index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; + index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; // Do not collide if there is only one particle in the cell if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } + // compute local density [1/m^3] + if (binary_collision_functor.m_computeSpeciesDensities) { + amrex::ParticleReal wtot1 = 0.0; + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + for (index_type i1=cell_start_1; i1 n1_vec, n2_vec; + amrex::Gpu::DeviceVector n12_vec; + amrex::Gpu::DeviceVector T1_vec, T2_vec; + if (binary_collision_functor.m_computeSpeciesDensities) { + n1_vec.resize(n_cells); + n2_vec.resize(n_cells); + n12_vec.resize(n_cells); + } + if (binary_collision_functor.m_computeSpeciesTemperatures) { + T1_vec.resize(n_cells); + T2_vec.resize(n_cells); + } + amrex::ParticleReal* AMREX_RESTRICT n1_in_each_cell = n1_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT n2_in_each_cell = n2_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT n12_in_each_cell = n12_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT T1_in_each_cell = T1_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT T2_in_each_cell = T2_vec.dataPtr(); // Loop over cells amrex::ParallelForRNG( n_cells, @@ -579,6 +675,43 @@ public: // ux_1[ indices_1[i] ], where i is between // cell_start_1 (inclusive) and cell_start_2 (exclusive) + // compute local densities [1/m^3] + if (binary_collision_functor.m_computeSpeciesDensities) { + amrex::ParticleReal w1tot = 0.0; + amrex::ParticleReal w2tot = 0.0; + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + for (index_type i1=cell_start_1; i1 /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const n1, amrex::ParticleReal const n2, + amrex::ParticleReal const n12, + amrex::ParticleReal const T1, amrex::ParticleReal const T2, amrex::ParticleReal const q1, amrex::ParticleReal const q2, amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const dV, index_type coll_idx, + amrex::Real const dt, amrex::Real const /*dV*/, index_type coll_idx, index_type const /*cell_start_pair*/, index_type* /*p_mask*/, index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, amrex::ParticleReal* /*p_pair_reaction_weight*/, @@ -99,12 +105,14 @@ public: ElasticCollisionPerez( I1s, I1e, I2s, I2e, I1, I2, - soa_1, soa_2, - q1, q2, m1, m2, -1.0_prt, -1.0_prt, - dt, m_CoulombLog, dV, engine, m_isSameSpecies, coll_idx); + soa_1, soa_2, n1, n2, n12, T1, T2, + q1, q2, m1, m2, + dt, m_CoulombLog, engine, coll_idx); } amrex::ParticleReal m_CoulombLog; + bool m_computeSpeciesDensities = true; + bool m_computeSpeciesTemperatures = false; bool m_isSameSpecies; }; diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H index f28f3fb984c..152dca4e1a1 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -96,6 +96,9 @@ public: index_type const* AMREX_RESTRICT I2, const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*n1*/, amrex::ParticleReal const /*n2*/, + amrex::ParticleReal const /*n12*/, + amrex::ParticleReal const /*T1*/, amrex::ParticleReal const /*T2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, amrex::Real const dt, amrex::Real const dV, index_type coll_idx, @@ -189,6 +192,8 @@ public: } int m_process_count; + bool m_computeSpeciesDensities = false; + bool m_computeSpeciesTemperatures = false; ScatteringProcess::Executor* m_scattering_processes_data; }; diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index b6d6fe171de..1f3e5e56582 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -131,6 +131,9 @@ public: index_type const* AMREX_RESTRICT I2, const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*n1*/, amrex::ParticleReal const /*n2*/, + amrex::ParticleReal const /*n12*/, + amrex::ParticleReal const /*T1*/, amrex::ParticleReal const /*T2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, amrex::Real const dt, amrex::Real const dV, index_type coll_idx, @@ -232,6 +235,8 @@ public: amrex::ParticleReal m_probability_threshold; amrex::ParticleReal m_probability_target_value; NuclearFusionType m_fusion_type; + bool m_computeSpeciesDensities = false; + bool m_computeSpeciesTemperatures = false; bool m_isSameSpecies; }; From 7dad8f33390cdeccc804e4a0c20418db7b34e12c Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Mon, 22 Jul 2024 10:23:23 -0700 Subject: [PATCH 167/190] GNU Make: fix make clean (#5067) --- Source/Make.WarpX | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 519eed52820..502cb8a6714 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -326,7 +326,7 @@ $(srcTempDir)/Utils/export.H: @echo "#endif" >> $@ cleanconfig:: - $(SILENT) $(RM) -r $(srcTempDir)/Utils + $(SILENT) $(RM) -r $(srcTempDir)/Utils $(srcTempDir)/ablastr include $(AMREX_HOME)/Tools/GNUMake/Make.rules From f1af6f40636f8ae3085d350de425ce52f06c5479 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 22 Jul 2024 11:38:32 -0700 Subject: [PATCH 168/190] AMReX/pyAMReX/PICSAR: Weekly Update (#5074) * AMReX: Weekly Update * pyAMReX: Weekly Update --- .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index e6ba115925a..c2ec9aaa763 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach dcb9cc0383dcc71e38dee9070574e325a812f8bf && cd - + cd ../amrex && git checkout --detach 0c3273f5e591815909180f8ffaf5b793cabbf9bc && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index b32ca61c709..a2447ac3cf5 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = dcb9cc0383dcc71e38dee9070574e325a812f8bf +branch = 0c3273f5e591815909180f8ffaf5b793cabbf9bc [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 6d7c57f0961..17ecfd64f29 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = dcb9cc0383dcc71e38dee9070574e325a812f8bf +branch = 0c3273f5e591815909180f8ffaf5b793cabbf9bc [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 8acbd974779..025b4d3b6e0 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "dcb9cc0383dcc71e38dee9070574e325a812f8bf" +set(WarpX_amrex_branch "0c3273f5e591815909180f8ffaf5b793cabbf9bc" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index a2fbd0ddee8..a4598953432 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "18f0026b1dd9b2aa4869c96f74e4b77262a067d0" +set(WarpX_pyamrex_branch "ff4643869c63d4ee40a87054b901f61eefcb97a3" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index c01c3fb1176..1692f66263c 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach dcb9cc0383dcc71e38dee9070574e325a812f8bf && cd - +cd amrex && git checkout --detach 0c3273f5e591815909180f8ffaf5b793cabbf9bc && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 396c06106ac7d635c70d46e112fb1b7ff2b720ad Mon Sep 17 00:00:00 2001 From: David Grote Date: Wed, 24 Jul 2024 23:05:44 -0700 Subject: [PATCH 169/190] Fix again the collisionXYZ benchmark (#5083) --- .../Tests/collision/analysis_collision_2d.py | 2 +- .../benchmarks_json/collisionXYZ.json | 28 +++++++++---------- .../Checksum/benchmarks_json/collisionXZ.json | 16 +++++++++++ 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/Examples/Tests/collision/analysis_collision_2d.py b/Examples/Tests/collision/analysis_collision_2d.py index bd013898a8a..29482909b2e 100755 --- a/Examples/Tests/collision/analysis_collision_2d.py +++ b/Examples/Tests/collision/analysis_collision_2d.py @@ -114,4 +114,4 @@ dim, species_name) test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 39df1b9eee1..2aa0e4686eb 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,25 +6,25 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 353604.6247926339, - "T_ion": 347976.6168136309 + "T_electron": 381488.6528070591, + "T_ion": 320091.2785835478 }, "electron": { - "particle_momentum_x": 8.370755929299189e-19, - "particle_momentum_y": 8.228112213603589e-19, - "particle_momentum_z": 8.204295817378347e-19, - "particle_position_x": 21284971.94721422, - "particle_position_y": 21212829.42991966, - "particle_position_z": 21214774.536558084, + "particle_momentum_x": 8.667025573698235e-19, + "particle_momentum_y": 8.457499789250831e-19, + "particle_momentum_z": 8.482438182280524e-19, + "particle_position_x": 21262567.138872623, + "particle_position_y": 21245135.070665065, + "particle_position_z": 21232644.283726066, "particle_weight": 7.168263344048695e+28 }, "ion": { - "particle_momentum_x": 2.0074097598289766e-18, - "particle_momentum_y": 1.8203553942782305e-18, - "particle_momentum_z": 1.823420185235695e-18, - "particle_position_x": 21227192.857240494, - "particle_position_y": 21286501.692027714, - "particle_position_z": 21245587.6706009, + "particle_momentum_x": 1.9300495097720012e-18, + "particle_momentum_y": 1.747257416857836e-18, + "particle_momentum_z": 1.7510296287537058e-18, + "particle_position_x": 21217348.883301035, + "particle_position_y": 21300859.0630925, + "particle_position_z": 21237901.246521123, "particle_weight": 7.168263344048695e+28 } } diff --git a/Regression/Checksum/benchmarks_json/collisionXZ.json b/Regression/Checksum/benchmarks_json/collisionXZ.json index 927848745a8..49138f0a1df 100644 --- a/Regression/Checksum/benchmarks_json/collisionXZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXZ.json @@ -6,5 +6,21 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0 + }, + "ion": { + "particle_momentum_x": 2.458306853810186e-19, + "particle_momentum_y": 2.272685285153902e-19, + "particle_momentum_z": 2.281205462681013e-19, + "particle_position_x": 2645436.647039526, + "particle_position_y": 2672571.48688055, + "particle_weight": 1.7256099431746894e+26 + }, + "electron": { + "particle_momentum_x": 1.0454942263455085e-19, + "particle_momentum_y": 1.0323735347957779e-19, + "particle_momentum_z": 1.0199134968670343e-19, + "particle_position_x": 2681776.3648108337, + "particle_position_y": 2663907.8843079703, + "particle_weight": 1.7256099431746894e+26 } } From 49df0ca46930229678c3bebe063f949e24b63013 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Thu, 25 Jul 2024 19:15:13 +0200 Subject: [PATCH 170/190] [WIP] Recompute the macroscopic properties everytime the moving window moves. (#5082) * Recompute the macroscopic properties everytime the moving window moves. * Minor cleanup * Separate allocation and initialization --- .../MacroscopicProperties.H | 14 +++++++++---- .../MacroscopicProperties.cpp | 20 ++++++++++--------- Source/Initialization/WarpXInitData.cpp | 5 +---- Source/Utils/WarpXMovingWindow.cpp | 12 +++++++++++ Source/WarpX.cpp | 7 +++++++ 5 files changed, 41 insertions(+), 17 deletions(-) diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H index 129b2e2f782..ca9c77334ea 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H @@ -38,20 +38,26 @@ public: void ReadParameters (); /** - * \brief Initialize multifabs storing macroscopic multifabs + * \brief Allocate multifabs storing macroscopic multifabs * * @param[in] ba the box array associated to the multifabs E and B * @param[in] dmap the distribution mapping * @param[in] ng_EB_alloc guard cells allocated for multifabs E and B + */ + void AllocateLevelMFs ( + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm, + const amrex::IntVect& ng_EB_alloc ); + + /** + * \brief Initialize multifabs storing macroscopic multifabs + * * @param[in] geom the geometry * @param[in] Ex_stag staggering of the Ex field * @param[in] Ey_stag staggering of the Ey field * @param[in] Ez_stag staggering of the Ez field */ void InitData ( - const amrex::BoxArray& ba, - const amrex::DistributionMapping& dmap, - const amrex::IntVect& ng_EB_alloc, const amrex::Geometry& geom, const amrex::IntVect& Ex_stag, const amrex::IntVect& Ey_stag, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index 1467c7a8c0c..a6a389fe056 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -119,24 +119,26 @@ MacroscopicProperties::ReadParameters () } void -MacroscopicProperties::InitData ( +MacroscopicProperties::AllocateLevelMFs ( const amrex::BoxArray& ba, const amrex::DistributionMapping& dmap, - const amrex::IntVect& ng_EB_alloc, - const amrex::Geometry& geom, - const amrex::IntVect& Ex_stag, - const amrex::IntVect& Ey_stag, - const amrex::IntVect& Ez_stag) + const amrex::IntVect& ng_EB_alloc ) { - amrex::Print() << Utils::TextMsg::Info("we are in init data of macro"); - - // Define material property multifabs using ba and dmap from WarpX instance // sigma is cell-centered MultiFab m_sigma_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); // epsilon is cell-centered MultiFab m_eps_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); // mu is cell-centered MultiFab m_mu_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); +} + +void +MacroscopicProperties::InitData ( + const amrex::Geometry& geom, + const amrex::IntVect& Ex_stag, + const amrex::IntVect& Ey_stag, + const amrex::IntVect& Ez_stag) +{ // Initialize sigma if (m_sigma_s == "constant") { diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 8af0ce23ce8..e9f63e34bb0 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -474,12 +474,9 @@ WarpX::InitData () BuildBufferMasks(); - if (WarpX::em_solver_medium==1) { + if (WarpX::em_solver_medium == MediumForEM::Macroscopic) { const int lev_zero = 0; m_macroscopic_properties->InitData( - boxArray(lev_zero), - DistributionMap(lev_zero), - getngEB(), Geom(lev_zero), getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index 5fcaccf761a..d64394dafc9 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -19,6 +19,7 @@ #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include @@ -452,6 +453,17 @@ WarpX::MoveWindow (const int step, bool move_j) } } + // Recompute macroscopic properties of the medium + if (WarpX::em_solver_medium == MediumForEM::Macroscopic) { + const int lev_zero = 0; + m_macroscopic_properties->InitData( + Geom(lev_zero), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + ); + } + return num_shift_base; } diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index fcd10418b23..b725431c2f1 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -2373,6 +2373,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm myfl->InitData(lev, geom[lev].Domain(),cur_time); } + // Allocate extra multifabs for macroscopic properties of the medium + if (em_solver_medium == MediumForEM::Macroscopic) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( lev==0, + "Macroscopic properties are not supported with mesh refinement."); + m_macroscopic_properties->AllocateLevelMFs(ba, dm, ngEB); + } + if (fft_do_time_averaging) { AllocInitMultiFab(Bfield_avg_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[x]", 0.0_rt); From eda6af5525d89eb251305f340f2f9b6a955b1b48 Mon Sep 17 00:00:00 2001 From: Luca Fedeli Date: Thu, 25 Jul 2024 19:26:15 +0200 Subject: [PATCH 171/190] Update script to compile WarpX on Frontier (OLCF) (#5076) * update profile script for Frontier * satisfy dependency for module load * change module versions and manually install adios2 * fix bug * fix bug * update instructions * update scripts * remove line for LibEnsemble --- .../frontier-olcf/frontier_warpx.profile.example | 14 +++++++------- .../machines/frontier-olcf/install_dependencies.sh | 3 --- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/Tools/machines/frontier-olcf/frontier_warpx.profile.example b/Tools/machines/frontier-olcf/frontier_warpx.profile.example index 50e12e99a38..f59f2d3d058 100644 --- a/Tools/machines/frontier-olcf/frontier_warpx.profile.example +++ b/Tools/machines/frontier-olcf/frontier_warpx.profile.example @@ -8,9 +8,9 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo # required dependencies module load cmake/3.23.2 module load craype-accel-amd-gfx90a -module load rocm/5.2.0 # waiting for 5.6 for next bump -module load cray-mpich -module load cce/15.0.0 # must be loaded after rocm +module load rocm/5.7.1 +module load cray-mpich/8.1.28 +module load cce/17.0.0 # must be loaded after rocm # optional: faster builds module load ccache @@ -26,14 +26,14 @@ export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/blaspp-2024.05.31/lib64:$LD_LIBRA export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support -module load boost/1.79.0-cxx17 +module load boost/1.79.0 # optional: for openPMD support -module load adios2/2.8.3 -module load hdf5/1.14.0 +module load adios2/2.8.3-mpi +module load hdf5/1.12.1-mpi # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.11.5 if [ -d "${HOME}/sw/frontier/gpu/venvs/warpx-frontier" ] then diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 503ab4525e9..fd1d28e76b5 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -74,7 +74,6 @@ CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B cmake --build $HOME/src/lapackpp-frontier-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-frontier-gpu-build - # Python ###################################################################### # python3 -m pip install --upgrade pip @@ -109,8 +108,6 @@ CUPY_INSTALL_USE_HIP=1 \ ROCM_HOME=${ROCM_PATH} \ HCC_AMDGPU_TARGET=${AMREX_AMD_ARCH} \ python3 -m pip install -v cupy -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2 #python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt From 675369d9797aa2bcfda8b64c8e500fff6a2b9f04 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 25 Jul 2024 10:29:56 -0700 Subject: [PATCH 172/190] PICMI (Bucket): NumPy 2.0 Compatibility (#5075) * PICMI (Bucket): NumPy 2.0 Compatibility `repr` returns `'np.float64(value)'` instead of `'value'` in NumPy 2.0 * Use fstrings * Remove outdated comment * `requirements.txt`: Relax `numpy` version Version 1 and 2 support. * Remove outdated comment --------- Co-authored-by: David Grote --- Python/pywarpx/Bucket.py | 6 ++---- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/Python/pywarpx/Bucket.py b/Python/pywarpx/Bucket.py index 91a34b9d3d6..9dbe4def88e 100644 --- a/Python/pywarpx/Bucket.py +++ b/Python/pywarpx/Bucket.py @@ -59,8 +59,6 @@ def attrlist(self): for attr, value in self.argvattrs.items(): if value is None: continue - # --- repr is applied to value so that for floats, all of the digits are included. - # --- The strip of "'" is then needed when value is a string. if isinstance(value, str): if value.find('=') > -1: # --- Expressions with temporary variables need to be inside quotes @@ -73,11 +71,11 @@ def attrlist(self): continue # --- For lists, tuples, and arrays make a space delimited string of the values. # --- The lambda is needed in case this is a list of strings. - rhs = ' '.join(map(lambda s : repr(s).strip("'"), value)) + rhs = ' '.join(map(lambda s : f'{s}', value)) elif isinstance(value, bool): rhs = 1 if value else 0 else: rhs = value - attrstring = '{0}.{1} = {2}'.format(self.instancename, attr, repr(rhs).strip("'")) + attrstring = f'{self.instancename}.{attr} = {rhs}' result += [attrstring] return result diff --git a/requirements.txt b/requirements.txt index bb0d8ebb704..8e664ae3096 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # basic dependencies -numpy~=1.15 +numpy>=1.15 periodictable~=1.5 # PICMI From 5343aa87aa1d98fb68aafd6546db09b0e69f8d8b Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Mon, 29 Jul 2024 11:24:32 +0200 Subject: [PATCH 173/190] Add PRL citation (#5093) --- Docs/source/highlights.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index 7baec74d606..108f685a551 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -14,6 +14,11 @@ Plasma-Based Acceleration Scientific works in laser-plasma and beam-plasma acceleration. +#. Shrock JE, Rockafellow E, Miao B, Le M, Hollinger RC, Wang S, Gonsalves AJ, Picksley A, Rocca JJ, and Milchberg HM + **Guided Mode Evolution and Ionization Injection in Meter-Scale Multi-GeV Laser Wakefield Accelerators**. + Phys. Rev. Lett. **133**, 045002, 2024 + `DOI:10.1103/PhysRevLett.133.045002 `__ + #. Ross AJ, Chappell J, van de Wetering JJ, Cowley J, Archer E, Bourgeois N, Corner L, Emerson DR, Feder L, Gu XJ, Jakobsson O, Jones H, Picksley A, Reid L, Wang W, Walczak R, Hooker SM. **Resonant excitation of plasma waves in a plasma channel**. Phys. Rev. Research **6**, L022001, 2024 From b1e7932cc2df0f07ede608f2a2eaf9cbde99d600 Mon Sep 17 00:00:00 2001 From: Justin Ray Angus Date: Mon, 29 Jul 2024 12:05:13 -0700 Subject: [PATCH 174/190] Improving Coulomb collision method for weighted-particles (#5091) * using corrected weighted-particle Coulomb collision method. * adding CI test for weighted Coulomb collisions. * using n12 in UpdateMomentumPerezElastic * updating Checksum * fixing type issue. * updating checksums * checksum --- .../Tests/collision/analysis_collision_1d.py | 126 ++++++++++++++++++ Examples/Tests/collision/inputs_1d | 90 +++++++++++++ .../benchmarks_json/collisionISO.json | 12 +- .../benchmarks_json/collisionXYZ.json | 28 ++-- .../Checksum/benchmarks_json/collisionXZ.json | 20 +-- .../Checksum/benchmarks_json/collisionZ.json | 20 +++ Regression/WarpX-tests.ini | 14 ++ .../BinaryCollision/BinaryCollision.H | 61 +-------- .../Coulomb/ElasticCollisionPerez.H | 31 +++-- .../Coulomb/PairWiseCoulombCollisionFunc.H | 7 +- .../Coulomb/UpdateMomentumPerezElastic.H | 11 +- .../Collision/BinaryCollision/DSMC/DSMCFunc.H | 1 - .../NuclearFusion/NuclearFusionFunc.H | 1 - 13 files changed, 313 insertions(+), 109 deletions(-) create mode 100755 Examples/Tests/collision/analysis_collision_1d.py create mode 100644 Examples/Tests/collision/inputs_1d create mode 100644 Regression/Checksum/benchmarks_json/collisionZ.json diff --git a/Examples/Tests/collision/analysis_collision_1d.py b/Examples/Tests/collision/analysis_collision_1d.py new file mode 100755 index 00000000000..7775a476dae --- /dev/null +++ b/Examples/Tests/collision/analysis_collision_1d.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Justin Angus +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from the script `inputs_1d`. +# run locally: python analysis_vandb_1d.py diags/diag1000600/ +# +# This is a 1D intra-species Coulomb scattering relaxation test consisting +# of a low-density population streaming into a higher density population at rest. +# Both populations belong to the same carbon12 ion species. +# See test T1b from JCP 413 (2020) by D. Higginson, et al. +# +import os +import sys + +import numpy as np +import yt +from scipy.constants import e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] +ds = yt.load(fn) +data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) + +# carbon 12 ion (mass = 12*amu - 6*me) +mass = 1.992100316897910e-26 + +# Separate macroparticles from group A (low weight) and group B (high weight) +# by sorting based on weight +sorted_indices = data['ions','particle_weight'].argsort() +sorted_wp = data['ions', 'particle_weight'][sorted_indices].value +sorted_px = data['ions', 'particle_momentum_x'][sorted_indices].value +sorted_py = data['ions', 'particle_momentum_y'][sorted_indices].value +sorted_pz = data['ions', 'particle_momentum_z'][sorted_indices].value + +# Find the index 'Npmin' that separates macroparticles from group A and group B +Np = len(sorted_wp) +wpmin = sorted_wp.min(); +wpmax = sorted_wp.max(); +for i in range(len(sorted_wp)): + if sorted_wp[i] > wpmin: + Npmin = i + break + +NpA = Npmin +wpA = wpmin; +NpB = Np - Npmin +wpB = wpmax; +NpAs = 0 +NpAe = Npmin +NpBs = Npmin +NpBe = Np + +############# + +sorted_px_sum = np.abs(sorted_px).sum(); +sorted_py_sum = np.abs(sorted_py).sum(); +sorted_pz_sum = np.abs(sorted_pz).sum(); +sorted_wp_sum = np.abs(sorted_wp).sum(); + +# compute mean velocities +wAtot = wpA*NpA +wBtot = wpB*NpB + +uBx = uBy = uBz = 0. +for i in range(NpBs,NpBe): + uBx += wpB*sorted_px[i] + uBy += wpB*sorted_py[i] + uBz += wpB*sorted_pz[i] +uBx /= (mass*wBtot) # [m/s] +uBy /= (mass*wBtot) # [m/s] +uBz /= (mass*wBtot) # [m/s] + +uAx = uAy = uAz = 0. +for i in range(NpAs,NpAe): + uAx += wpA*sorted_px[i] + uAy += wpA*sorted_py[i] + uAz += wpA*sorted_pz[i] +uAx /= (mass*wAtot) # [m/s] +uAy /= (mass*wAtot) # [m/s] +uAz /= (mass*wAtot) # [m/s] + +# compute temperatures +TBx = TBy = TBz = 0. +for i in range(NpBs,NpBe): + TBx += wpB*(sorted_px[i]/mass - uBx)**2 + TBy += wpB*(sorted_py[i]/mass - uBy)**2 + TBz += wpB*(sorted_pz[i]/mass - uBz)**2 +TBx *= mass/(e*wBtot) +TBy *= mass/(e*wBtot) +TBz *= mass/(e*wBtot) + +TAx = TAy = TAz = 0. +for i in range(NpAs,NpAe): + TAx += wpA*(sorted_px[i]/mass - uAx)**2 + TAy += wpA*(sorted_py[i]/mass - uAy)**2 + TAz += wpA*(sorted_pz[i]/mass - uAz)**2 +TAx *= mass/(e*wAtot) +TAy *= mass/(e*wAtot) +TAz *= mass/(e*wAtot) + +TApar = TAz +TAperp = (TAx + TAy)/2.0 +TA = (TAx + TAy + TAz)/3.0 + +TBpar = TBz +TBperp = (TBx + TBy)/2.0 +TB = (TBx + TBy + TBz)/3.0 + +TApar_30ps_soln = 6.15e3 # TA parallel solution at t = 30 ps +error = np.abs(TApar-TApar_30ps_soln)/TApar_30ps_soln +tolerance = 0.02 +print('TApar at 30ps error = ', error); +print('tolerance = ', tolerance); +assert error < tolerance + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/collision/inputs_1d b/Examples/Tests/collision/inputs_1d new file mode 100644 index 00000000000..b2de17192ae --- /dev/null +++ b/Examples/Tests/collision/inputs_1d @@ -0,0 +1,90 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.nA = 1.e25 # m^-3 +my_constants.NpA = 400 # m^-3 +my_constants.UA = 6.55e5 # m/s +my_constants.TA = 500. # eV +# +my_constants.nB = 1.e26 # m^-3 +my_constants.NpB = 400 # m^-3 +my_constants.UB = 0. # m/s +my_constants.TB = 500. # eV +# +my_constants.q_c12 = 6.*q_e +my_constants.m_c12 = 12.*m_u - 6.*m_e + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 600 +amr.n_cell = 180 +amr.max_level = 0 +amr.blocking_factor = 4 +geometry.dims = 1 +geometry.prob_lo = 0. +geometry.prob_hi = 0.01 + +################################# +###### Boundary Condition ####### +################################# +boundary.field_lo = periodic +boundary.field_hi = periodic + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = 0.05e-12 +warpx.use_filter = 0 + +# Do not evolve the E and B fields +algo.maxwell_solver = none + +# Order of particle shape factors +algo.particle_shape = 1 + +################################# +############ PLASMA ############# +################################# +particles.species_names = ions + +ions.charge = q_c12 +ions.mass = m_c12 +ions.do_not_deposit = 1 + +ions.injection_sources = groupA groupB + +ions.groupA.injection_style = "NUniformPerCell" +ions.groupA.num_particles_per_cell_each_dim = NpA +ions.groupA.profile = constant +ions.groupA.density = nA # number per m^3 +ions.groupA.momentum_distribution_type = "gaussian" +ions.groupA.uz_m = UA/clight +ions.groupA.ux_th = sqrt(TA*q_e/m_c12)/clight +ions.groupA.uy_th = sqrt(TA*q_e/m_c12)/clight +ions.groupA.uz_th = sqrt(TA*q_e/m_c12)/clight + +ions.groupB.injection_style = "NUniformPerCell" +ions.groupB.num_particles_per_cell_each_dim = NpB +ions.groupB.profile = constant +ions.groupB.density = nB # number per m^3 +ions.groupB.momentum_distribution_type = "gaussian" +ions.groupB.uz_m = UB/clight +ions.groupB.ux_th = sqrt(TB*q_e/m_c12)/clight +ions.groupB.uy_th = sqrt(TB*q_e/m_c12)/clight +ions.groupB.uz_th = sqrt(TB*q_e/m_c12)/clight + +################################# +############ COLLISION ########## +################################# +collisions.collision_names = collision1 +collision1.species = ions ions +collision1.CoulombLog = 10.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 600 +diag1.diag_type = Full diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json index a2fd6116cb8..350848d4aee 100644 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ b/Regression/Checksum/benchmarks_json/collisionISO.json @@ -11,12 +11,12 @@ "jz": 0.0 }, "electron": { - "particle_momentum_x": 3.579989064013309e-19, - "particle_momentum_y": 3.5822945977746767e-19, - "particle_momentum_z": 3.579753452653627e-19, - "particle_position_x": 1.0241322532163375, - "particle_position_y": 1.0238995904625479, - "particle_position_z": 1.02402135051502, + "particle_momentum_x": 3.5790777034053853e-19, + "particle_momentum_y": 3.5815348106229496e-19, + "particle_momentum_z": 3.577963316718249e-19, + "particle_position_x": 1.024180253191667, + "particle_position_y": 1.023919590453571, + "particle_position_z": 1.0240653505082926, "particle_weight": 714240000000.0 } } diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 2aa0e4686eb..c13f404857a 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,25 +6,25 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 381488.6528070591, - "T_ion": 320091.2785835478 + "T_electron": 381957.7394223898, + "T_ion": 319565.6269784763 }, "electron": { - "particle_momentum_x": 8.667025573698235e-19, - "particle_momentum_y": 8.457499789250831e-19, - "particle_momentum_z": 8.482438182280524e-19, - "particle_position_x": 21262567.138872623, - "particle_position_y": 21245135.070665065, - "particle_position_z": 21232644.283726066, + "particle_momentum_x": 8.631833485301232e-19, + "particle_momentum_y": 8.476316745254719e-19, + "particle_momentum_z": 8.514139891331418e-19, + "particle_position_x": 21253105.73686561, + "particle_position_y": 21282643.519070115, + "particle_position_z": 21239057.457948968, "particle_weight": 7.168263344048695e+28 }, "ion": { - "particle_momentum_x": 1.9300495097720012e-18, - "particle_momentum_y": 1.747257416857836e-18, - "particle_momentum_z": 1.7510296287537058e-18, - "particle_position_x": 21217348.883301035, - "particle_position_y": 21300859.0630925, - "particle_position_z": 21237901.246521123, + "particle_momentum_x": 1.9215585867122464e-18, + "particle_momentum_y": 1.7471481568315848e-18, + "particle_momentum_z": 1.7510887207292533e-18, + "particle_position_x": 21228900.948879313, + "particle_position_y": 21304564.011731848, + "particle_position_z": 21250585.221808463, "particle_weight": 7.168263344048695e+28 } } diff --git a/Regression/Checksum/benchmarks_json/collisionXZ.json b/Regression/Checksum/benchmarks_json/collisionXZ.json index 49138f0a1df..f90c34bc86d 100644 --- a/Regression/Checksum/benchmarks_json/collisionXZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXZ.json @@ -8,19 +8,19 @@ "Ez": 0.0 }, "ion": { - "particle_momentum_x": 2.458306853810186e-19, - "particle_momentum_y": 2.272685285153902e-19, - "particle_momentum_z": 2.281205462681013e-19, - "particle_position_x": 2645436.647039526, - "particle_position_y": 2672571.48688055, + "particle_momentum_x": 2.4932317055825563e-19, + "particle_momentum_y": 2.274916403334278e-19, + "particle_momentum_z": 2.2528161767665816e-19, + "particle_position_x": 2648815.601036139, + "particle_position_y": 2662836.7581390506, "particle_weight": 1.7256099431746894e+26 }, "electron": { - "particle_momentum_x": 1.0454942263455085e-19, - "particle_momentum_y": 1.0323735347957779e-19, - "particle_momentum_z": 1.0199134968670343e-19, - "particle_position_x": 2681776.3648108337, - "particle_position_y": 2663907.8843079703, + "particle_momentum_x": 1.0489203687862582e-19, + "particle_momentum_y": 1.0209657029567292e-19, + "particle_momentum_z": 1.0248962872393911e-19, + "particle_position_x": 2657004.8285825616, + "particle_position_y": 2670174.272797987, "particle_weight": 1.7256099431746894e+26 } } diff --git a/Regression/Checksum/benchmarks_json/collisionZ.json b/Regression/Checksum/benchmarks_json/collisionZ.json new file mode 100644 index 00000000000..3be8d5ae893 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/collisionZ.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "ions": { + "particle_momentum_x": 3.425400072687143e-16, + "particle_momentum_y": 3.421937133999805e-16, + "particle_momentum_z": 5.522701882677923e-16, + "particle_position_x": 720.0011611411148, + "particle_weight": 1.0999999999999999e+24 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 17ecfd64f29..c1a6316e7e2 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -195,6 +195,20 @@ useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py +[collisionZ] +buildDir = . +inputFile = Examples/Tests/collision/inputs_1d +runtime_params = +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +analysisRoutine = Examples/Tests/collision/analysis_collision_1d.py + [collisionISO] buildDir = . inputFile = Examples/Tests/collision/inputs_3d_isotropization diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index 0ba49e9b09f..6f8f61e66b1 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -363,17 +363,14 @@ public: // create vectors to store density and temperature on cell level amrex::Gpu::DeviceVector n1_vec; - amrex::Gpu::DeviceVector n12_vec; amrex::Gpu::DeviceVector T1_vec; if (binary_collision_functor.m_computeSpeciesDensities) { n1_vec.resize(n_cells); - n12_vec.resize(n_cells); } if (binary_collision_functor.m_computeSpeciesTemperatures) { T1_vec.resize(n_cells); } amrex::ParticleReal* AMREX_RESTRICT n1_in_each_cell = n1_vec.dataPtr(); - amrex::ParticleReal* AMREX_RESTRICT n12_in_each_cell = n12_vec.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT T1_in_each_cell = T1_vec.dataPtr(); // Loop over cells @@ -384,7 +381,6 @@ public: // given by the `indices_1[cell_start_1:cell_stop_1]` index_type const cell_start_1 = cell_offsets_1[i_cell]; index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; - index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; // Do not collide if there is only one particle in the cell if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } @@ -415,29 +411,6 @@ public: // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); - - // compute n12 for intra-species - if (binary_collision_functor.m_computeSpeciesDensities) { - amrex::ParticleReal n12 = 0.0; - index_type const NI1 = cell_half_1 - cell_start_1; - index_type const NI2 = cell_stop_1 - cell_half_1; - index_type const max_N = amrex::max(NI1,NI2); - index_type i1 = cell_start_1; - index_type i2 = cell_half_1; - amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; - for (index_type k=0; k n1_vec, n2_vec; - amrex::Gpu::DeviceVector n12_vec; amrex::Gpu::DeviceVector T1_vec, T2_vec; if (binary_collision_functor.m_computeSpeciesDensities) { n1_vec.resize(n_cells); n2_vec.resize(n_cells); - n12_vec.resize(n_cells); } if (binary_collision_functor.m_computeSpeciesTemperatures) { T1_vec.resize(n_cells); @@ -655,7 +624,6 @@ public: } amrex::ParticleReal* AMREX_RESTRICT n1_in_each_cell = n1_vec.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT n2_in_each_cell = n2_vec.dataPtr(); - amrex::ParticleReal* AMREX_RESTRICT n12_in_each_cell = n12_vec.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT T1_in_each_cell = T1_vec.dataPtr(); amrex::ParticleReal* AMREX_RESTRICT T2_in_each_cell = T2_vec.dataPtr(); @@ -719,29 +687,6 @@ public: // shuffle ShuffleFisherYates(indices_1, cell_start_1, cell_stop_1, engine); ShuffleFisherYates(indices_2, cell_start_2, cell_stop_2, engine); - - // compute n12 for inter-species - if (binary_collision_functor.m_computeSpeciesDensities) { - amrex::ParticleReal n12 = 0.0; - index_type const NI1 = cell_stop_1 - cell_start_1; - index_type const NI2 = cell_stop_2 - cell_start_2; - index_type const max_N = amrex::max(NI1,NI2); - index_type i1 = cell_start_1; - index_type i2 = cell_start_2; - amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; - amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; - for (index_type k=0; k( 1.0/std::cbrt(4.0*MathConst::pi/3.0*maxn) ); + + // bmax (screening length) cannot be smaller than atomic spacing + const T_PR bmax = amrex::max(lmdD, rmin); #if (defined WARPX_DIM_RZ) T_PR * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; @@ -108,12 +110,23 @@ void ElasticCollisionPerez ( u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); #endif + // Compute the effective density n12 used to compute the normalized + // scattering path s12 in UpdateMomentumPerezElastic(). + // s12 is defined such that the expected value of the change in particle + // velocity is equal to that from the full NxN pairing method, as described + // here https://arxiv.org/submit/5758216/view. This method is a direct extension + // of the original method by Takizuka and Abe JCP 25 (1977) to weighted particles. + T_PR n12; + const T_PR wpmax = amrex::max(w1[ I1[i1] ],w2[ I2[i2] ]); + if (isSameSpecies) { n12 = wpmax*static_cast(min_N+max_N-1)/dV; } + else { n12 = wpmax*static_cast(min_N)/dV; } + UpdateMomentumPerezElastic( u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], n1, n2, n12, q1, m1, w1[ I1[i1] ], q2, m2, w2[ I2[i2] ], - dt, L, lmdD, + dt, L, bmax, engine); #if (defined WARPX_DIM_RZ) diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index cd3cd5a5c83..3322c19ed2d 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -91,11 +91,10 @@ public: const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const n1, amrex::ParticleReal const n2, - amrex::ParticleReal const n12, amrex::ParticleReal const T1, amrex::ParticleReal const T2, amrex::ParticleReal const q1, amrex::ParticleReal const q2, amrex::ParticleReal const m1, amrex::ParticleReal const m2, - amrex::Real const dt, amrex::Real const /*dV*/, index_type coll_idx, + amrex::Real const dt, amrex::Real const dV, index_type coll_idx, index_type const /*cell_start_pair*/, index_type* /*p_mask*/, index_type* /*p_pair_indices_1*/, index_type* /*p_pair_indices_2*/, amrex::ParticleReal* /*p_pair_reaction_weight*/, @@ -105,9 +104,9 @@ public: ElasticCollisionPerez( I1s, I1e, I2s, I2e, I1, I2, - soa_1, soa_2, n1, n2, n12, T1, T2, + soa_1, soa_2, n1, n2, T1, T2, q1, q2, m1, m2, - dt, m_CoulombLog, engine, coll_idx); + dt, m_CoulombLog, dV, engine, m_isSameSpecies, coll_idx); } amrex::ParticleReal m_CoulombLog; diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H b/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H index 3101047e211..93acc3d8aef 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H @@ -18,9 +18,10 @@ /* \brief Update particle velocities according to * F. Perez et al., Phys.Plasmas.19.083104 (2012), * which is based on Nanbu's method, PhysRevE.55.4642 (1997). - * @param[in] LmdD is max(Debye length, minimal interparticle distance). + * @param[in] bmax is max(Debye length, minimal interparticle distance). * @param[in] L is the Coulomb log. A fixed L will be used if L > 0, * otherwise L will be calculated based on the algorithm. + * @param[in] n12 = max(w1,w2)*min(N1,N2)/dV is the effective density used for s12 * To see if there are nan or inf updated velocities, * compile with USE_ASSERTION=TRUE. * @@ -36,7 +37,7 @@ void UpdateMomentumPerezElastic ( T_PR const n1, T_PR const n2, T_PR const n12, T_PR const q1, T_PR const m1, T_PR const w1, T_PR const q2, T_PR const m2, T_PR const w2, - T_R const dt, T_PR const L, T_PR const lmdD, + T_R const dt, T_PR const L, T_PR const bmax, amrex::RandomEngine const& engine) { @@ -128,13 +129,13 @@ void UpdateMomentumPerezElastic ( // Compute the Coulomb log lnLmd lnLmd = amrex::max( T_PR(2.0), - T_PR(0.5)*std::log(T_PR(1.0)+lmdD*lmdD/(bmin*bmin)) ); + T_PR(0.5)*std::log(T_PR(1.0) + bmax*bmax/(bmin*bmin)) ); } // Compute s const auto tts = m1*g1s*m2*g2s/(inv_c2*p1sm*p1sm) + T_PR(1.0); const auto tts2 = tts*tts; - s = n1*n2/n12 * dt*lnLmd*q1*q1*q2*q2 / + s = n12 * dt*lnLmd*q1*q1*q2*q2 / ( T_PR(4.0) * MathConst::pi * PhysConst::ep0 * PhysConst::ep0 * m1*g1*m2*g2/(inv_c2*inv_c2) ) * gc*p1sm/mass_g * tts2; @@ -144,7 +145,7 @@ void UpdateMomentumPerezElastic ( const auto coeff = static_cast( std::pow(4.0*MathConst::pi/3.0,1.0/3.0)); T_PR const vrel = mass_g*p1sm/(m1*g1s*m2*g2s*gc); - T_PR const sp = coeff * n1*n2/n12 * dt * vrel * (m1+m2) / + T_PR const sp = coeff * n12 * dt * vrel * (m1+m2) / amrex::max( m1*cbrt_n1*cbrt_n1, m2*cbrt_n2*cbrt_n2); diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H index 152dca4e1a1..30b466e2ec2 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -97,7 +97,6 @@ public: const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const /*n1*/, amrex::ParticleReal const /*n2*/, - amrex::ParticleReal const /*n12*/, amrex::ParticleReal const /*T1*/, amrex::ParticleReal const /*T2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index 1f3e5e56582..3113dc69839 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -132,7 +132,6 @@ public: const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, amrex::ParticleReal const /*n1*/, amrex::ParticleReal const /*n2*/, - amrex::ParticleReal const /*n12*/, amrex::ParticleReal const /*T1*/, amrex::ParticleReal const /*T2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, From e26f7ef0d42903a96cb67c3bf1fbe56af88712c1 Mon Sep 17 00:00:00 2001 From: Thomas Marks Date: Tue, 30 Jul 2024 02:34:10 +0300 Subject: [PATCH 175/190] Fix FFT python interface, take 2 (#5081) --- Python/pywarpx/picmi.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 42c0313a6e9..d06fe90d2d6 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1351,6 +1351,9 @@ def init(self, kw): def solver_initialize_inputs(self): + # Open BC means FieldBoundaryType::Open for electrostatic sims, rather than perfectly-matched layer + BC_map['open'] = 'open' + self.grid.grid_initialize_inputs() if self.relativistic: @@ -1371,6 +1374,8 @@ def solver_initialize_inputs(self): pywarpx.boundary.potential_hi_y = self.grid.potential_ymax pywarpx.boundary.potential_hi_z = self.grid.potential_zmax + pywarpx.warpx.poisson_solver = self.method + class GaussianLaser(picmistandard.PICMI_GaussianLaser): def laser_initialize_inputs(self): From 24c0711fc92577b3ed8c59af316c78d76894e10f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 29 Jul 2024 18:35:13 -0500 Subject: [PATCH 176/190] AMReX/pyAMReX/PICSAR: Weekly Update (#5095) * AMReX: Weekly Update * pyAMReX: Weekly Update * Increase Clang-Tidy `timeout-minutes` * Cast long long to int when using IParser --------- Co-authored-by: Edoardo Zoni --- .github/workflows/clang_tidy.yml | 1 + .github/workflows/cuda.yml | 2 +- Regression/WarpX-GPU-tests.ini | 2 +- Regression/WarpX-tests.ini | 2 +- Source/ablastr/utils/SignalHandling.cpp | 2 +- cmake/dependencies/AMReX.cmake | 2 +- cmake/dependencies/pyAMReX.cmake | 2 +- run_test.sh | 2 +- 8 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index 28b46b52530..af8632ed698 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -13,6 +13,7 @@ jobs: dim: [1, 2, RZ, 3] name: clang-tidy-${{ matrix.dim }}D runs-on: ubuntu-22.04 + timeout-minutes: 120 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index c2ec9aaa763..8f5e59ccaa4 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -115,7 +115,7 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 0c3273f5e591815909180f8ffaf5b793cabbf9bc && cd - + cd ../amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index a2447ac3cf5..68d59d850a0 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 0c3273f5e591815909180f8ffaf5b793cabbf9bc +branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 [source] dir = /home/regtester/git/WarpX diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index c1a6316e7e2..8e2f723b8c5 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 0c3273f5e591815909180f8ffaf5b793cabbf9bc +branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 [source] dir = /home/regtester/AMReX_RegTesting/warpx diff --git a/Source/ablastr/utils/SignalHandling.cpp b/Source/ablastr/utils/SignalHandling.cpp index 4095b9207ce..5eeaeec259f 100644 --- a/Source/ablastr/utils/SignalHandling.cpp +++ b/Source/ablastr/utils/SignalHandling.cpp @@ -104,7 +104,7 @@ SignalHandling::parseSignalNameToNumber (const std::string &str) auto spf = signals_parser.compileHost<0>(); - const int sig = spf(); + const auto sig = int(spf()); ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(sig < NUM_SIGNALS, "Parsed signal value is outside the supported range of [1, 31]"); diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 025b4d3b6e0..f1e5b3f62e2 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "0c3273f5e591815909180f8ffaf5b793cabbf9bc" +set(WarpX_amrex_branch "20e6f2eadf0c297517588ba38973ec7c7084fa31" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index a4598953432..133747aaeae 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "ff4643869c63d4ee40a87054b901f61eefcb97a3" +set(WarpX_pyamrex_branch "e007e730d48cb5fdbe1e10462d7d0a14e2bc8f8e" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/run_test.sh b/run_test.sh index 1692f66263c..f56a957c17a 100755 --- a/run_test.sh +++ b/run_test.sh @@ -68,7 +68,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 0c3273f5e591815909180f8ffaf5b793cabbf9bc && cd - +cd amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets From 3413568d0d372d508de3c56ffdebd25b884f8d97 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:36:10 -0700 Subject: [PATCH 177/190] CI: remove unused parameter `analysisOutputImage` (#5097) --- Regression/WarpX-tests.ini | 45 -------------------------------------- 1 file changed, 45 deletions(-) diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 8e2f723b8c5..a1dc0a168f7 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -380,7 +380,6 @@ useOMP = 1 numthreads = 1 runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/dive_cleaning/analysis.py -analysisOutputImage = Comparison.png [dive_cleaning_3d] buildDir = . @@ -395,7 +394,6 @@ useOMP = 1 numthreads = 1 runtime_params = analysisRoutine = Examples/Tests/dive_cleaning/analysis.py -analysisOutputImage = Comparison.png [ElectrostaticSphere] buildDir = . @@ -946,7 +944,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_fluid_1D] buildDir = . @@ -961,7 +958,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_1d.py -analysisOutputImage = langmuir_fluid_multi_1d_analysis.png [Langmuir_fluid_RZ] buildDir = . @@ -976,7 +972,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_rz.py -analysisOutputImage = langmuir_fluid_rz_analysis.png [Langmuir_fluid_2D] buildDir = . @@ -991,7 +986,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_2d.py -analysisOutputImage = langmuir_fluid_multi_2d_analysis.png [Langmuir_fluid_multi] buildDir = . @@ -1006,7 +1000,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_3d.py -analysisOutputImage = langmuir_fluid_multi_analysis.png [Langmuir_multi_1d] buildDir = . @@ -1021,7 +1014,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_1d.py -analysisOutputImage = langmuir_multi_1d_analysis.png [Langmuir_multi_2d_MR] buildDir = . @@ -1036,7 +1028,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_anisotropic] buildDir = . @@ -1051,7 +1042,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_momentum_conserving] buildDir = . @@ -1066,7 +1056,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR_momentum_conserving.png [Langmuir_multi_2d_MR_psatd] buildDir = . @@ -1081,7 +1070,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR_psatd.png [Langmuir_multi_2d_nodal] buildDir = . @@ -1096,7 +1084,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd] buildDir = . @@ -1111,7 +1098,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction] buildDir = . @@ -1126,7 +1112,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction_nodal] buildDir = . @@ -1141,7 +1126,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_momentum_conserving] buildDir = . @@ -1156,7 +1140,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_multiJ] buildDir = . @@ -1171,7 +1154,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_psatd_multiJ.png [Langmuir_multi_2d_psatd_multiJ_nodal] buildDir = . @@ -1186,7 +1168,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_psatd_multiJ_nodal.png [Langmuir_multi_2d_psatd_nodal] buildDir = . @@ -1201,7 +1182,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition] buildDir = . @@ -1216,7 +1196,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4] buildDir = . @@ -1231,7 +1210,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_nodal] buildDir = . @@ -1246,7 +1224,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_nodal] buildDir = . @@ -1261,7 +1238,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd] buildDir = . @@ -1276,7 +1252,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_current_correction] buildDir = . @@ -1291,7 +1266,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_current_correction_nodal] buildDir = . @@ -1306,7 +1280,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_div_cleaning] buildDir = . @@ -1321,7 +1294,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_momentum_conserving] buildDir = . @@ -1336,7 +1308,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_multiJ] buildDir = . @@ -1351,7 +1322,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = Langmuir_multi_psatd_multiJ.png [Langmuir_multi_psatd_multiJ_nodal] buildDir = . @@ -1366,7 +1336,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = Langmuir_multi_psatd_multiJ_nodal.png [Langmuir_multi_psatd_nodal] buildDir = . @@ -1381,7 +1350,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_single_precision] buildDir = . @@ -1396,7 +1364,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_Vay_deposition] buildDir = . @@ -1411,7 +1378,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_Vay_deposition_nodal] buildDir = . @@ -1426,7 +1392,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_rz] buildDir = . @@ -1441,7 +1406,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd] @@ -1457,7 +1421,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_psatd_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_current_correction] @@ -1473,7 +1436,6 @@ numprocs = 1 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_psatd_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_multiJ] @@ -1489,7 +1451,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_psatd_multiJ_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_single_precision] @@ -1505,7 +1466,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Larmor] buildDir = . @@ -1677,7 +1637,6 @@ numprocs = 2 useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/laser_injection/analysis_laser.py -analysisOutputImage = laser_analysis.png [LaserInjection_1d] buildDir = . @@ -3487,7 +3446,6 @@ useOMP = 1 numthreads = 1 runtime_params = analysisRoutine = Examples/Tests/relativistic_space_charge_initialization/analysis.py -analysisOutputImage = Comparison.png [RepellingParticles] buildDir = . @@ -3701,7 +3659,6 @@ useOMP = 1 numthreads = 1 runtime_params = analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py -analysisOutputImage = Comparison.png [space_charge_initialization_2d] buildDir = . @@ -3716,7 +3673,6 @@ useOMP = 1 numthreads = 1 runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py -analysisOutputImage = Comparison.png [subcyclingMR] buildDir = . @@ -3848,7 +3804,6 @@ useOMP = 1 numthreads = 1 outputFile = spacecraft_charging_plt analysisRoutine = Examples/Physics_applications/spacecraft_charging/analysis.py -analysisOutputImage = min_phi_analysis.png [Point_of_contact_EB_3d] buildDir = . From 4ac5962ef4fce6e4f812ab64bed2f584ea412e91 Mon Sep 17 00:00:00 2001 From: Olga Shapoval <30510597+oshapoval@users.noreply.github.com> Date: Mon, 29 Jul 2024 17:07:18 -0700 Subject: [PATCH 178/190] Fix bug with ES solver and MR: `E_aux=E_fp` in `UpdateAuxilaryData` (#4922) * Removal of asserttion which prevented from usung the averaged PSATD algorithms with PML BC * Clean-up * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fixed to arr_aux(j,k,l) = fine when ES solve is used * Removed temporary print statements * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Clean-up * Added CI test ElectrostaticSphereEB_RZ_MR_lev_1 to check the fields on the level=1 * Updated becnmarks for ElectrostaticSphereLabFrame_MR_emass_10 * Imported regular expression (re) in the analysis script. * Fixed typo * United two CI tests for different levels of MR in one test. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated CI test ElectrostaticSphereEB_RZ_MR and the corresponding analysis script with smaller MR patch. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Clean-up * Do deepcopy for lev>0 and collocated grid * Fix bugs to resolve failure of CI tests * Preserve plotfile output, update benchmark file * Working on CI test * Update benchmark of `ElectrostaticSphereEB_RZ_MR` * Initialize with value all `aux`, `cax` fields * Remove changes related to averaged Galilean PSATD with PML * Remove style changes (e.g., changes to empty lines) * Replace `MFIter`/`ParallelFor` loop with simple copy * Apply suggestions from code review * Revert part of the code to its previous, equivalent state * Removed DeepCopy & no need for ghost cells in temp phi_cp. * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update Source/ablastr/fields/PoissonSolver.H * Add inline comments * Update analysis script * Use `TilingIfNotGPU`, `growntilebox` in E loop --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Edoardo Zoni Co-authored-by: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Co-authored-by: Remi Lehe Co-authored-by: Weiqun Zhang --- .../electrostatic_sphere_eb/analysis_rz_mr.py | 99 ++++ .../electrostatic_sphere_eb/inputs_rz_mr | 1 + .../ElectrostaticSphereEB_RZ_MR.json | 8 +- ...ectrostaticSphereLabFrame_MR_emass_10.json | 12 +- Regression/WarpX-tests.ini | 5 +- Source/Parallelization/WarpXComm.cpp | 510 ++++++++++-------- Source/Parallelization/WarpXComm_K.H | 90 +++- Source/ablastr/fields/PoissonSolver.H | 21 +- 8 files changed, 508 insertions(+), 238 deletions(-) create mode 100755 Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py diff --git a/Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py b/Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py new file mode 100755 index 00000000000..0b01b128362 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +# Copyright 2024 Olga Shapoval, Edoardo Zoni +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the embedded boundary in RZ. +# A cylindrical surface (r=0.1) has a fixed potential 1 V. +# The outer surface has 0 V fixed. +# Thus the analytical solution has the form: +# phi(r) = A+B*log(r), Er(r) = -B/r. + +import os +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +tolerance = 0.004 +print(f'tolerance = {tolerance}') + +fn = sys.argv[1] + +def find_first_non_zero_from_bottom_left(matrix): + for i in range(matrix.shape[0]): + for j in range(matrix.shape[1]): + if (matrix[i][j] != 0) and (matrix[i][j] != np.nan): + return (i, j) + return i, j + +def find_first_non_zero_from_upper_right(matrix): + for i in range(matrix.shape[0]-1, -1, -1): + for j in range(matrix.shape[1]-1, -1, -1): + if (matrix[i][j] != 0) and (matrix[i][j] != np.nan): + return (i, j) + return i,j + +def get_fields(ts, level): + if level == 0: + Er, info = ts.get_field('E', 'r', iteration=0) + phi, info = ts.get_field('phi', iteration=0) + else: + Er, info = ts.get_field(f'E_lvl{level}', 'r', iteration=0) + phi, info = ts.get_field(f'phi_lvl{level}', iteration=0) + return Er, phi, info + +def get_error_per_lev(ts,level): + Er, phi, info = get_fields(ts, level) + + nr_half = info.r.shape[0] // 2 + dr = info.dr + + Er_patch = Er[:,nr_half:] + phi_patch = phi[:,nr_half:] + r1 = info.r[nr_half:] + patch_left_lower_i, patch_left_lower_j = find_first_non_zero_from_bottom_left(Er_patch) + patch_right_upper_i, patch_right_upper_j = find_first_non_zero_from_upper_right(Er_patch) + + # phi and Er field on the MR patch + phi_sim = phi_patch[patch_left_lower_i:patch_right_upper_i+1, patch_left_lower_j:patch_right_upper_j+1] + Er_sim = Er_patch[patch_left_lower_i:patch_right_upper_i+1, patch_left_lower_j:patch_right_upper_j+1] + r = r1[patch_left_lower_j:patch_right_upper_j+1] + + B = 1.0/np.log(0.1/0.5) + A = -B*np.log(0.5) + + # outside EB and last cutcell + rmin = np.min(np.argwhere(r >= (0.1+dr))) + rmax = -1 + r = r[rmin:rmax] + phi_sim = phi_sim[:,rmin:rmax] + Er_sim = Er_sim[:,rmin:rmax] + + phi_theory = A + B*np.log(r) + phi_theory = np.tile(phi_theory, (phi_sim.shape[0],1)) + phi_error = np.max(np.abs(phi_theory-phi_sim) / np.abs(phi_theory)) + + Er_theory = -B/r + Er_theory = np.tile(Er_theory, (Er_sim.shape[0],1)) + Er_error = np.max(np.abs(Er_theory-Er_sim) / np.abs(Er_theory)) + + print(f'max error of phi[lev={level}]: {phi_error}') + print(f'max error of Er[lev={level}]: {Er_error}') + assert(phi_error < tolerance) + assert(Er_error < tolerance) + +ts = OpenPMDTimeSeries(fn) +level_fields = [field for field in ts.avail_fields if 'lvl' in field] +nlevels = 0 if level_fields == [] else int(level_fields[-1][-1]) +for level in range(nlevels+1): + get_error_per_lev(ts,level) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn, output_format="openpmd") diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr b/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr index 3bea63d76fb..722fc916416 100644 --- a/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr +++ b/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr @@ -30,3 +30,4 @@ diagnostics.diags_names = diag1 diag1.intervals = 1 diag1.diag_type = Full diag1.fields_to_plot = Er phi +diag1.format = openpmd diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json index ffa8d68f9d9..6bbfce0e3b3 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json @@ -1,10 +1,10 @@ { "lev=0": { - "Er": 8487.661571739109, - "phi": 2036.0428085225362 + "Er": 16975.32314347822, + "phi": 4072.085617045073 }, "lev=1": { - "Er": 19519.172334977942, - "phi": 3291.0262856782897 + "Er": 26818.189739547757, + "phi_lvl1": 8731.176548788893 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json index 024127a1bd2..21d5208c59a 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json @@ -6,15 +6,15 @@ "rho": 0.0 }, "lev=1": { - "Ex": 14.281015560380963, - "Ey": 14.281015560380965, - "Ez": 14.281015560380965, + "Ex": 7.170105936287823, + "Ey": 7.17010593628782, + "Ez": 7.170105936287821, "rho": 2.6092568008333786e-10 }, "electron": { - "particle_momentum_x": 1.80842228672388e-24, - "particle_momentum_y": 1.8084222867238806e-24, - "particle_momentum_z": 1.7598771525647628e-24, + "particle_momentum_x": 9.257577597262615e-25, + "particle_momentum_y": 9.257577597262618e-25, + "particle_momentum_z": 9.257577597262624e-25, "particle_position_x": 327.46875, "particle_position_y": 327.46875, "particle_position_z": 327.46875, diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index a1dc0a168f7..8048318de7a 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -483,7 +483,7 @@ analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereEB_RZ_MR] buildDir = . inputFile = Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr -runtime_params = warpx.abort_on_warning_threshold = medium +runtime_params = warpx.abort_on_warning_threshold = medium amr.ref_ratio_vect = 2 2 2 dim = 2 addToCompileString = USE_EB=TRUE USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON @@ -492,7 +492,8 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py +outputFile = ElectrostaticSphereEB_RZ_MR_plt +analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py [ElectrostaticSphereLabFrame] buildDir = . diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index e7df489236e..2887bd4d056 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -171,136 +171,191 @@ WarpX::UpdateAuxilaryDataStagToNodal () // Bfield { - Array,3> Btmp; - if (Bfield_cax[lev][0]) { - for (int i = 0; i < 3; ++i) { - Btmp[i] = std::make_unique( - *Bfield_cax[lev][i], amrex::make_alias, 0, 1); + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { + Array,3> Btmp; + if (Bfield_cax[lev][0]) { + for (int i = 0; i < 3; ++i) { + Btmp[i] = std::make_unique( + *Bfield_cax[lev][i], amrex::make_alias, 0, 1); + } + } else { + const IntVect ngtmp = Bfield_aux[lev-1][0]->nGrowVect(); + for (int i = 0; i < 3; ++i) { + Btmp[i] = std::make_unique(cnba, dm, 1, ngtmp); + } } - } else { - const IntVect ngtmp = Bfield_aux[lev-1][0]->nGrowVect(); + Btmp[0]->setVal(0.0); + Btmp[1]->setVal(0.0); + Btmp[2]->setVal(0.0); + // ParallelCopy from coarse level for (int i = 0; i < 3; ++i) { - Btmp[i] = std::make_unique(cnba, dm, 1, ngtmp); + const IntVect ng = Btmp[i]->nGrowVect(); + // Guard cells may not be up to date beyond ng_FieldGather + const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + // Copy Bfield_aux to Btmp, using up to ng_src (=ng_FieldGather) guard cells from + // Bfield_aux and filling up to ng (=nGrow) guard cells in Btmp + ablastr::utils::communication::ParallelCopy(*Btmp[i], *Bfield_aux[lev - 1][i], 0, 0, 1, + ng_src, ng, WarpX::do_single_precision_comms, cperiod); } - } - Btmp[0]->setVal(0.0); - Btmp[1]->setVal(0.0); - Btmp[2]->setVal(0.0); - // ParallelCopy from coarse level - for (int i = 0; i < 3; ++i) { - const IntVect ng = Btmp[i]->nGrowVect(); - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; - // Copy Bfield_aux to Btmp, using up to ng_src (=ng_FieldGather) guard cells from - // Bfield_aux and filling up to ng (=nGrow) guard cells in Btmp - ablastr::utils::communication::ParallelCopy(*Btmp[i], *Bfield_aux[lev - 1][i], 0, 0, 1, - ng_src, ng, WarpX::do_single_precision_comms, cperiod); - } - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); - const amrex::IntVect& Bx_cp_stag = Bfield_cp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& By_cp_stag = Bfield_cp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_cp_stag = Bfield_cp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_cp_stag = Bfield_cp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_cp_stag = Bfield_cp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_cp_stag = Bfield_cp[lev][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Bfield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); - Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); - Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); - Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); - Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); - Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); - Array4 const& bx_cp = Bfield_cp[lev][0]->const_array(mfi); - Array4 const& by_cp = Bfield_cp[lev][1]->const_array(mfi); - Array4 const& bz_cp = Bfield_cp[lev][2]->const_array(mfi); - Array4 const& bx_c = Btmp[0]->const_array(mfi); - Array4 const& by_c = Btmp[1]->const_array(mfi); - Array4 const& bz_c = Btmp[2]->const_array(mfi); - - const Box& bx = mfi.growntilebox(); - amrex::ParallelFor(bx, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Bfield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, bx_aux, bx_fp, bx_cp, bx_c, Bx_fp_stag, Bx_cp_stag, refinement_ratio); - warpx_interp(j, k, l, by_aux, by_fp, by_cp, by_c, By_fp_stag, By_cp_stag, refinement_ratio); - warpx_interp(j, k, l, bz_aux, bz_fp, bz_cp, bz_c, Bz_fp_stag, Bz_cp_stag, refinement_ratio); - }); + Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); + Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); + Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); + Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); + Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); + Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); + Array4 const& bx_cp = Bfield_cp[lev][0]->const_array(mfi); + Array4 const& by_cp = Bfield_cp[lev][1]->const_array(mfi); + Array4 const& bz_cp = Bfield_cp[lev][2]->const_array(mfi); + Array4 const& bx_c = Btmp[0]->const_array(mfi); + Array4 const& by_c = Btmp[1]->const_array(mfi); + Array4 const& bz_c = Btmp[2]->const_array(mfi); + + const Box& bx = mfi.growntilebox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bx_aux, bx_fp, bx_cp, bx_c, Bx_fp_stag, Bx_cp_stag, refinement_ratio); + warpx_interp(j, k, l, by_aux, by_fp, by_cp, by_c, By_fp_stag, By_cp_stag, refinement_ratio); + warpx_interp(j, k, l, bz_aux, bz_fp, bz_cp, bz_c, Bz_fp_stag, Bz_cp_stag, refinement_ratio); + }); + } + } + else { // electrostatic + const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*Bfield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); + Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); + Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); + Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); + Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); + Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); + + const Box& bx = mfi.growntilebox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bx_aux, bx_fp, Bx_fp_stag); + warpx_interp(j, k, l, by_aux, by_fp, By_fp_stag); + warpx_interp(j, k, l, bz_aux, bz_fp, Bz_fp_stag); + }); + } } } - // Efield { - Array,3> Etmp; - if (Efield_cax[lev][0]) { - for (int i = 0; i < 3; ++i) { - Etmp[i] = std::make_unique( - *Efield_cax[lev][i], amrex::make_alias, 0, 1); + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { + Array,3> Etmp; + if (Efield_cax[lev][0]) { + for (int i = 0; i < 3; ++i) { + Etmp[i] = std::make_unique( + *Efield_cax[lev][i], amrex::make_alias, 0, 1); + } + } else { + const IntVect ngtmp = Efield_aux[lev-1][0]->nGrowVect(); + for (int i = 0; i < 3; ++i) { + Etmp[i] = std::make_unique( + cnba, dm, 1, ngtmp); + } } - } else { - const IntVect ngtmp = Efield_aux[lev-1][0]->nGrowVect(); + Etmp[0]->setVal(0.0); + Etmp[1]->setVal(0.0); + Etmp[2]->setVal(0.0); + // ParallelCopy from coarse level for (int i = 0; i < 3; ++i) { - Etmp[i] = std::make_unique( - cnba, dm, 1, ngtmp); + const IntVect ng = Etmp[i]->nGrowVect(); + // Guard cells may not be up to date beyond ng_FieldGather + const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + // Copy Efield_aux to Etmp, using up to ng_src (=ng_FieldGather) guard cells from + // Efield_aux and filling up to ng (=nGrow) guard cells in Etmp + ablastr::utils::communication::ParallelCopy(*Etmp[i], *Efield_aux[lev - 1][i], 0, 0, 1, + ng_src, ng, WarpX::do_single_precision_comms, cperiod); } - } - Etmp[0]->setVal(0.0); - Etmp[1]->setVal(0.0); - Etmp[2]->setVal(0.0); - // ParallelCopy from coarse level - for (int i = 0; i < 3; ++i) { - const IntVect ng = Etmp[i]->nGrowVect(); - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; - // Copy Efield_aux to Etmp, using up to ng_src (=ng_FieldGather) guard cells from - // Efield_aux and filling up to ng (=nGrow) guard cells in Etmp - ablastr::utils::communication::ParallelCopy(*Etmp[i], *Efield_aux[lev - 1][i], 0, 0, 1, - ng_src, ng, WarpX::do_single_precision_comms, cperiod); - } - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); - const amrex::IntVect& Ex_cp_stag = Efield_cp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_cp_stag = Efield_cp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_cp_stag = Efield_cp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_cp_stag = Efield_cp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_cp_stag = Efield_cp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_cp_stag = Efield_cp[lev][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) - { - Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); - Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); - Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); - Array4 const& ex_cp = Efield_cp[lev][0]->const_array(mfi); - Array4 const& ey_cp = Efield_cp[lev][1]->const_array(mfi); - Array4 const& ez_cp = Efield_cp[lev][2]->const_array(mfi); - Array4 const& ex_c = Etmp[0]->const_array(mfi); - Array4 const& ey_c = Etmp[1]->const_array(mfi); - Array4 const& ez_c = Etmp[2]->const_array(mfi); - - const Box& bx = mfi.fabbox(); - amrex::ParallelFor(bx, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, ex_aux, ex_fp, ex_cp, ex_c, Ex_fp_stag, Ex_cp_stag, refinement_ratio); - warpx_interp(j, k, l, ey_aux, ey_fp, ey_cp, ey_c, Ey_fp_stag, Ey_cp_stag, refinement_ratio); - warpx_interp(j, k, l, ez_aux, ez_fp, ez_cp, ez_c, Ez_fp_stag, Ez_cp_stag, refinement_ratio); - }); + Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); + Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); + Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); + Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); + Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); + Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_cp = Efield_cp[lev][0]->const_array(mfi); + Array4 const& ey_cp = Efield_cp[lev][1]->const_array(mfi); + Array4 const& ez_cp = Efield_cp[lev][2]->const_array(mfi); + Array4 const& ex_c = Etmp[0]->const_array(mfi); + Array4 const& ey_c = Etmp[1]->const_array(mfi); + Array4 const& ez_c = Etmp[2]->const_array(mfi); + + const Box& bx = mfi.fabbox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ex_aux, ex_fp, ex_cp, ex_c, Ex_fp_stag, Ex_cp_stag, refinement_ratio); + warpx_interp(j, k, l, ey_aux, ey_fp, ey_cp, ey_c, Ey_fp_stag, Ey_cp_stag, refinement_ratio); + warpx_interp(j, k, l, ez_aux, ez_fp, ez_cp, ez_c, Ez_fp_stag, Ez_cp_stag, refinement_ratio); + }); + } + } + else { // electrostatic + const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*Efield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); + Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); + Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); + Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); + Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); + Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + + const Box& bx = mfi.growntilebox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ex_aux, ex_fp, Ex_fp_stag); + warpx_interp(j, k, l, ey_aux, ey_fp, Ey_fp_stag); + warpx_interp(j, k, l, ez_aux, ez_fp, Ez_fp_stag); + }); + } } } } @@ -341,141 +396,158 @@ WarpX::UpdateAuxilaryDataSameType () // B field { - MultiFab dBx(Bfield_cp[lev][0]->boxArray(), dm, Bfield_cp[lev][0]->nComp(), ng); - MultiFab dBy(Bfield_cp[lev][1]->boxArray(), dm, Bfield_cp[lev][1]->nComp(), ng); - MultiFab dBz(Bfield_cp[lev][2]->boxArray(), dm, Bfield_cp[lev][2]->nComp(), ng); - dBx.setVal(0.0); - dBy.setVal(0.0); - dBz.setVal(0.0); - - // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard - // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs - - ablastr::utils::communication::ParallelCopy(dBx, *Bfield_aux[lev - 1][0], 0, 0, - Bfield_aux[lev - 1][0]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dBy, *Bfield_aux[lev - 1][1], 0, 0, - Bfield_aux[lev - 1][1]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dBz, *Bfield_aux[lev - 1][2], 0, 0, - Bfield_aux[lev - 1][2]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, - crse_period); - - if (Bfield_cax[lev][0]) + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab::Copy(*Bfield_cax[lev][0], dBx, 0, 0, Bfield_cax[lev][0]->nComp(), ng); - MultiFab::Copy(*Bfield_cax[lev][1], dBy, 0, 0, Bfield_cax[lev][1]->nComp(), ng); - MultiFab::Copy(*Bfield_cax[lev][2], dBz, 0, 0, Bfield_cax[lev][2]->nComp(), ng); - } - MultiFab::Subtract(dBx, *Bfield_cp[lev][0], 0, 0, Bfield_cp[lev][0]->nComp(), ng); - MultiFab::Subtract(dBy, *Bfield_cp[lev][1], 0, 0, Bfield_cp[lev][1]->nComp(), ng); - MultiFab::Subtract(dBz, *Bfield_cp[lev][2], 0, 0, Bfield_cp[lev][2]->nComp(), ng); + MultiFab dBx(Bfield_cp[lev][0]->boxArray(), dm, Bfield_cp[lev][0]->nComp(), ng); + MultiFab dBy(Bfield_cp[lev][1]->boxArray(), dm, Bfield_cp[lev][1]->nComp(), ng); + MultiFab dBz(Bfield_cp[lev][2]->boxArray(), dm, Bfield_cp[lev][2]->nComp(), ng); + dBx.setVal(0.0); + dBy.setVal(0.0); + dBz.setVal(0.0); + + // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard + // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs + + ablastr::utils::communication::ParallelCopy(dBx, *Bfield_aux[lev - 1][0], 0, 0, + Bfield_aux[lev - 1][0]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dBy, *Bfield_aux[lev - 1][1], 0, 0, + Bfield_aux[lev - 1][1]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dBz, *Bfield_aux[lev - 1][2], 0, 0, + Bfield_aux[lev - 1][2]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, + crse_period); + + if (Bfield_cax[lev][0]) + { + MultiFab::Copy(*Bfield_cax[lev][0], dBx, 0, 0, Bfield_cax[lev][0]->nComp(), ng); + MultiFab::Copy(*Bfield_cax[lev][1], dBy, 0, 0, Bfield_cax[lev][1]->nComp(), ng); + MultiFab::Copy(*Bfield_cax[lev][2], dBz, 0, 0, Bfield_cax[lev][2]->nComp(), ng); + } + MultiFab::Subtract(dBx, *Bfield_cp[lev][0], 0, 0, Bfield_cp[lev][0]->nComp(), ng); + MultiFab::Subtract(dBy, *Bfield_cp[lev][1], 0, 0, Bfield_cp[lev][1]->nComp(), ng); + MultiFab::Subtract(dBz, *Bfield_cp[lev][2], 0, 0, Bfield_cp[lev][2]->nComp(), ng); - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Bx_stag = Bfield_aux[lev-1][0]->ixType().toIntVect(); - const amrex::IntVect& By_stag = Bfield_aux[lev-1][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_stag = Bfield_aux[lev-1][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_stag = Bfield_aux[lev-1][0]->ixType().toIntVect(); + const amrex::IntVect& By_stag = Bfield_aux[lev-1][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_stag = Bfield_aux[lev-1][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Bfield_aux[lev][0]); mfi.isValid(); ++mfi) - { - Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); - Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); - Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); - Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); - Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); - Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); - Array4 const& bx_c = dBx.const_array(mfi); - Array4 const& by_c = dBy.const_array(mfi); - Array4 const& bz_c = dBz.const_array(mfi); - - amrex::ParallelFor(Box(bx_aux), Box(by_aux), Box(bz_aux), - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, bx_aux, bx_fp, bx_c, Bx_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, by_aux, by_fp, by_c, By_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Bfield_aux[lev][0]); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, bz_aux, bz_fp, bz_c, Bz_stag, refinement_ratio); - }); + Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); + Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); + Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); + Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); + Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); + Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); + Array4 const& bx_c = dBx.const_array(mfi); + Array4 const& by_c = dBy.const_array(mfi); + Array4 const& bz_c = dBz.const_array(mfi); + + amrex::ParallelFor(Box(bx_aux), Box(by_aux), Box(bz_aux), + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bx_aux, bx_fp, bx_c, Bx_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, by_aux, by_fp, by_c, By_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bz_aux, bz_fp, bz_c, Bz_stag, refinement_ratio); + }); + } + } + else // electrostatic + { + MultiFab::Copy(*Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, 0, Bfield_aux[lev][0]->nComp(), Bfield_aux[lev][0]->nGrowVect()); + MultiFab::Copy(*Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, 0, Bfield_aux[lev][1]->nComp(), Bfield_aux[lev][1]->nGrowVect()); + MultiFab::Copy(*Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, 0, Bfield_aux[lev][2]->nComp(), Bfield_aux[lev][2]->nGrowVect()); } } - // E field { - MultiFab dEx(Efield_cp[lev][0]->boxArray(), dm, Efield_cp[lev][0]->nComp(), ng); - MultiFab dEy(Efield_cp[lev][1]->boxArray(), dm, Efield_cp[lev][1]->nComp(), ng); - MultiFab dEz(Efield_cp[lev][2]->boxArray(), dm, Efield_cp[lev][2]->nComp(), ng); - dEx.setVal(0.0); - dEy.setVal(0.0); - dEz.setVal(0.0); - - // Copy Efield_aux to the dE MultiFabs, using up to ng_src (=ng_FieldGather) guard - // cells from Efield_aux and filling up to ng (=nGrow) guard cells in the dE MultiFabs - ablastr::utils::communication::ParallelCopy(dEx, *Efield_aux[lev - 1][0], 0, 0, - Efield_aux[lev - 1][0]->nComp(), ng_src, ng, - WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dEy, *Efield_aux[lev - 1][1], 0, 0, - Efield_aux[lev - 1][1]->nComp(), ng_src, ng, - WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dEz, *Efield_aux[lev - 1][2], 0, 0, - Efield_aux[lev - 1][2]->nComp(), ng_src, ng, - WarpX::do_single_precision_comms, - crse_period); - - if (Efield_cax[lev][0]) + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab::Copy(*Efield_cax[lev][0], dEx, 0, 0, Efield_cax[lev][0]->nComp(), ng); - MultiFab::Copy(*Efield_cax[lev][1], dEy, 0, 0, Efield_cax[lev][1]->nComp(), ng); - MultiFab::Copy(*Efield_cax[lev][2], dEz, 0, 0, Efield_cax[lev][2]->nComp(), ng); - } - MultiFab::Subtract(dEx, *Efield_cp[lev][0], 0, 0, Efield_cp[lev][0]->nComp(), ng); - MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, Efield_cp[lev][1]->nComp(), ng); - MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, Efield_cp[lev][2]->nComp(), ng); + MultiFab dEx(Efield_cp[lev][0]->boxArray(), dm, Efield_cp[lev][0]->nComp(), ng); + MultiFab dEy(Efield_cp[lev][1]->boxArray(), dm, Efield_cp[lev][1]->nComp(), ng); + MultiFab dEz(Efield_cp[lev][2]->boxArray(), dm, Efield_cp[lev][2]->nComp(), ng); + dEx.setVal(0.0); + dEy.setVal(0.0); + dEz.setVal(0.0); + + // Copy Efield_aux to the dE MultiFabs, using up to ng_src (=ng_FieldGather) guard + // cells from Efield_aux and filling up to ng (=nGrow) guard cells in the dE MultiFabs + ablastr::utils::communication::ParallelCopy(dEx, *Efield_aux[lev - 1][0], 0, 0, + Efield_aux[lev - 1][0]->nComp(), ng_src, ng, + WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dEy, *Efield_aux[lev - 1][1], 0, 0, + Efield_aux[lev - 1][1]->nComp(), ng_src, ng, + WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dEz, *Efield_aux[lev - 1][2], 0, 0, + Efield_aux[lev - 1][2]->nComp(), ng_src, ng, + WarpX::do_single_precision_comms, + crse_period); + + if (Efield_cax[lev][0]) + { + MultiFab::Copy(*Efield_cax[lev][0], dEx, 0, 0, Efield_cax[lev][0]->nComp(), ng); + MultiFab::Copy(*Efield_cax[lev][1], dEy, 0, 0, Efield_cax[lev][1]->nComp(), ng); + MultiFab::Copy(*Efield_cax[lev][2], dEz, 0, 0, Efield_cax[lev][2]->nComp(), ng); + } + MultiFab::Subtract(dEx, *Efield_cp[lev][0], 0, 0, Efield_cp[lev][0]->nComp(), ng); + MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, Efield_cp[lev][1]->nComp(), ng); + MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, Efield_cp[lev][2]->nComp(), ng); - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Ex_stag = Efield_aux[lev-1][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_stag = Efield_aux[lev-1][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_stag = Efield_aux[lev-1][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_stag = Efield_aux[lev-1][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_stag = Efield_aux[lev-1][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_stag = Efield_aux[lev-1][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) - { - Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); - Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); - Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); - Array4 const& ex_c = dEx.const_array(mfi); - Array4 const& ey_c = dEy.const_array(mfi); - Array4 const& ez_c = dEz.const_array(mfi); - - amrex::ParallelFor(Box(ex_aux), Box(ey_aux), Box(ez_aux), - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, ex_aux, ex_fp, ex_c, Ex_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, ey_aux, ey_fp, ey_c, Ey_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, ez_aux, ez_fp, ez_c, Ez_stag, refinement_ratio); - }); + Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); + Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); + Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); + Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); + Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); + Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_c = dEx.const_array(mfi); + Array4 const& ey_c = dEy.const_array(mfi); + Array4 const& ez_c = dEz.const_array(mfi); + + amrex::ParallelFor(Box(ex_aux), Box(ey_aux), Box(ez_aux), + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ex_aux, ex_fp, ex_c, Ex_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ey_aux, ey_fp, ey_c, Ey_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ez_aux, ez_fp, ez_c, Ez_stag, refinement_ratio); + }); + } + } + else // electrostatic + { + MultiFab::Copy(*Efield_aux[lev][0], *Efield_fp[lev][0], 0, 0, Efield_aux[lev][0]->nComp(), Efield_aux[lev][0]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][1], *Efield_fp[lev][1], 0, 0, Efield_aux[lev][1]->nComp(), Efield_aux[lev][1]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][2], *Efield_fp[lev][2], 0, 0, Efield_aux[lev][2]->nComp(), Efield_aux[lev][2]->nGrowVect()); } } } diff --git a/Source/Parallelization/WarpXComm_K.H b/Source/Parallelization/WarpXComm_K.H index a2b8fe38ed4..c3362087ad9 100644 --- a/Source/Parallelization/WarpXComm_K.H +++ b/Source/Parallelization/WarpXComm_K.H @@ -12,7 +12,7 @@ /** * \brief Interpolation function called within WarpX::UpdateAuxilaryDataSameType - * to interpolate data from the coarse and fine grids to the fine aux grid, + * with electromagnetic solver to interpolate data from the coarse and fine grids to the fine aux grid, * assuming that all grids have the same staggering (either collocated or staggered). * * \param[in] j index along x of the output array @@ -285,6 +285,94 @@ void warpx_interp (int j, int k, int l, // Final result arr_aux(j,k,l) = tmp + (fine - coarse); } +/** + * \brief Interpolation function called within WarpX::UpdateAuxilaryDataStagToNodal + * to interpolate data from the coarse and fine grids to the fine aux grid, + * with momentum-conserving field gathering, hence between grids with different staggering, + * and assuming that the aux grid is collocated. + * + * \param[in] j index along x of the output array + * \param[in] k index along y (in 3D) or z (in 2D) of the output array + * \param[in] l index along z (in 3D, l=0 in 2D) of the output array + * \param[in,out] arr_aux output array where interpolated values are stored + * \param[in] arr_fine input fine-patch array storing the values to interpolate + * \param[in] arr_fine_stag IndexType of the fine-patch arrays + */ +AMREX_GPU_DEVICE AMREX_FORCE_INLINE +void warpx_interp (int j, int k, int l, + amrex::Array4 const& arr_aux, + amrex::Array4 const& arr_fine, + const amrex::IntVect& arr_fine_stag) +{ + using namespace amrex; + + // Pad input arrays with zeros beyond ghost cells + // for out-of-bound accesses due to large-stencil operations + const auto arr_fine_zeropad = [arr_fine] (const int jj, const int kk, const int ll) noexcept + { + return arr_fine.contains(jj,kk,ll) ? arr_fine(jj,kk,ll) : 0.0_rt; + }; + + // NOTE Indices (j,k,l) in the following refer to: + // - (z,-,-) in 1D + // - (x,z,-) in 2D + // - (r,z,-) in RZ + // - (x,y,z) in 3D + + // Staggering of fine array (0: cell-centered; 1: nodal) + const int sj_fp = arr_fine_stag[0]; +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const int sk_fp = arr_fine_stag[1]; +#elif defined(WARPX_DIM_3D) + const int sk_fp = arr_fine_stag[1]; + const int sl_fp = arr_fine_stag[2]; +#endif + + // Number of points used for interpolation from coarse grid to fine grid + int nj; + int nk; + int nl; + + amrex::Real fine = 0.0_rt; + + // 3) Interpolation from fine staggered to fine nodal + + nj = (sj_fp == 0) ? 2 : 1; +#if defined(WARPX_DIM_1D_Z) + nk = 1; + nl = 1; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + nk = (sk_fp == 0) ? 2 : 1; + nl = 1; +#else + nk = (sk_fp == 0) ? 2 : 1; + nl = (sl_fp == 0) ? 2 : 1; +#endif + + const int jm = (sj_fp == 0) ? j-1 : j; +#if defined(WARPX_DIM_1D_Z) + const int km = k; + const int lm = l; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const int km = (sk_fp == 0) ? k-1 : k; + const int lm = l; +#else + const int km = (sk_fp == 0) ? k-1 : k; + const int lm = (sl_fp == 0) ? l-1 : l; +#endif + + for (int jj = 0; jj < nj; jj++) { + for (int kk = 0; kk < nk; kk++) { + for (int ll = 0; ll < nl; ll++) { + fine += arr_fine_zeropad(jm+jj,km+kk,lm+ll); + } + } + } + fine = fine/static_cast(nj*nk*nl); + + // Final result + arr_aux(j,k,l) = fine; +} /** * \brief Arbitrary-order interpolation function used to center a given MultiFab between two grids diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index ca262981010..589c6ec1835 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -263,10 +263,15 @@ computePhi (amrex::Vector const & rho, mlmg.setVerbose(verbosity); mlmg.setMaxIter(max_iters); mlmg.setAlwaysUseBNorm(always_use_bnorm); + if (WarpX::grid_type == GridType::Collocated) { + // In this case, computeE needs to use ghost nodes data. So we + // ask MLMG to fill BC for us after it solves the problem. + mlmg.setFinalFillBC(true); + } // Solve Poisson equation at lev mlmg.solve( {phi[lev]}, {rho[lev]}, - relative_tolerance, absolute_tolerance ); + relative_tolerance, absolute_tolerance ); // needed for solving the levels by levels: // - coarser level is initial guess for finer level @@ -280,10 +285,14 @@ computePhi (amrex::Vector const & rho, const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; ba.coarsen(refratio); const int ncomp = linop.getNComp(); - amrex::MultiFab phi_cp(ba, phi[lev+1]->DistributionMap(), ncomp, 1); + const int ng = (WarpX::grid_type == GridType::Collocated) ? 1 : 0; + amrex::MultiFab phi_cp(ba, phi[lev+1]->DistributionMap(), ncomp, ng); + if (ng > 0) { + // Set all values outside the domain to zero + phi_cp.setDomainBndry(0.0_rt, geom[lev]); + } // Copy from phi[lev] to phi_cp (in parallel) - const amrex::IntVect& ng = amrex::IntVect::TheUnitVector(); const amrex::Periodicity& crse_period = geom[lev].periodicity(); ablastr::utils::communication::ParallelCopy( @@ -292,8 +301,8 @@ computePhi (amrex::Vector const & rho, 0, 0, 1, - ng, - ng, + amrex::IntVect(0), + amrex::IntVect(ng), do_single_precision_comms, crse_period ); @@ -308,7 +317,7 @@ computePhi (amrex::Vector const & rho, details::PoissonInterpCPtoFP const interp(phi_fp_arr, phi_cp_arr, refratio); - amrex::Box const b = mfi.tilebox(phi[lev + 1]->ixType().toIntVect()); + amrex::Box const& b = mfi.growntilebox(ng); amrex::ParallelFor(b, interp); } From c8a2479a8b4cb94e8be7e6d2c268ec9bd8581a0f Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 30 Jul 2024 06:57:53 -0700 Subject: [PATCH 179/190] Enable tiling in some `MFIter` loops (#5096) --- Source/Parallelization/WarpXComm.cpp | 4 ++-- Source/Utils/WarpXMovingWindow.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index 2887bd4d056..6c44df061fd 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -306,7 +306,7 @@ WarpX::UpdateAuxilaryDataStagToNodal () #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) + for (MFIter mfi(*Efield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) { Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); @@ -321,7 +321,7 @@ WarpX::UpdateAuxilaryDataStagToNodal () Array4 const& ey_c = Etmp[1]->const_array(mfi); Array4 const& ez_c = Etmp[2]->const_array(mfi); - const Box& bx = mfi.fabbox(); + const Box& bx = mfi.growntilebox(); amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept { diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index d64394dafc9..73696838cd4 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -534,7 +534,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (amrex::MFIter mfi(tmpmf); mfi.isValid(); ++mfi ) + for (amrex::MFIter mfi(tmpmf, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { @@ -545,7 +545,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, auto const& dstfab = mf.array(mfi); auto const& srcfab = tmpmf.array(mfi); - const amrex::Box& outbox = mfi.fabbox() & adjBox; + const amrex::Box& outbox = mfi.growntilebox() & adjBox; if (outbox.ok()) { if (!useparser) { From 870577711ffe44bcba48c193af1f7246b98a4c21 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 30 Jul 2024 12:05:59 -0700 Subject: [PATCH 180/190] Add documentation for binomial smoother (#5099) --- Docs/source/usage/python.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 4d53b311764..c068447ddbc 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -53,6 +53,10 @@ Field solvers define the updates of electric and magnetic fields. .. autoclass:: pywarpx.picmi.ElectrostaticSolver +Object that allows smoothing of fields. + +.. autoclass:: pywarpx.picmi.BinomialSmoother + Constants --------- From 128ed9e9e5516a403245d8d6f4467edcfe9396e3 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 30 Jul 2024 17:35:43 -0700 Subject: [PATCH 181/190] Expose more diagnostics options in PICMI (#5100) --- Python/pywarpx/picmi.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index d06fe90d2d6..2aaa6b6122b 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -2329,6 +2329,11 @@ class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): warpx_openpmd_backend: {bp, h5, json}, optional Openpmd backend file format + warpx_openpmd_encoding: 'v' (variable based), 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Variable based is an experimental feature with ADIOS2. Default: `'f'`. + warpx_file_prefix: string, optional Prefix on the diagnostic file name @@ -2338,6 +2343,9 @@ class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): warpx_dump_rz_modes: bool, optional Flag whether to dump the data for all RZ modes + warpx_dump_last_timestep: bool, optional + If true, the last timestep is dumped regardless of the diagnostic period/intervals. + warpx_particle_fields_to_plot: list of ParticleFieldDiagnostics List of ParticleFieldDiagnostic classes to install in the simulation. Error checking is handled in the class itself. @@ -2355,9 +2363,11 @@ def init(self, kw): self.plot_crsepatch = kw.pop('warpx_plot_crsepatch', None) self.format = kw.pop('warpx_format', 'plotfile') self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.dump_rz_modes = kw.pop('warpx_dump_rz_modes', None) + self.dump_last_timestep = kw.pop('warpx_dump_last_timestep', None) self.particle_fields_to_plot = kw.pop('warpx_particle_fields_to_plot', []) self.particle_fields_species = kw.pop('warpx_particle_fields_species', None) @@ -2368,8 +2378,10 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'Full' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.dump_rz_modes = self.dump_rz_modes + self.diagnostic.dump_last_timestep = self.dump_last_timestep self.diagnostic.intervals = self.period self.diagnostic.diag_lo = self.lower_bound self.diagnostic.diag_hi = self.upper_bound @@ -2533,6 +2545,11 @@ class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnostic warpx_openpmd_backend: {bp, h5, json}, optional Openpmd backend file format + warpx_openpmd_encoding: 'v' (variable based), 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Variable based is an experimental feature with ADIOS2. Default: `'f'`. + warpx_file_prefix: string, optional Prefix on the diagnostic file name @@ -2551,6 +2568,9 @@ class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnostic a dictionary is given the keys should be species with the value specifying the stride for that species. + warpx_dump_last_timestep: bool, optional + If true, the last timestep is dumped regardless of the diagnostic period/intervals. + warpx_plot_filter_function: string, optional Analytic expression to down select the particles to in the diagnostic """ @@ -2558,11 +2578,13 @@ def init(self, kw): self.format = kw.pop('warpx_format', 'plotfile') self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.random_fraction = kw.pop('warpx_random_fraction', None) self.uniform_stride = kw.pop('warpx_uniform_stride', None) self.plot_filter_function = kw.pop('warpx_plot_filter_function', None) + self.dump_last_timestep = kw.pop('warpx_dump_last_timestep', None) self.user_defined_kw = {} if self.plot_filter_function is not None: @@ -2582,7 +2604,9 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'Full' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits + self.diagnostic.dump_last_timestep = self.dump_last_timestep self.diagnostic.intervals = self.period self.diagnostic.set_or_replace_attr('write_species', True) if 'fields_to_plot' not in self.diagnostic.argvattrs: @@ -2699,6 +2723,11 @@ class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic, warpx_openpmd_backend: string, optional Passed to .openpmd_backend + warpx_openpmd_encoding: 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Default: `'f'`. + warpx_file_prefix: string, optional Passed to .file_prefix @@ -2723,6 +2752,7 @@ def init(self, kw): self.format = kw.pop('warpx_format', None) self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.intervals = kw.pop('warpx_intervals', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) @@ -2737,6 +2767,7 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'BackTransformed' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.diag_lo = self.lower_bound self.diagnostic.diag_hi = self.upper_bound @@ -2808,6 +2839,11 @@ class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, warpx_openpmd_backend: string, optional Passed to .openpmd_backend + warpx_openpmd_encoding: 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Default: `'f'`. + warpx_file_prefix: string, optional Passed to .file_prefix @@ -2824,6 +2860,7 @@ class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, def init(self, kw): self.format = kw.pop('warpx_format', None) self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.intervals = kw.pop('warpx_intervals', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) @@ -2836,6 +2873,7 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'BackTransformed' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.do_back_transformed_particles = True From f10aa7070a6725b9979284cbe888c90b1395e237 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 31 Jul 2024 09:48:01 -0500 Subject: [PATCH 182/190] Doc: HPSF & Gov Links (#5085) Add agreed upon HPSF boilerplate text in README and manual. Update acknowledegments. --- Docs/source/acknowledgements.rst | 9 +++++++-- README.md | 10 +++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/Docs/source/acknowledgements.rst b/Docs/source/acknowledgements.rst index 14877839294..6d2529705d0 100644 --- a/Docs/source/acknowledgements.rst +++ b/Docs/source/acknowledgements.rst @@ -3,13 +3,18 @@ Funding and Acknowledgements ============================ -WarpX is supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration) responsible for the planning and preparation of a capable exascale ecosystem, including software, applications, hardware, advanced system engineering, and early testbed platforms, in support of the nation's exascale computing imperative. +WarpX is hosted by the High Performance Computing Foundation (HPSF). +If your organization wants to help steer the evolution of the HPC software ecosystem, visit `hpsf.io `__ and consider joining! WarpX is supported by the CAMPA collaboration, a project of the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research and Office of High Energy Physics, Scientific Discovery through Advanced Computing (SciDAC) program. -ABLASTR seed development is supported by the Laboratory Directed Research and Development Program of Lawrence Berkeley National Laboratory under U.S. Department of Energy Contract No. DE-AC02-05CH11231. +WarpX is supported by the KISMET project, a project of the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research and Office of Fusion Energy Science, Scientific Discovery through Advanced Computing (SciDAC) program. CEA-LIDYL actively contributes to the co-development of WarpX. As part of this initiative, WarpX also receives funding from the French National Research Agency (ANR - Plasm-On-Chip), the Horizon H2020 program and CEA. +WarpX was supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration) responsible for the planning and preparation of a capable exascale ecosystem, including software, applications, hardware, advanced system engineering, and early testbed platforms, in support of the nation's exascale computing imperative. + +ABLASTR seed development was supported by the Laboratory Directed Research and Development Program of Lawrence Berkeley National Laboratory under U.S. Department of Energy Contract No. DE-AC02-05CH11231. + We acknowledge all the contributors and users of the WarpX community who participate to the code quality with valuable code improvement and important feedback. diff --git a/README.md b/README.md index b2de2211b60..7760a004081 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Discussions](https://img.shields.io/badge/chat-discussions-turquoise.svg)](https://github.com/ECP-WarpX/WarpX/discussions) [![Supported Platforms](https://img.shields.io/badge/platforms-linux%20|%20osx%20|%20win-blue)](https://warpx.readthedocs.io/en/latest/install/users.html) [![GitHub commits since last release](https://img.shields.io/github/commits-since/ECP-WarpX/WarpX/latest/development.svg)](https://github.com/ECP-WarpX/WarpX/compare/development) -[![Exascale Computing Project](https://img.shields.io/badge/supported%20by-ECP-orange)](https://www.exascaleproject.org/research/) +[![HPSF](https://img.shields.io/badge/hosted%20by-HPSF-orange)](https://hpsf.io) [![Language: C++17](https://img.shields.io/badge/language-C%2B%2B17-orange.svg)](https://isocpp.org/) [![Language: Python](https://img.shields.io/badge/language-Python-orange.svg)](https://python.org/) [![License WarpX](https://img.shields.io/badge/license-BSD--3--Clause--LBNL-blue.svg)](https://spdx.org/licenses/BSD-3-Clause-LBNL.html) @@ -45,6 +45,14 @@ To contact the developers, feel free to open an issue on this repo, or visit our [![SENSEI](https://img.shields.io/static/v1?label="runs%20on"&message="SENSEI"&color="blueviolet")](https://sensei-insitu.org) Our workflow is described in [CONTRIBUTING.rst](CONTRIBUTING.rst). +We invite you to contribute to WarpX in any form following our [Code of Conduct](https://warpx.readthedocs.io/en/latest/coc.html), e.g., contribute to [discussions](https://github.com/ECP-WarpX/WarpX/discussions), help each other in [issues](https://github.com/ECP-WarpX/WarpX/issues), fix bugs, or add [documentation](https://warpx.readthedocs.io/en/latest/developers/documentation.html) and new functionality! + +## Governance + +WarpX is hosted by the High Performance Computing Foundation (HPSF). +If your organization wants to help steer the evolution of the HPC software ecosystem, visit [hpsf.io](https://hpsf.io) and consider joining! + +The WarpX open governance model is described in [GOVERNANCE.rst](GOVERNANCE.rst). ## Copyright Notice From 9cab6f98e67b2e286cfa2388bbb9bd23f19061af Mon Sep 17 00:00:00 2001 From: Jonathan Wang Date: Thu, 1 Aug 2024 14:29:12 -0400 Subject: [PATCH 183/190] Doc: Pitzer (OSC) (#5064) * add: profile, dependencies, and sbatch scripts for OSC pitzer * add: pitzer documentation page * fix: add pitzer name to manual sidebar * chore: update pitzer dependencies installation script filename * fix formatting issues in doc * separate cpu and gpu scripts * add boost support * add separate instructions for CPU and GPU dependencies * rename venv name * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add header to dependencies intalling scripts * chores for doc * fix: dependencies installing scripts typo * fix typos * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * chore: Update batch script and profile file names for Pitzer CPU and V100 * remove gpu allocation in CPU profile * Update Docs/source/install/hpc/pitzer.rst Co-authored-by: Axel Huebl * Update Docs/source/install/hpc.rst Co-authored-by: Axel Huebl * Update Docs/source/install/hpc/pitzer.rst Co-authored-by: Axel Huebl * use built-in cuda-aware MPI * Remove repetition of source profile --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Axel Huebl --- .gitignore | 2 +- Docs/source/install/hpc.rst | 3 +- Docs/source/install/hpc/pitzer.rst | 274 ++++++++++++++++++ .../pitzer-osc/install_cpu_dependencies.sh | 156 ++++++++++ .../pitzer-osc/install_v100_dependencies.sh | 156 ++++++++++ Tools/machines/pitzer-osc/pitzer_cpu.sbatch | 19 ++ .../pitzer_cpu_warpx.profile.example | 54 ++++ Tools/machines/pitzer-osc/pitzer_v100.sbatch | 26 ++ .../pitzer_v100_warpx.profile.example | 64 ++++ 9 files changed, 752 insertions(+), 2 deletions(-) create mode 100644 Docs/source/install/hpc/pitzer.rst create mode 100644 Tools/machines/pitzer-osc/install_cpu_dependencies.sh create mode 100644 Tools/machines/pitzer-osc/install_v100_dependencies.sh create mode 100644 Tools/machines/pitzer-osc/pitzer_cpu.sbatch create mode 100644 Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example create mode 100644 Tools/machines/pitzer-osc/pitzer_v100.sbatch create mode 100644 Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example diff --git a/.gitignore b/.gitignore index 4fa65351331..094fee87efc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ Python/pywarpx/libwarpx*.so d/ f/ o/ -build/ +build*/ tmp_build_dir/ test_dir test_dir/ diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst index 5efdeae39a8..5b388b9d0b2 100644 --- a/Docs/source/install/hpc.rst +++ b/Docs/source/install/hpc.rst @@ -36,6 +36,7 @@ This section documents quick-start guides for a selection of supercomputers that hpc/crusher hpc/frontier hpc/fugaku + hpc/greatlakes hpc/hpc3 hpc/juwels hpc/karolina @@ -46,12 +47,12 @@ This section documents quick-start guides for a selection of supercomputers that hpc/lxplus hpc/ookami hpc/perlmutter + hpc/pitzer hpc/polaris hpc/quartz hpc/spock hpc/summit hpc/taurus - hpc/greatlakes .. tip:: diff --git a/Docs/source/install/hpc/pitzer.rst b/Docs/source/install/hpc/pitzer.rst new file mode 100644 index 00000000000..4f1e53bc309 --- /dev/null +++ b/Docs/source/install/hpc/pitzer.rst @@ -0,0 +1,274 @@ +.. _building-pitzer: + +Pitzer (OSC) +============ + +The `Pitzer cluster `__ is located at the Ohio Supercomputer Center (OSC). It is currently the main CPU/GPU cluster at OSC. However, the `Cardinal cluster `__ is soon going to take over Pitzer to become the next major CPU/GPU cluster at OSC in the second half of 2024. A list of all OSC clusters can be found `here `__. + +The Pitzer cluster offers a variety of partitions suitable for different computational needs, including GPU nodes, CPU nodes, and nodes with large memory capacities. For more information on the specifications and capabilities of these partitions, visit the `Ohio Supercomputer Center's Pitzer page `__. + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `Pitzer user guide `__ +* Batch system: `Slurm `__ +* `Jupyter service `__ +* `Filesystems `__: + + * ``$HOME``: per-user directory, use only for inputs, source, and scripts; backed up (500GB) + * ``/fs/ess``: per-project storage directory, use for long-term storage of data and analysis; backed up (1-5TB) + * ``/fs/scratch``: per-project production directory; fast I/O for parallel jobs; not backed up (100TB) + +.. _building-pitzer-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + +On Pitzer, you can run either on GPU nodes with V100 GPUs or CPU nodes. + +.. tab-set:: + + .. tab-item:: V100 GPUs + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/pitzer_v100_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example $HOME/pitzer_v100_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``pas2024``, then run ``nano $HOME/pitzer_v100_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="pas2024" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to pitzer, activate these environment settings: + + .. code-block:: bash + + source $HOME/pitzer_v100_warpx.profile + + Finally, since pitzer does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/pitzer-osc/install_v100_dependencies.sh + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/install_v100_dependencies.sh + :language: bash + + + .. tab-item:: CPU Nodes + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/pitzer_cpu_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example $HOME/pitzer_cpu_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``pas2024``, then run ``nano $HOME/pitzer_cpu_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="pas2024" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to pitzer, activate these environment settings: + + .. code-block:: bash + + source $HOME/pitzer_cpu_warpx.profile + + Finally, since pitzer does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/pitzer-osc/install_cpu_dependencies.sh + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/install_cpu_dependencies.sh + :language: bash + + +.. _building-pitzer-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. tab-set:: + .. tab-item:: V100 GPUs + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100 + + cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100 -j 48 + + The WarpX application executables are now in ``$HOME/src/warpx/build_v100/bin/``. Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100_py + + cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100_py -j 48 --target pip_install + + .. tab-item:: CPU Nodes + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build + + cmake -S . -B build -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build -j 48 + + The WarpX application executables are now in ``$HOME/src/warpx/build/bin/``. Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_py + + cmake -S . -B build_py -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_py -j 48 --target pip_install + +Now, you can :ref:`submit Pitzer compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Pitzer jobs (:ref:`example inputs `). For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch``. + +.. _building-pitzer-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd $HOME/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the pitzer_cpu_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. + +.. _running-pitzer: + +Running +------- + +.. tab-set:: + + .. tab-item:: V100 GPUs + + Pitzer's GPU partition includes: + + - 32 nodes, each equipped with two V100 (16GB) GPUs. + - 42 nodes, each with two V100 (32GB) GPUs. + - 4 large memory nodes, each with quad V100 (32GB) GPUs. + + To run a WarpX simulation on the GPU nodes, use the batch script provided below. Adjust the ``-N`` parameter in the script to match the number of nodes you intend to use. Each node in this partition supports running one MPI rank per GPU. + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_v100.sbatch + :language: bash + :caption: Copy this file from ``$HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_v100.sbatch``. + + After preparing your script, submit your job with the following command: + + .. code-block:: bash + + sbatch pitzer_v100.sbatch + + .. tab-item:: CPU Nodes + + For CPU-based computations, Pitzer offers: + + - 224 nodes, each with dual Intel Xeon Gold 6148 CPUs and 192 GB RAM. + - 340 nodes, each with dual Intel Xeon Platinum 8268 CPUs and 192 GB RAM. + - 16 large memory nodes. + + To submit a job to the CPU partition, use the provided batch script. Ensure you have copied the script to your working directory. + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_cpu.sbatch + :language: bash + :caption: Copy this file from ``$HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_cpu.sbatch``. + + Submit your job with: + + .. code-block:: bash + + sbatch pitzer_cpu.sbatch + +.. _post-processing-osc: + +Post-Processing +--------------- + +For post-processing, many users prefer to use the online `Jupyter service `__ (`documentation `__) that is directly connected to the cluster's fast filesystem. + +.. note:: + + This section is a stub and contributions are welcome. + We can document further details, e.g., which recommended post-processing Python software to install or how to customize Jupyter kernels here. diff --git a/Tools/machines/pitzer-osc/install_cpu_dependencies.sh b/Tools/machines/pitzer-osc/install_cpu_dependencies.sh new file mode 100644 index 00000000000..f0b6ce4b950 --- /dev/null +++ b/Tools/machines/pitzer-osc/install_cpu_dependencies.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Zhongwei Wang +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + +# Check: ###################################################################### +# +# Was pitzer_cpu_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your pitzer_cpu_warpx.profile file! Please edit its line 2 to continue!" + exit 1 +fi + +# Remove old dependencies ##################################################### +# +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + +# General extra dependencies ################################################## +# +SRC_DIR="${HOME}/src" +build_dir=$(mktemp -d) + +# boost (for QED table generation support) +cd ${SRC_DIR} +wget https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz +tar -xzvf boost_1_82_0.tar.gz +rm -rf boost_1_82_0.tar.gz +cd - + +cd ${SRC_DIR}/boost_1_82_0 +./bootstrap.sh --prefix=$SW_DIR/boost-1.82.0 +./b2 install +cd - + +# BLAS++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/blaspp ]; then + cd ${SRC_DIR}/blaspp + git fetch --prune + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp +fi +rm -rf ${build_dir}/blaspp-pitzer-cpu-build +CXX=$(which CC) cmake -S ${SRC_DIR}/blaspp \ + -B ${build_dir}/blaspp-pitzer-cpu-build \ + -Duse_openmp=ON \ + -Dgpu_backend=OFF \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 +cmake --build ${build_dir}/blaspp-pitzer-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pitzer-cpu-build + +# LAPACK++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/lapackpp ]; then + cd ${SRC_DIR}/lapackpp + git fetch --prune + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp +fi +rm -rf ${build_dir}/lapackpp-pitzer-cpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp \ + -B ${build_dir}/lapackpp-pitzer-cpu-build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 +cmake --build ${build_dir}/lapackpp-pitzer-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pitzer-cpu-build + +# c-blosc (I/O compression, for openPMD) +if [ -d ${SRC_DIR}/c-blosc ]; then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.6 + cd - +else + git clone -b v1.21.6 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi +rm -rf ${build_dir}/c-blosc-pitzer-build +cmake -S ${SRC_DIR}/c-blosc \ + -B ${build_dir}/c-blosc-pitzer-build \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DDEACTIVATE_AVX2=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.6 +cmake --build ${build_dir}/c-blosc-pitzer-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pitzer-build + +# ADIOS2 (for openPMD) +if [ -d ${SRC_DIR}/adios2 ]; then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.10.1 + cd - +else + git clone -b v2.10.1 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +rm -rf ${build_dir}/adios2-pitzer-build +cmake -S ${SRC_DIR}/adios2 \ + -B ${build_dir}/adios2-pitzer-build \ + -DBUILD_TESTING=OFF \ + -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_USE_Blosc=ON \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_SST=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.1 +cmake --build ${build_dir}/adios2-pitzer-build --target install -j 16 +rm -rf ${build_dir}/adios2-pitzer-build + +rm -rf ${build_dir} + +# Python ###################################################################### +# +python3 -m pip install --upgrade --user virtualenv +rm -rf ${SW_DIR}/venvs/${VENV_NAME} +python3 -m venv ${SW_DIR}/venvs/${VENV_NAME} +source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip cache purge +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt + +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt + +# ML dependencies +python3 -m pip install --upgrade torch diff --git a/Tools/machines/pitzer-osc/install_v100_dependencies.sh b/Tools/machines/pitzer-osc/install_v100_dependencies.sh new file mode 100644 index 00000000000..5601b4d76c9 --- /dev/null +++ b/Tools/machines/pitzer-osc/install_v100_dependencies.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Zhongwei Wang +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + +# Check: ###################################################################### +# +# Was pitzer_v100_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your pitzer_v100_warpx.profile file! Please edit its line 2 to continue!" + exit 1 +fi + +# Remove old dependencies ##################################################### +# +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + +# General extra dependencies ################################################## +# +SRC_DIR="${HOME}/src" +build_dir=$(mktemp -d) + +# boost (for QED table generation support) +cd ${SRC_DIR} +wget https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz +tar -xzvf boost_1_82_0.tar.gz +rm -rf boost_1_82_0.tar.gz +cd - + +cd ${SRC_DIR}/boost_1_82_0 +./bootstrap.sh --prefix=${SW_DIR}/boost-1.82.0 +./b2 install +cd - + +# BLAS++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/blaspp ]; then + cd ${SRC_DIR}/blaspp + git fetch + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp +fi +rm -rf ${build_dir}/blaspp-pitzer-v100-build +CXX=$(which CC) cmake -S ${SRC_DIR}/blaspp \ + -B ${build_dir}/blaspp-pitzer-v100-build \ + -Duse_openmp=ON \ + -Dgpu_backend=cuda \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 +cmake --build ${build_dir}/blaspp-pitzer-v100-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pitzer-v100-build + +# LAPACK++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/lapackpp ]; then + cd ${SRC_DIR}/lapackpp + git fetch + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp +fi +rm -rf ${build_dir}/lapackpp-pitzer-v100-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp \ + -B ${build_dir}/lapackpp-pitzer-v100-build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 +cmake --build ${build_dir}/lapackpp-pitzer-v100-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pitzer-v100-build + +# c-blosc (I/O compression, for openPMD) +if [ -d ${SRC_DIR}/c-blosc ]; then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.6 + cd - +else + git clone -b v1.21.6 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi +rm -rf ${build_dir}/c-blosc-pitzer-build +cmake -S ${SRC_DIR}/c-blosc \ + -B ${build_dir}/c-blosc-pitzer-build \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DDEACTIVATE_AVX2=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.6 +cmake --build ${build_dir}/c-blosc-pitzer-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pitzer-build + +# ADIOS2 (for openPMD) +if [ -d ${SRC_DIR}/adios2 ]; then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.10.1 + cd - +else + git clone -b v2.10.1 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +rm -rf ${build_dir}/adios2-pitzer-build +cmake -S ${SRC_DIR}/adios2 \ + -B ${build_dir}/adios2-pitzer-build \ + -DBUILD_TESTING=OFF \ + -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_USE_Blosc=ON \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_SST=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.1 +cmake --build ${build_dir}/adios2-pitzer-build --target install -j 16 +rm -rf ${build_dir}/adios2-pitzer-build + +rm -rf ${build_dir} + +# Python ###################################################################### +# +python3 -m pip install --upgrade --user virtualenv +rm -rf ${SW_DIR}/venvs/${VENV_NAME} +python3 -m venv ${SW_DIR}/venvs/${VENV_NAME} +source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip cache purge +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt + +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt + +# ML dependencies +python3 -m pip install --upgrade torch diff --git a/Tools/machines/pitzer-osc/pitzer_cpu.sbatch b/Tools/machines/pitzer-osc/pitzer_cpu.sbatch new file mode 100644 index 00000000000..8097b3842f9 --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_cpu.sbatch @@ -0,0 +1,19 @@ +#!/bin/bash +#SBATCH --time=0:20:00 +#SBATCH --nodes=1 --ntasks-per-node=6 +#SBATCH --cpus-per-task=8 +#SBATCH --job-name= +#SBATCH --account= +#SBATCH --output=./logs/%x_%j.out +#SBATCH --error=./logs/%x_%j.err + +# Pitzer cluster has 224 CPU nodes equipped with dual Intel Xeon 6148 (40 cores per node) and 340 CPU nodes with dual Intel Xeon 8268 (48 cores per node). https://www.osc.edu/resources/technical_support/supercomputers/pitzer + +source ${HOME}/pitzer_cpu_warpx.profile +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + +# executable & inputs file or python interpreter & PICMI script here +EXE=${HOME}/src/warpx/build/bin/warpx.2d +INPUTS=inputs + +srun --cpu-bind=cores ${EXE} ${INPUTS} >./logs/${SLURM_JOB_NAME}_${SLURM_JOBID}.log 2>&1 diff --git a/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example b/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example new file mode 100644 index 00000000000..d99c16b6cb6 --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example @@ -0,0 +1,54 @@ +# please set your project account +export proj="" # change me! + +# remembers the location of this script +export MY_CPU_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_CPU_PROFILE file! Please edit its line 2 to continue!" + return +fi + +export SW_DIR="${HOME}/sw/osc/pitzer/cpu" + +module purge +module load cmake/3.25.2 +module load gnu/12.3.0 +module load openmpi/4.1.5-hpcx + +# optional: for python binding support +module load miniconda3/24.1.2-py310 +export VENV_NAME="warpx-pitzer-cpu" +if [ -d "${SW_DIR}/venvs/${VENV_NAME}" ]; then + source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 --ntasks-per-node=2 --cpus-per-task=20 -t 1:00:00 -A $proj" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --ntasks-per-node=2 --cpus-per-task=20 -t 1:00:00 -A $proj" + +# optional: for PSATD in RZ geometry support +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH + +# optional: for QED lookup table generation support +# use self-installed boost +export CMAKE_PREFIX_PATH=${SW_DIR}/boost-1.82.0:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/boost-1.82.0/lib:$LD_LIBRARY_PATH + +# optional: for openPMD support (hdf5 and adios2) +module load hdf5/1.12.2 +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.6:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.1:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.6/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.1/lib64:$LD_LIBRARY_PATH +export PATH=${SW_DIR}/adios2-2.10.1/bin:${PATH} + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) diff --git a/Tools/machines/pitzer-osc/pitzer_v100.sbatch b/Tools/machines/pitzer-osc/pitzer_v100.sbatch new file mode 100644 index 00000000000..d34868da16c --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_v100.sbatch @@ -0,0 +1,26 @@ +#!/bin/bash +#SBATCH --time=0:20:00 +#SBATCH --nodes=1 --ntasks-per-node=2 +#SBATCH --cpus-per-task=24 +#SBATCH --gpus-per-task=1 +#SBATCH --gpu-bind=closest +#SBATCH --job-name= +#SBATCH --account= +#SBATCH --output=./logs/%x_%j.out +#SBATCH --error=./logs/%x_%j.err + +# Pitzer cluster has 32 GPU nodes with dual Intel Xeon 6148 and dual V100 (16GB) GPUs and 42 nodes with dual Intel Xeon 8268 and dual V100 (32GB) GPUs. https://www.osc.edu/resources/technical_support/supercomputers/pitzer + +source ${HOME}/pitzer_v100_warpx.profile +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + +echo "GPU Information:" +nvidia-smi + +GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" + +# executable & inputs file or python interpreter & PICMI script here +EXE=${HOME}/src/warpx/build_v100/bin/warpx.2d +INPUTS=inputs + +srun --cpu-bind=cores ${EXE} ${INPUTS} ${GPU_AWARE_MPI} >./logs/${SLURM_JOB_NAME}_${SLURM_JOBID}.log 2>&1 diff --git a/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example b/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example new file mode 100644 index 00000000000..061794f5f68 --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example @@ -0,0 +1,64 @@ +# please set your project account +export proj="" # change me! + +# remembers the location of this script +export MY_V100_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_V100_PROFILE file! Please edit its line 2 to continue!" + return +fi + +export SW_DIR="${HOME}/sw/osc/pitzer/v100" + +module purge +module load cmake/3.25.2 +module load intel/19.0.5 +module load cuda/11.8.0 +module load openmpi-cuda/4.1.5-hpcx +module load gcc-compatibility/11.2.0 + +# optional: for python binding support +module load miniconda3/24.1.2-py310 +export VENV_NAME="warpx-pitzer-v100" +if [ -d "${SW_DIR}/venvs/${VENV_NAME}" ]; then + source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" + +# optional: for PSATD in RZ geometry support +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH + +# optional: for QED lookup table generation support +# use self-installed boost +export CMAKE_PREFIX_PATH=${SW_DIR}/boost-1.82.0:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/boost-1.82.0/lib:$LD_LIBRARY_PATH + +# optional: for openPMD support (hdf5 and adios2) +# use self-installed hdf5 +module load hdf5/1.12.0 + +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.6:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.1:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.6/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.1/lib64:$LD_LIBRARY_PATH +export PATH=${SW_DIR}/adios2-2.10.1/bin:${PATH} + +# avoid relocation truncation error which result from large executable size +export CUDAFLAGS="--host-linker-script=use-lcs" # https://github.com/ECP-WarpX/WarpX/pull/3673 +export AMREX_CUDA_ARCH=7.0 # 7.0: V100, 8.0: V100, 9.0: H100 https://github.com/ECP-WarpX/WarpX/issues/3214 + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} From 67796e4be25484a1aab5839896595cee7230ea14 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Thu, 1 Aug 2024 14:52:29 -0500 Subject: [PATCH 184/190] ABLASTR: Pass `GridType` instead of Global (#5106) We cannot use global variables from WarpX in the ABLASTR library, otherwise it will not build on its own or in ImpactX. --- Source/BoundaryConditions/PML.H | 3 +- Source/BoundaryConditions/PML.cpp | 6 +++- Source/FieldSolver/ElectrostaticSolver.cpp | 1 + .../FiniteDifferenceSolver/ComputeDivE.cpp | 2 +- .../FiniteDifferenceSolver/EvolveBPML.cpp | 2 +- .../FiniteDifferenceSolver.H | 6 ++-- .../FiniteDifferenceSolver.cpp | 4 +-- .../PsatdAlgorithmComoving.H | 4 ++- .../PsatdAlgorithmComoving.cpp | 12 ++++--- .../PsatdAlgorithmFirstOrder.H | 4 ++- .../PsatdAlgorithmFirstOrder.cpp | 7 ++-- .../PsatdAlgorithmGalileanRZ.H | 5 ++- .../PsatdAlgorithmGalileanRZ.cpp | 2 +- .../PsatdAlgorithmJConstantInTime.H | 4 ++- .../PsatdAlgorithmJConstantInTime.cpp | 2 +- .../PsatdAlgorithmJLinearInTime.H | 7 ++-- .../PsatdAlgorithmJLinearInTime.cpp | 2 +- .../SpectralAlgorithms/PsatdAlgorithmPml.H | 5 +-- .../SpectralAlgorithms/PsatdAlgorithmPml.cpp | 2 +- .../SpectralAlgorithms/PsatdAlgorithmPmlRZ.H | 9 +++-- .../PsatdAlgorithmPmlRZ.cpp | 6 ++-- .../SpectralAlgorithms/PsatdAlgorithmRZ.H | 6 +++- .../SpectralAlgorithms/PsatdAlgorithmRZ.cpp | 6 ++-- .../SpectralBaseAlgorithm.H | 8 +++-- .../SpectralBaseAlgorithm.cpp | 10 ++++-- .../SpectralBaseAlgorithmRZ.H | 6 +++- .../SpectralSolver/SpectralKSpace.H | 10 ++++-- .../SpectralSolver/SpectralKSpace.cpp | 4 +-- .../SpectralSolver/SpectralSolver.H | 8 +++-- .../SpectralSolver/SpectralSolver.cpp | 10 ++++-- .../SpectralSolver/SpectralSolverRZ.H | 6 +++- .../SpectralSolver/SpectralSolverRZ.cpp | 3 +- Source/Parallelization/GuardCellManager.H | 4 ++- Source/Parallelization/GuardCellManager.cpp | 3 +- Source/Utils/WarpXAlgorithmSelection.H | 18 +++------- Source/Utils/WarpXAlgorithmSelection.cpp | 15 ++++---- Source/WarpX.H | 8 +++-- Source/WarpX.cpp | 8 ++--- Source/ablastr/fields/PoissonSolver.H | 7 ++-- Source/ablastr/utils/Enums.H | 35 +++++++++++++++++++ 40 files changed, 186 insertions(+), 84 deletions(-) create mode 100644 Source/ablastr/utils/Enums.H diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 7ed1def8f72..8f3ca1da1d7 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -140,7 +140,8 @@ public: const amrex::DistributionMapping& dm, bool do_similar_dm_pml, const amrex::Geometry* geom, const amrex::Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, - amrex::Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, + amrex::Real dt, int nox_fft, int noy_fft, int noz_fft, + ablastr::utils::enums::GridType grid_type, int do_moving_window, int pml_has_particles, int do_pml_in_domain, int psatd_solution_type, int J_in_time, int rho_in_time, bool do_pml_dive_cleaning, bool do_pml_divb_cleaning, diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index ed82cc07bb9..340005e9211 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -22,6 +22,7 @@ #include "WarpX.H" #include +#include #include #include @@ -548,7 +549,8 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& grid_dm, const bool do_similar_dm_pml, const Geometry* geom, const Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, - Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, + Real dt, int nox_fft, int noy_fft, int noz_fft, + ablastr::utils::enums::GridType grid_type, int do_moving_window, int /*pml_has_particles*/, int do_pml_in_domain, const int psatd_solution_type, const int J_in_time, const int rho_in_time, const bool do_pml_dive_cleaning, const bool do_pml_divb_cleaning, @@ -631,6 +633,8 @@ PML::PML (const int lev, const BoxArray& grid_ba, } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + using namespace ablastr::utils::enums; + // Increase the number of guard cells, in order to fit the extent // of the stencil for the spectral solver int ngFFt_x = (grid_type == GridType::Collocated) ? nox_fft : nox_fft/2; diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index 3860bd55325..189f2f2bb0a 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -396,6 +396,7 @@ WarpX::computePhi (const amrex::Vector >& rho, this->geom, this->dmap, this->grids, + WarpX::grid_type, this->m_poisson_boundary_handler, is_solver_igf_on_lev0, WarpX::do_single_precision_comms, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp index 3cc05c58068..0702b264874 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp @@ -52,7 +52,7 @@ void FiniteDifferenceSolver::ComputeDivE ( ComputeDivECylindrical ( Efield, divEfield ); #else - if (m_grid_type == GridType::Collocated) { + if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { ComputeDivECartesian ( Efield, divEfield ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 1fb2dd85e60..0ad2c8d6802 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -53,7 +53,7 @@ void FiniteDifferenceSolver::EvolveBPML ( WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else - if (m_grid_type == GridType::Collocated) { + if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { EvolveBPMLCartesian (Bfield, Efield, dt, dive_cleaning); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index a7a4f10a713..00e87525a75 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -17,6 +17,8 @@ #include "HybridPICModel/HybridPICModel_fwd.H" #include "MacroscopicProperties/MacroscopicProperties_fwd.H" +#include + #include #include @@ -47,7 +49,7 @@ class FiniteDifferenceSolver FiniteDifferenceSolver ( int fdtd_algo, std::array cell_size, - short grid_type ); + ablastr::utils::enums::GridType grid_type ); void EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, @@ -178,7 +180,7 @@ class FiniteDifferenceSolver private: int m_fdtd_algo; - short m_grid_type; + ablastr::utils::enums::GridType m_grid_type; #ifdef WARPX_DIM_RZ amrex::Real m_dr, m_rmin; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp index 1bbc6c9b337..9af610031c0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp @@ -30,7 +30,7 @@ FiniteDifferenceSolver::FiniteDifferenceSolver ( int const fdtd_algo, std::array cell_size, - short grid_type): + ablastr::utils::enums::GridType grid_type): // Register the type of finite-difference algorithm m_fdtd_algo{fdtd_algo}, m_grid_type{grid_type} @@ -63,7 +63,7 @@ FiniteDifferenceSolver::FiniteDifferenceSolver ( "FiniteDifferenceSolver: Unknown algorithm"); } #else - if (grid_type == GridType::Collocated) { + if (grid_type == ablastr::utils::enums::GridType::Collocated) { CartesianNodalAlgorithm::InitializeStencilCoefficients( cell_size, m_h_stencil_coefs_x, m_h_stencil_coefs_y, m_h_stencil_coefs_z ); diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H index 70468277749..b7ee256f414 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H @@ -5,6 +5,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -32,7 +34,7 @@ class PsatdAlgorithmComoving : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_comoving, amrex::Real dt, bool update_with_rho); diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp index 6da6641c052..f809c3f2c20 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp @@ -5,6 +5,8 @@ #include "Utils/WarpXConst.H" #include "Utils/WarpX_Complex.H" +#include + #include #include #include @@ -26,8 +28,10 @@ using namespace amrex; PsatdAlgorithmComoving::PsatdAlgorithmComoving (const SpectralKSpace& spectral_kspace, const DistributionMapping& dm, const SpectralFieldIndex& spectral_index, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type, + const int norder_x, + const int norder_y, + const int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_comoving, const amrex::Real dt, const bool update_with_rho) @@ -35,12 +39,12 @@ PsatdAlgorithmComoving::PsatdAlgorithmComoving (const SpectralKSpace& spectral_k : SpectralBaseAlgorithm{spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type}, // Initialize the infinite-order k vectors (the argument n_order = -1 selects // the infinite order option, the argument grid_type=GridType::Staggered is then irrelevant) - kx_vec{spectral_kspace.getModifiedKComponent(dm, 0, -1, GridType::Staggered)}, + kx_vec{spectral_kspace.getModifiedKComponent(dm, 0, -1, ablastr::utils::enums::GridType::Staggered)}, #if defined(WARPX_DIM_3D) ky_vec{spectral_kspace.getModifiedKComponent(dm, 1, -1, GridType::Staggered)}, kz_vec{spectral_kspace.getModifiedKComponent(dm, 2, -1, GridType::Staggered)}, #else - kz_vec{spectral_kspace.getModifiedKComponent(dm, 1, -1, GridType::Staggered)}, + kz_vec{spectral_kspace.getModifiedKComponent(dm, 1, -1, ablastr::utils::enums::GridType::Staggered)}, #endif m_v_comoving{v_comoving}, m_dt{dt} diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H index 23d4c2f82e3..23c38f33a85 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H @@ -11,6 +11,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -50,7 +52,7 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, amrex::Real dt, bool div_cleaning, int J_in_time, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp index 041b70b8edf..14db3c9af48 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp @@ -28,18 +28,19 @@ using namespace amrex::literals; -PsatdAlgorithmFirstOrder::PsatdAlgorithmFirstOrder( +PsatdAlgorithmFirstOrder::PsatdAlgorithmFirstOrder ( const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, const SpectralFieldIndex& spectral_index, const int norder_x, const int norder_y, const int norder_z, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Real dt, const bool div_cleaning, const int J_in_time, - const int rho_in_time) + const int rho_in_time +) // Initializer list : SpectralBaseAlgorithm(spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type), m_dt(dt), diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H index ad0aca1ecf5..0861eba3abf 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H @@ -9,6 +9,9 @@ #include "SpectralBaseAlgorithmRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -20,7 +23,7 @@ class PsatdAlgorithmGalileanRZ : public SpectralBaseAlgorithmRZ amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, int n_rz_azimuthal_modes, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real dt_step, bool update_with_rho); diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp index c24648fbf07..63127969149 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp @@ -20,7 +20,7 @@ PsatdAlgorithmGalileanRZ::PsatdAlgorithmGalileanRZ (SpectralKSpaceRZ const & spe amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, int const n_rz_azimuthal_modes, int const norder_z, - short const grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real const dt, bool const update_with_rho): diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H index 5d83b5519a6..12e3203db39 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H @@ -11,6 +11,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -52,7 +54,7 @@ class PsatdAlgorithmJConstantInTime : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real dt, bool update_with_rho, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp index 59d56b99afc..991bf981924 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp @@ -36,7 +36,7 @@ PsatdAlgorithmJConstantInTime::PsatdAlgorithmJConstantInTime( const int norder_x, const int norder_y, const int norder_z, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, const amrex::Real dt, const bool update_with_rho, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H index 970657978bb..c26fc4a81a4 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H @@ -11,6 +11,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -53,11 +55,12 @@ class PsatdAlgorithmJLinearInTime : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, amrex::Real dt, bool time_averaging, bool dive_cleaning, - bool divb_cleaning); + bool divb_cleaning + ); /** * \brief Updates the E and B fields in spectral space, according to the multi-J PSATD equations diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp index a6ee55147c9..2626dd0376e 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp @@ -35,7 +35,7 @@ PsatdAlgorithmJLinearInTime::PsatdAlgorithmJLinearInTime( const int norder_x, const int norder_y, const int norder_z, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Real dt, const bool time_averaging, const bool dive_cleaning, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H index 1c19827a0a7..f01b8cad379 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H @@ -12,8 +12,9 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData_fwd.H" #include "FieldSolver/SpectralSolver/SpectralKSpace_fwd.H" -#include +#include +#include #include #include @@ -51,7 +52,7 @@ class PsatdAlgorithmPml : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real dt, bool dive_cleaning, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp index ee776a71a20..5907485a526 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp @@ -37,7 +37,7 @@ PsatdAlgorithmPml::PsatdAlgorithmPml( int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, Real dt, bool dive_cleaning, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H index 00264956900..9f2a21db595 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H @@ -9,6 +9,9 @@ #include "SpectralBaseAlgorithmRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -19,8 +22,10 @@ class PsatdAlgorithmPmlRZ : public SpectralBaseAlgorithmRZ PsatdAlgorithmPmlRZ (SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int n_rz_azimuthal_modes, int norder_z, - short grid_type, amrex::Real dt_step); + int n_rz_azimuthal_modes, + int norder_z, + ablastr::utils::enums::GridType grid_type, + amrex::Real dt_step); // Redefine functions from base class void pushSpectralFields (SpectralFieldDataRZ & f) final; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp index 66aa463503f..50522f3c9b2 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp @@ -19,8 +19,10 @@ using namespace amrex::literals; PsatdAlgorithmPmlRZ::PsatdAlgorithmPmlRZ (SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int const n_rz_azimuthal_modes, int const norder_z, - short const grid_type, amrex::Real const dt): + int const n_rz_azimuthal_modes, + int const norder_z, + ablastr::utils::enums::GridType grid_type, + amrex::Real const dt): // Initialize members of base class and member variables SpectralBaseAlgorithmRZ{spectral_kspace, dm, spectral_index, norder_z, grid_type}, m_dt{dt} diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H index 10feafec38b..c2efbd79935 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H @@ -9,6 +9,9 @@ #include "SpectralBaseAlgorithmRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -20,7 +23,8 @@ class PsatdAlgorithmRZ : public SpectralBaseAlgorithmRZ amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, int n_rz_azimuthal_modes, int norder_z, - short grid_type, amrex::Real dt_step, + ablastr::utils::enums::GridType grid_type, + amrex::Real dt_step, bool update_with_rho, bool time_averaging, int J_in_time, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp index f621d3c7af9..efff5c30a41 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp @@ -19,8 +19,10 @@ using namespace amrex::literals; PsatdAlgorithmRZ::PsatdAlgorithmRZ (SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int const n_rz_azimuthal_modes, int const norder_z, - short const grid_type, amrex::Real const dt, + int const n_rz_azimuthal_modes, + int const norder_z, + ablastr::utils::enums::GridType grid_type, + amrex::Real const dt, bool const update_with_rho, const bool time_averaging, const int J_in_time, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H index e6787c9e52d..c72e7db250d 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H @@ -13,6 +13,8 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData_fwd.H" #include "FieldSolver/SpectralSolver/SpectralFieldData.H" +#include + #include #include #include @@ -88,8 +90,10 @@ class SpectralBaseAlgorithm SpectralBaseAlgorithm(const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, const SpectralFieldIndex& spectral_index, - int norder_x, int norder_y, - int norder_z, short grid_type); + int norder_x, + int norder_y, + int norder_z, + ablastr::utils::enums::GridType grid_type); SpectralFieldIndex m_spectral_index; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp index d635a5debe3..b3f18dd6912 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp @@ -27,11 +27,15 @@ using namespace amrex; /** * \brief Constructor */ -SpectralBaseAlgorithm::SpectralBaseAlgorithm(const SpectralKSpace& spectral_kspace, +SpectralBaseAlgorithm::SpectralBaseAlgorithm ( + const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, const SpectralFieldIndex& spectral_index, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type): + const int norder_x, + const int norder_y, + const int norder_z, + ablastr::utils::enums::GridType grid_type +): m_spectral_index(spectral_index), // Compute and assign the modified k vectors modified_kx_vec(spectral_kspace.getModifiedKComponent(dm,0,norder_x,grid_type)), diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H index 95c68f6d61b..8e03a2a2559 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H @@ -10,6 +10,9 @@ #include "FieldSolver/SpectralSolver/SpectralKSpaceRZ.H" #include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. * @@ -84,7 +87,8 @@ class SpectralBaseAlgorithmRZ SpectralBaseAlgorithmRZ(SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int const norder_z, short const grid_type) + int const norder_z, + ablastr::utils::enums::GridType grid_type) // Compute and assign the modified k vectors : m_spectral_index(spectral_index), modified_kz_vec(spectral_kspace.getModifiedKComponent(dm, 1, norder_z, grid_type)) diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.H b/Source/FieldSolver/SpectralSolver/SpectralKSpace.H index 2a4dd3d1580..16f93d8292a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.H +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.H @@ -12,6 +12,8 @@ #include "Utils/WarpX_Complex.H" +#include + #include #include #include @@ -59,9 +61,11 @@ class SpectralKSpace const amrex::BoxArray& realspace_ba, int i_dim, bool only_positive_k ) const; - KVectorComponent getModifiedKComponent( - const amrex::DistributionMapping& dm, int i_dim, - int n_order, short grid_type ) const; + KVectorComponent getModifiedKComponent ( + const amrex::DistributionMapping& dm, + int i_dim, + int n_order, + ablastr::utils::enums::GridType grid_type) const; SpectralShiftFactor getSpectralShiftFactor( const amrex::DistributionMapping& dm, int i_dim, diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp index 91fe2953668..94bd384f265 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp @@ -188,10 +188,10 @@ SpectralKSpace::getSpectralShiftFactor( const DistributionMapping& dm, * \param grid_type type of grid (collocated or not) */ KVectorComponent -SpectralKSpace::getModifiedKComponent( const DistributionMapping& dm, +SpectralKSpace::getModifiedKComponent (const DistributionMapping& dm, const int i_dim, const int n_order, - const short grid_type ) const + ablastr::utils::enums::GridType grid_type) const { // Initialize an empty DeviceVector in each box KVectorComponent modified_k_comp(spectralspace_ba, dm); diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.H b/Source/FieldSolver/SpectralSolver/SpectralSolver.H index 3d751974a2b..26581900c02 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.H @@ -12,6 +12,8 @@ #include "SpectralAlgorithms/SpectralBaseAlgorithm.H" #include "SpectralFieldData.H" +#include + #include #include #include @@ -72,8 +74,10 @@ class SpectralSolver SpectralSolver (int lev, const amrex::BoxArray& realspace_ba, const amrex::DistributionMapping& dm, - int norder_x, int norder_y, - int norder_z, short grid_type, + int norder_x, + int norder_y, + int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, const amrex::Vector& v_comoving, amrex::RealVect dx, diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index 9fc8f8056ca..c6192187064 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -17,16 +17,20 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXProfilerWrapper.H" +#include + #include #if WARPX_USE_FFT -SpectralSolver::SpectralSolver( +SpectralSolver::SpectralSolver ( const int lev, const amrex::BoxArray& realspace_ba, const amrex::DistributionMapping& dm, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type, + const int norder_x, + const int norder_y, + const int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, const amrex::Vector& v_comoving, const amrex::RealVect dx, const amrex::Real dt, diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H index 30bfe5cf16c..2e94fe95da2 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H @@ -12,6 +12,9 @@ #include "SpectralAlgorithms/SpectralBaseAlgorithmRZ.H" #include "SpectralFieldDataRZ.H" +#include + + /* \brief Top-level class for the electromagnetic spectral solver * * Stores the field in spectral space, and has member functions @@ -31,7 +34,8 @@ class SpectralSolverRZ amrex::BoxArray const & realspace_ba, amrex::DistributionMapping const & dm, int n_rz_azimuthal_modes, - int norder_z, short grid_type, + int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::RealVect dx, amrex::Real dt, bool with_pml, diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp index 9b0d2f21f94..3529247ef56 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp @@ -29,7 +29,8 @@ SpectralSolverRZ::SpectralSolverRZ (const int lev, amrex::BoxArray const & realspace_ba, amrex::DistributionMapping const & dm, int const n_rz_azimuthal_modes, - int const norder_z, short const grid_type, + int const norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::RealVect const dx, amrex::Real const dt, bool const with_pml, diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 561456943f1..c70bd6d3a35 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -7,6 +7,8 @@ #ifndef WARPX_GUARDCELLMANAGER_H_ #define WARPX_GUARDCELLMANAGER_H_ +#include + #include #include #include @@ -55,7 +57,7 @@ public: const amrex::Real * dx, bool do_subcycling, bool do_fdtd_nci_corr, - short grid_type, + ablastr::utils::enums::GridType grid_type, bool do_moving_window, int moving_window_dir, int nox, diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index bf010736853..4c2f978484d 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -36,7 +36,7 @@ guardCellManager::Init ( const amrex::Real *dx, const bool do_subcycling, const bool do_fdtd_nci_corr, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const bool do_moving_window, const int moving_window_dir, const int nox, @@ -200,6 +200,7 @@ guardCellManager::Init ( // currents in the latter case). This does not seem to be necessary in x and y, // where it still seems fine to set half the number of guard cells of the nodal case. + using namespace ablastr::utils::enums; int ngFFt_x = (grid_type == GridType::Collocated) ? nox_fft : nox_fft / 2; int ngFFt_y = (grid_type == GridType::Collocated) ? noy_fft : noy_fft / 2; int ngFFt_z = (grid_type == GridType::Collocated || galilean) ? noz_fft : noz_fft / 2; diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index 527b7766895..b9105557462 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -9,9 +9,13 @@ #define WARPX_UTILS_WARPXALGORITHMSELECTION_H_ #include +#include #include +using namespace ablastr::utils::enums; // NOLINT(google-global-names-in-headers) + + /** * \brief struct to determine the computational medium, i.e., vacuum or material/macroscopic default is vacuum. @@ -47,20 +51,6 @@ struct MacroscopicSolverAlgo { }; }; -struct GridType { - enum { - Collocated = 0, - Staggered = 1, - Hybrid = 2 - }; -}; - -enum struct PatchType -{ - fine, - coarse -}; - struct ElectromagneticSolverAlgo { enum { None = 0, diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index 89784c3b86c..9e2360315b8 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -10,8 +10,9 @@ #include "WarpXAlgorithmSelection.H" #include "Utils/TextMsg.H" -#include +#include +#include #include #include @@ -31,10 +32,10 @@ const std::map evolve_scheme_to_int = { }; const std::map grid_to_int = { - {"collocated", GridType::Collocated}, - {"staggered", GridType::Staggered}, - {"hybrid", GridType::Hybrid}, - {"default", GridType::Staggered} + {"collocated", static_cast(ablastr::utils::enums::GridType::Collocated)}, + {"staggered", static_cast(ablastr::utils::enums::GridType::Staggered)}, + {"hybrid", static_cast(ablastr::utils::enums::GridType::Hybrid)}, + {"default", static_cast(ablastr::utils::enums::GridType::Staggered)} }; const std::map electromagnetic_solver_algo_to_int = { @@ -153,8 +154,8 @@ const std::map ReductionType_algo_to_int = { }; int -GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ - +GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ) +{ // Read user input ; use "default" if it is not found std::string algo = "default"; pp.query( pp_search_key, algo ); diff --git a/Source/WarpX.H b/Source/WarpX.H index 3372dd869cc..c10be712a22 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -49,6 +49,8 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/export.H" +#include + #include #include #include @@ -371,7 +373,7 @@ public: //! Integer that corresponds to the type of grid used in the simulation //! (collocated, staggered, hybrid) - static short grid_type; + static ablastr::utils::enums::GridType grid_type; // Global rho nodal flag to know about rho index type when rho MultiFab is not allocated amrex::IntVect m_rho_nodal_flag; @@ -1085,7 +1087,7 @@ public: * \param[in] n_order order of the finite-difference approximation * \param[in] a_grid_type type of grid (collocated or not) */ - static amrex::Vector getFornbergStencilCoefficients(int n_order, short a_grid_type); + static amrex::Vector getFornbergStencilCoefficients (int n_order, ablastr::utils::enums::GridType a_grid_type); // Device vectors of stencil coefficients used for finite-order centering of fields amrex::Gpu::DeviceVector device_field_centering_stencil_coeffs_x; @@ -1441,7 +1443,7 @@ private: int centering_nox, int centering_noy, int centering_noz, - short a_grid_type); + ablastr::utils::enums::GridType a_grid_type); void AllocLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, const amrex::IntVect& ngEB, amrex::IntVect& ngJ, diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index b725431c2f1..ca45e883089 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -209,7 +209,7 @@ IntVect WarpX::filter_npass_each_dir(1); int WarpX::n_field_gather_buffer = -1; int WarpX::n_current_deposition_buffer = -1; -short WarpX::grid_type; +ablastr::utils::enums::GridType WarpX::grid_type; amrex::IntVect m_rho_nodal_flag; WarpX* WarpX::m_instance = nullptr; @@ -1066,7 +1066,7 @@ WarpX::ReadParameters () // Integer that corresponds to the type of grid used in the simulation // (collocated, staggered, hybrid) - grid_type = static_cast(GetAlgorithmInteger(pp_warpx, "grid_type")); + grid_type = static_cast(GetAlgorithmInteger(pp_warpx, "grid_type")); // Use same shape factors in all directions, for gathering if (grid_type == GridType::Collocated) { galerkin_interpolation = false; } @@ -3203,7 +3203,7 @@ WarpX::BuildBufferMasksInBox ( const amrex::Box tbx, amrex::IArrayBox &buffer_ma }); } -amrex::Vector WarpX::getFornbergStencilCoefficients(const int n_order, const short a_grid_type) +amrex::Vector WarpX::getFornbergStencilCoefficients (const int n_order, ablastr::utils::enums::GridType a_grid_type) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(n_order % 2 == 0, "n_order must be even"); @@ -3265,7 +3265,7 @@ void WarpX::AllocateCenteringCoefficients (amrex::Gpu::DeviceVector const int centering_nox, const int centering_noy, const int centering_noz, - const short a_grid_type) + ablastr::utils::enums::GridType a_grid_type) { // Vectors of Fornberg stencil coefficients amrex::Vector Fornberg_stencil_coeffs_x; diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index 589c6ec1835..26a4c72208d 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -9,6 +9,7 @@ #include #include +#include #include #include #include @@ -81,6 +82,7 @@ namespace ablastr::fields { * \param[in] geom the geometry per level (e.g., from AmrMesh) * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) + * \param[in] grid_type Integer that corresponds to the type of grid used in the simulation (collocated, staggered, hybrid) * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler * \param[in] is_solver_igf_on_lev0 boolean to select the Poisson solver: 1 for FFT on level 0 & Multigrid on other levels, 0 for Multigrid on all levels * \param[in] do_single_precision_comms perform communications in single precision @@ -105,6 +107,7 @@ computePhi (amrex::Vector const & rho, amrex::Vector const& geom, amrex::Vector const& dmap, amrex::Vector const& grids, + utils::enums::GridType grid_type, T_BoundaryHandler const boundary_handler, bool is_solver_igf_on_lev0, bool const do_single_precision_comms = false, @@ -263,7 +266,7 @@ computePhi (amrex::Vector const & rho, mlmg.setVerbose(verbosity); mlmg.setMaxIter(max_iters); mlmg.setAlwaysUseBNorm(always_use_bnorm); - if (WarpX::grid_type == GridType::Collocated) { + if (grid_type == utils::enums::GridType::Collocated) { // In this case, computeE needs to use ghost nodes data. So we // ask MLMG to fill BC for us after it solves the problem. mlmg.setFinalFillBC(true); @@ -285,7 +288,7 @@ computePhi (amrex::Vector const & rho, const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; ba.coarsen(refratio); const int ncomp = linop.getNComp(); - const int ng = (WarpX::grid_type == GridType::Collocated) ? 1 : 0; + const int ng = (grid_type == utils::enums::GridType::Collocated) ? 1 : 0; amrex::MultiFab phi_cp(ba, phi[lev+1]->DistributionMap(), ncomp, ng); if (ng > 0) { // Set all values outside the domain to zero diff --git a/Source/ablastr/utils/Enums.H b/Source/ablastr/utils/Enums.H new file mode 100644 index 00000000000..1f89bede9e4 --- /dev/null +++ b/Source/ablastr/utils/Enums.H @@ -0,0 +1,35 @@ +/* Copyright 2024 Axel Huebl + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef ABLASTR_UTILS_ENUMS_H_ +#define ABLASTR_UTILS_ENUMS_H_ + +namespace ablastr::utils::enums +{ + /** Type of grids used in a simulation: + * + * Collocated at the same location (AMReX: all "NODAL"), staggered (Yee-style), or hybrid. + */ + enum struct GridType { + Collocated = 0, + Staggered = 1, + Hybrid = 2 + }; + + /** Mesh-refinement patch + * + * The fine or coarse patch (in terms of spatial resolution) on the same MR level. + * https://warpx.readthedocs.io/en/latest/theory/amr.html + */ + enum struct PatchType { + fine, + coarse + }; + +} // namespace ablastr::utils::enums + +#endif // ABLASTR_UTILS_ENUMS_H_ From 6d5486405d543e169ac923688b7f82d8bcb6a619 Mon Sep 17 00:00:00 2001 From: David Grote Date: Thu, 1 Aug 2024 13:19:05 -0700 Subject: [PATCH 185/190] PICMI interface for implicit solvers (#5101) * Implement PICMI interface for implicit solvers * Add CI test * Add ExplicitEvolveScheme * Add evolve scheme classes to PICMI documentation * Made evolve_scheme argument to Simulation * Small fix, removing unused variable * Apply suggestions from code review --------- Co-authored-by: Remi Lehe --- Docs/source/usage/python.rst | 20 ++ .../Implicit/PICMI_inputs_vandb_jfnk_2d.py | 151 ++++++++++++++ Python/pywarpx/WarpX.py | 13 +- Python/pywarpx/picmi.py | 195 ++++++++++++++++++ .../ThetaImplicitJFNK_VandB_2d_PICMI.json | 31 +++ Regression/WarpX-tests.ini | 16 ++ 6 files changed, 425 insertions(+), 1 deletion(-) create mode 100755 Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py create mode 100644 Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index c068447ddbc..38b0a31d7f3 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -57,6 +57,26 @@ Object that allows smoothing of fields. .. autoclass:: pywarpx.picmi.BinomialSmoother +Evolve Schemes +-------------- + +These define the scheme use to evolve the fields and particles. +An instance of one of these would be passed as the `evolve_scheme` into the `Simulation`. + +.. autoclass:: pywarpx.picmi.ExplicitEvolveScheme + +.. autoclass:: pywarpx.picmi.ThetaImplicitEMEvolveScheme + +.. autoclass:: pywarpx.picmi.SemiImplicitEMEvolveScheme + +There are several support classes use to specify components of the evolve schemes + +.. autoclass:: pywarpx.picmi.PicardNonlinearSolver + +.. autoclass:: pywarpx.picmi.NewtonNonlinearSolver + +.. autoclass:: pywarpx.picmi.GMRESLinearSolver + Constants --------- diff --git a/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py b/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py new file mode 100755 index 00000000000..2f919124e13 --- /dev/null +++ b/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# +# --- Tests the python interface to the Implicit solver + +import numpy as np + +from pywarpx import picmi + +constants = picmi.constants + +########################## +# physics parameters +########################## + +n0 = 1.e30 # m^-3 +Ti = 100. # eV +Te = 100. # eV +wpe = constants.q_e*np.sqrt(n0/(constants.m_e*constants.ep0)) +de0 = constants.c/wpe +nppcz = 10 # number of particles/cell in z +dt = 0.1/wpe # s + +vthe = np.sqrt(Te*constants.q_e/constants.m_e) +vthi = np.sqrt(Ti*constants.q_e/constants.m_p) + +########################## +# numerics parameters +########################## + +# --- Number of time steps +max_steps = 20 +diagnostic_intervals = "::20" + +# --- Grid +nx = 40 +ny = 40 + +xmin = 0. +ymin = 0. +xmax = 10.0*de0 +ymax = 10.0*de0 + +number_per_cell_each_dim = [nppcz, nppcz] + +########################## +# physics components +########################## + +electrons_uniform_plasma = picmi.UniformDistribution(density = n0, + rms_velocity = [vthe, vthe, vthe]) + +electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=electrons_uniform_plasma) + +protons_uniform_plasma = picmi.UniformDistribution(density = n0, + rms_velocity = [vthi, vthi, vthi]) + +protons = picmi.Species(particle_type='proton', name='protons', initial_distribution=protons_uniform_plasma) + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid(number_of_cells = [nx, ny], + lower_bound = [xmin, ymin], + upper_bound = [xmax, ymax], + lower_boundary_conditions = ['periodic', 'periodic'], + upper_boundary_conditions = ['periodic', 'periodic'], + warpx_max_grid_size = 8, + warpx_blocking_factor = 8) + +solver = picmi.ElectromagneticSolver(grid = grid, + method = 'Yee') + +GMRES_solver = picmi.GMRESLinearSolver(verbose_int = 2, + max_iterations = 1000, + relative_tolerance = 1.0e-8, + absolute_tolerance = 0.0) + +newton_solver = picmi.NewtonNonlinearSolver(verbose = True, + max_iterations = 20, + relative_tolerance = 1.0e-12, + absolute_tolerance = 0.0, + require_convergence = False, + linear_solver = GMRES_solver, + max_particle_iterations = 21, + particle_tolerance = 1.0e-12) + +evolve_scheme = picmi.ThetaImplicitEMEvolveScheme(theta = 0.5, + nonlinear_solver = newton_solver) + +########################## +# diagnostics +########################## + +field_diag1 = picmi.FieldDiagnostic(name = 'diag1', + grid = grid, + period = diagnostic_intervals, + data_list = ['Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Jx', 'Jy', 'Jz', 'rho', 'divE'], + write_dir = '.', + warpx_file_prefix = 'ThetaImplicitJFNK_VandB_2d_PICMI_plt') + +part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', + period = diagnostic_intervals, + species = [electrons, protons], + data_list = ['weighting', 'position', 'momentum'], + write_dir = '.', + warpx_file_prefix = 'ThetaImplicitJFNK_VandB_2d_PICMI_plt') + +particle_energy_diag = picmi.ReducedDiagnostic(diag_type = 'ParticleEnergy', + name = 'particle_energy', + period = 1) + +field_energy_diag = picmi.ReducedDiagnostic(diag_type = 'FieldEnergy', + name = 'field_energy', + period = 1) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation(solver = solver, + particle_shape = 2, + time_step_size = dt, + max_steps = max_steps, + verbose = 1, + warpx_evolve_scheme = evolve_scheme, + warpx_current_deposition_algo = 'villasenor', + warpx_particle_pusher_algo = 'boris', + warpx_serialize_initial_conditions = 1, + warpx_use_filter = 0) + +sim.add_species(electrons, + layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) +sim.add_species(protons, + layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) + +sim.add_diagnostic(field_diag1) +sim.add_diagnostic(part_diag1) +sim.add_diagnostic(particle_energy_diag) +sim.add_diagnostic(field_energy_diag) + +########################## +# simulation run +########################## + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +sim.write_input_file(file_name = 'inputs2d_from_PICMI') + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() diff --git a/Python/pywarpx/WarpX.py b/Python/pywarpx/WarpX.py index 6b24b35e918..6752b00f371 100644 --- a/Python/pywarpx/WarpX.py +++ b/Python/pywarpx/WarpX.py @@ -89,8 +89,19 @@ def create_argv_list(self, **kw): for diagnostic in reduced_diagnostics._diagnostics_dict.values(): argv += diagnostic.attrlist() + for bucket in self._bucket_dict.values(): + argv += bucket.attrlist() + return argv + def get_bucket(self, bucket_name): + try: + return self._bucket_dict[bucket_name] + except KeyError: + bucket = Bucket(bucket_name) + self._bucket_dict[bucket_name] = bucket + return bucket + def init(self, mpi_comm=None, **kw): # note: argv[0] needs to be an absolute path so it works with AMReX backtraces # https://github.com/AMReX-Codes/amrex/issues/3435 @@ -130,4 +141,4 @@ def write_inputs(self, filename='inputs', **kw): ff.write(f'{arg}\n') -warpx = WarpX('warpx') +warpx = WarpX('warpx', _bucket_dict = {}) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 2aaa6b6122b..8be6d9c6212 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1233,6 +1233,194 @@ def solver_initialize_inputs(self): pywarpx.warpx.do_pml_j_damping = self.do_pml_j_damping +class ExplicitEvolveScheme(picmistandard.base._ClassWithInit): + """ + Sets up the explicit evolve scheme + """ + def solver_scheme_initialize_inputs(self): + pywarpx.algo.evolve_scheme = 'explicit' + + +class ThetaImplicitEMEvolveScheme(picmistandard.base._ClassWithInit): + """ + Sets up the "theta implicit" electromagnetic evolve scheme + + Parameters + ---------- + nonlinear_solver: nonlinear solver instance + The nonlinear solver to use for the iterations + + theta: float, optional + The "theta" parameter, determining the level of implicitness + """ + def __init__(self, nonlinear_solver, theta = None): + self.nonlinear_solver = nonlinear_solver + self.theta = theta + + def solver_scheme_initialize_inputs(self): + pywarpx.algo.evolve_scheme = 'theta_implicit_em' + implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') + implicit_evolve.theta = self.theta + + self.nonlinear_solver.nonlinear_solver_initialize_inputs() + + +class SemiImplicitEMEvolveScheme(picmistandard.base._ClassWithInit): + """ + Sets up the "semi-implicit" electromagnetic evolve scheme + + Parameters + ---------- + nonlinear_solver: nonlinear solver instance + The nonlinear solver to use for the iterations + """ + def __init__(self, nonlinear_solver): + self.nonlinear_solver = nonlinear_solver + + def solver_scheme_initialize_inputs(self): + pywarpx.algo.evolve_scheme = 'semi_implicit_em' + + self.nonlinear_solver.nonlinear_solver_initialize_inputs() + + +class PicardNonlinearSolver(picmistandard.base._ClassWithInit): + """ + Sets up the iterative Picard nonlinear solver for the implicit evolve scheme + + Parameters + ---------- + verbose: bool, default=True + Whether there is verbose output from the solver + + absolute_tolerance: float, default=0. + Absoluate tolerence of the convergence + + relative_tolerance: float, default=1.e-6 + Relative tolerance of the convergence + + max_iterations: integer, default=100 + Maximum number of iterations + + require_convergence: bool, default True + Whether convergence is required. If True and convergence is not obtained, the code will exit. + """ + def __init__(self, verbose=None, absolute_tolerance=None, relative_tolerance=None, + max_iterations=None, require_convergence=None): + self.verbose = verbose + self.absolute_tolerance = absolute_tolerance + self.relative_tolerance = relative_tolerance + self.max_iterations = max_iterations + self.require_convergence = require_convergence + + def nonlinear_solver_initialize_inputs(self): + implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') + implicit_evolve.nonlinear_solver = 'picard' + + picard = pywarpx.warpx.get_bucket('picard') + picard.verbose = self.verbose + picard.absolute_tolerance = self.absolute_tolerance + picard.relative_tolerance = self.relative_tolerance + picard.max_iterations = self.max_iterations + picard.require_convergence = self.require_convergence + + +class NewtonNonlinearSolver(picmistandard.base._ClassWithInit): + """ + Sets up the iterative Newton nonlinear solver for the implicit evolve scheme + + Parameters + ---------- + verbose: bool, default=True + Whether there is verbose output from the solver + + absolute_tolerance: float, default=0. + Absoluate tolerence of the convergence + + relative_tolerance: float, default=1.e-6 + Relative tolerance of the convergence + + max_iterations: integer, default=100 + Maximum number of iterations + + require_convergence: bool, default True + Whether convergence is required. If True and convergence is not obtained, the code will exit. + + linear_solver: linear solver instance, optional + Specifies input arguments to the linear solver + + max_particle_iterations: integer, optional + The maximum number of particle iterations + + particle_tolerance: float, optional + The tolerance of parrticle quantities for convergence + + """ + def __init__(self, verbose=None, absolute_tolerance=None, relative_tolerance=None, + max_iterations=None, require_convergence=None, linear_solver=None, + max_particle_iterations=None, particle_tolerance=None): + self.verbose = verbose + self.absolute_tolerance = absolute_tolerance + self.relative_tolerance = relative_tolerance + self.max_iterations = max_iterations + self.require_convergence = require_convergence + self.linear_solver = linear_solver + self.max_particle_iterations = max_particle_iterations + self.particle_tolerance = particle_tolerance + + def nonlinear_solver_initialize_inputs(self): + implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') + implicit_evolve.nonlinear_solver = 'newton' + implicit_evolve.max_particle_iterations = self.max_particle_iterations + implicit_evolve.particle_tolerance = self.particle_tolerance + + newton = pywarpx.warpx.get_bucket('newton') + newton.verbose = self.verbose + newton.absolute_tolerance = self.absolute_tolerance + newton.relative_tolerance = self.relative_tolerance + newton.max_iterations = self.max_iterations + newton.require_convergence = self.require_convergence + + self.linear_solver.linear_solver_initialize_inputs() + + +class GMRESLinearSolver(picmistandard.base._ClassWithInit): + """ + Sets up the iterative GMRES linear solver for the implicit Newton nonlinear solver + + Parameters + ---------- + verbose_int: integer, default=2 + Level of verbosity of output + + restart_length: integer, default=30 + How often to restart the GMRES iterations + + absolute_tolerance: float, default=0. + Absoluate tolerence of the convergence + + relative_tolerance: float, default=1.e-4 + Relative tolerance of the convergence + + max_iterations: integer, default=1000 + Maximum number of iterations + """ + def __init__(self, verbose_int=None, restart_length=None, absolute_tolerance=None, relative_tolerance=None, + max_iterations=None): + self.verbose_int = verbose_int + self.restart_length = restart_length + self.absolute_tolerance = absolute_tolerance + self.relative_tolerance = relative_tolerance + self.max_iterations = max_iterations + + def linear_solver_initialize_inputs(self): + gmres = pywarpx.warpx.get_bucket('gmres') + gmres.verbose_int = self.verbose_int + gmres.restart_length = self.restart_length + gmres.absolute_tolerance = self.absolute_tolerance + gmres.relative_tolerance = self.relative_tolerance + gmres.max_iterations = self.max_iterations + + class HybridPICSolver(picmistandard.base._ClassWithInit): """ Hybrid-PIC solver based on Ohm's law. @@ -1883,6 +2071,9 @@ class Simulation(picmistandard.PICMI_Simulation): Parameters ---------- + warpx_evolve_scheme: solver scheme instance, optional + Which evolve scheme to use + warpx_current_deposition_algo: {'direct', 'esirkepov', and 'vay'}, optional Current deposition algorithm. The default depends on conditions. @@ -2046,6 +2237,7 @@ class Simulation(picmistandard.PICMI_Simulation): def init(self, kw): + self.evolve_scheme = kw.pop('warpx_evolve_scheme', None) self.current_deposition_algo = kw.pop('warpx_current_deposition_algo', None) self.charge_deposition_algo = kw.pop('warpx_charge_deposition_algo', None) self.field_gathering_algo = kw.pop('warpx_field_gathering_algo', None) @@ -2113,6 +2305,9 @@ def initialize_inputs(self): pywarpx.warpx.sort_idx_type = self.sort_idx_type pywarpx.warpx.sort_bin_size = self.sort_bin_size + if self.evolve_scheme is not None: + self.evolve_scheme.solver_scheme_initialize_inputs() + pywarpx.algo.current_deposition = self.current_deposition_algo pywarpx.algo.charge_deposition = self.charge_deposition_algo pywarpx.algo.field_gathering = self.field_gathering_algo diff --git a/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json b/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json new file mode 100644 index 00000000000..d97eb04883f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 72730.70321925254, + "By": 89276.6097395453, + "Bz": 66911.00019634314, + "Ex": 92036838733000.64, + "Ey": 15583500940725.84, + "Ez": 89163420502164.97, + "divE": 8.998871921763322e+22, + "jx": 2.7748639888523993e+19, + "jy": 2.9501400595579277e+19, + "jz": 2.6976140199337787e+19, + "rho": 796777020986.2787 + }, + "protons": { + "particle_momentum_x": 2.0873315539608036e-17, + "particle_momentum_y": 2.0858882907322405e-17, + "particle_momentum_z": 2.0877345477243595e-17, + "particle_position_x": 0.004251275869323399, + "particle_position_y": 0.0042512738905209615, + "particle_weight": 2823958719279159.5 + }, + "electrons": { + "particle_momentum_x": 4.882673707817137e-19, + "particle_momentum_y": 4.879672470952739e-19, + "particle_momentum_z": 4.872329687213274e-19, + "particle_position_x": 0.004251641684258687, + "particle_position_y": 0.004251751978637919, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 8048318de7a..96dbea5c380 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3864,6 +3864,22 @@ useOMP = 0 numthreads = 1 analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py +[ThetaImplicitJFNK_VandB_2d_PICMI] +buildDir = . +inputFile = Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_vandb_jfnk_2d.py +dim = 2 +addToCompileString = USE_PYTHON_MAIN=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PYTHON=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py + [SemiImplicitPicard_1d] buildDir = . inputFile = Examples/Tests/Implicit/inputs_1d_semiimplicit From f1f3c385e0cfd277c0662e41687a7b4db5eb234a Mon Sep 17 00:00:00 2001 From: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Date: Thu, 1 Aug 2024 19:58:10 -0700 Subject: [PATCH 186/190] Treat external field types consistently (#5104) * use external field multifabs consistently for different external field types * make external field MF vectors length equal to `nlevs_max` * avoid adding external grid fields multiple times * load parse ext grid fields after restart * rename `LoadExternalFieldsFromFile` to `LoadExternalFields` * Update inline comment on why external fields are added to ES solutions Co-authored-by: Remi Lehe --------- Co-authored-by: Remi Lehe --- Source/Evolve/WarpXEvolve.cpp | 7 +- Source/Initialization/WarpXInitData.cpp | 131 +++++++++++++----------- Source/WarpX.H | 4 +- Source/WarpX.cpp | 10 +- 4 files changed, 82 insertions(+), 70 deletions(-) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 7bdba64dc4e..32f7a493916 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -246,7 +246,12 @@ WarpX::Evolve (int numsteps) // This is currently a lab frame calculation. ComputeMagnetostaticField(); } - AddExternalFields(); + // Since the fields were reset above, the external fields are added + // back on to the fine patch fields. This make it so that the net fields + // are the sum of the field solution and any external field. + for (int lev = 0; lev <= max_level; ++lev) { + AddExternalFields(lev); + } } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { // Hybrid-PIC case: // The particles are now at p^{n+1/2} and x^{n+1}. The fields diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index e9f63e34bb0..1d5e859c227 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -513,12 +513,6 @@ WarpX::InitData () if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { ComputeMagnetostaticField(); } - - // Set up an invariant condition through the rest of - // execution, that any code besides the field solver that - // looks at field values will see the composite of the field - // solution and any external field - AddExternalFields(); } if (restart_chkfile.empty() || write_diagnostics_on_restart) { @@ -539,15 +533,27 @@ WarpX::InitData () } void -WarpX::AddExternalFields () { - for (int lev = 0; lev <= finest_level; ++lev) { - // FIXME: RZ multimode has more than one component for all these - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { +WarpX::AddExternalFields (int const lev) { + // FIXME: RZ multimode has more than one component for all these + if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant) { + Efield_fp[lev][0]->plus(m_p_ext_field_params->E_external_grid[0], guard_cells.ng_alloc_EB.min()); + Efield_fp[lev][1]->plus(m_p_ext_field_params->E_external_grid[1], guard_cells.ng_alloc_EB.min()); + Efield_fp[lev][2]->plus(m_p_ext_field_params->E_external_grid[2], guard_cells.ng_alloc_EB.min()); + } + else { amrex::MultiFab::Add(*Efield_fp[lev][0], *Efield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][1], *Efield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][2], *Efield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); } - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + } + if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant) { + Bfield_fp[lev][0]->plus(m_p_ext_field_params->B_external_grid[0], guard_cells.ng_alloc_EB.min()); + Bfield_fp[lev][1]->plus(m_p_ext_field_params->B_external_grid[1], guard_cells.ng_alloc_EB.min()); + Bfield_fp[lev][2]->plus(m_p_ext_field_params->B_external_grid[2], guard_cells.ng_alloc_EB.min()); + } + else { amrex::MultiFab::Add(*Bfield_fp[lev][0], *Bfield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][1], *Bfield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][2], *Bfield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); @@ -788,7 +794,7 @@ WarpX::PostRestart () { mypc->PostRestart(); for (int lev = 0; lev <= maxLevel(); ++lev) { - LoadExternalFieldsFromFile(lev); + LoadExternalFields(lev); } } @@ -810,7 +816,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::default_zero; if ( is_B_ext_const && (lev <= maxlevel_extEMfield_init) ) { - Bfield_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); if (fft_do_time_averaging) { Bfield_avg_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); } @@ -831,7 +836,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::default_zero; if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) { - Efield_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); if (fft_do_time_averaging) { Efield_avg_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); } @@ -856,13 +860,12 @@ WarpX::InitLevelData (int lev, Real /*time*/) // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation if ((m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) - && (lev <= maxlevel_extEMfield_init)) { + && (lev > 0) && (lev <= maxlevel_extEMfield_init)) { - // Initialize Bfield_fp with external function InitializeExternalFieldsOnGridUsingParser( - Bfield_fp[lev][0].get(), - Bfield_fp[lev][1].get(), - Bfield_fp[lev][2].get(), + Bfield_aux[lev][0].get(), + Bfield_aux[lev][1].get(), + Bfield_aux[lev][2].get(), m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), @@ -871,31 +874,17 @@ WarpX::InitLevelData (int lev, Real /*time*/) 'B', lev, PatchType::fine); - if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser( - Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser( - Bfield_cp[lev][0].get(), - Bfield_cp[lev][1].get(), - Bfield_cp[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::coarse); - } + InitializeExternalFieldsOnGridUsingParser( + Bfield_cp[lev][0].get(), + Bfield_cp[lev][1].get(), + Bfield_cp[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::coarse); } // if the input string for the E-field is "parse_e_ext_grid_function", @@ -906,19 +895,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ((m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) && (lev <= maxlevel_extEMfield_init)) { - // Initialize Efield_fp with external function - InitializeExternalFieldsOnGridUsingParser( - Efield_fp[lev][0].get(), - Efield_fp[lev][1].get(), - Efield_fp[lev][2].get(), - m_p_ext_field_params->Exfield_parser->compile<3>(), - m_p_ext_field_params->Eyfield_parser->compile<3>(), - m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::fine); - #ifdef AMREX_USE_EB // We initialize ECTRhofield consistently with the Efield if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { @@ -963,7 +939,10 @@ WarpX::InitLevelData (int lev, Real /*time*/) } } - LoadExternalFieldsFromFile(lev); + // load external grid fields into E/Bfield_fp_external multifabs + LoadExternalFields(lev); + // add the external fields to the fine patch fields as initial conditions for the fields + AddExternalFields(lev); if (costs[lev]) { const auto iarr = costs[lev]->IndexArray(); @@ -1354,7 +1333,7 @@ void WarpX::CheckKnownIssues() } void -WarpX::LoadExternalFieldsFromFile (int const lev) +WarpX::LoadExternalFields (int const lev) { // External fields from file are currently not compatible with the moving window // In order to support the moving window, the MultiFab containing the external @@ -1370,8 +1349,21 @@ WarpX::LoadExternalFieldsFromFile (int const lev) } // External grid fields - - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + // Initialize Bfield_fp_external with external function + InitializeExternalFieldsOnGridUsingParser( + Bfield_fp_external[lev][0].get(), + Bfield_fp_external[lev][1].get(), + Bfield_fp_external[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::fine); + } + else if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); @@ -1384,7 +1376,22 @@ WarpX::LoadExternalFieldsFromFile (int const lev) ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); #endif } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { + + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + // Initialize Efield_fp_external with external function + InitializeExternalFieldsOnGridUsingParser( + Efield_fp_external[lev][0].get(), + Efield_fp_external[lev][1].get(), + Efield_fp_external[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::fine); + } + else if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); diff --git a/Source/WarpX.H b/Source/WarpX.H index c10be712a22..ad6e8a3abfd 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -1050,7 +1050,7 @@ public: * \brief Load field values from a user-specified openPMD file, * for the fields Ex, Ey, Ez, Bx, By, Bz */ - void LoadExternalFieldsFromFile (int lev); + void LoadExternalFields (int lev); /** * \brief Load field values from a user-specified openPMD file @@ -1293,7 +1293,7 @@ private: void FillBoundaryB_avg (int lev, PatchType patch_type, amrex::IntVect ng); void FillBoundaryE_avg (int lev, PatchType patch_type, amrex::IntVect ng); - void AddExternalFields (); + void AddExternalFields (int lev); void OneStep_nosub (amrex::Real cur_time); void OneStep_sub1 (amrex::Real cur_time); diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index ca45e883089..cd86587f2d8 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -285,7 +285,7 @@ WarpX::WarpX () // Loop over species (particles and lasers) // and set current injection position per species - if (do_moving_window){ + if (do_moving_window){ const int n_containers = mypc->nContainers(); for (int i=0; iB_ext_grid_type == ExternalFieldType::read_from_file) { + if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bfield_fp[lev][0]->ixType()), dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); @@ -2614,7 +2614,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(B_external_particle_field[lev][2], amrex::convert(ba, Bfield_aux[lev][2]->ixType()), dm, ncomps, ngEB, lev, "B_external_particle_field[z]", 0.0_rt); } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { + if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::constant) { // These fields will be added directly to the grid, i.e. to fp, and need to match the index type AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Efield_fp[lev][0]->ixType()), dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); From 2837f55d19cf1728751b788619911e44034bbe1e Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 6 Aug 2024 14:01:35 -0700 Subject: [PATCH 187/190] Update test to use lasy 0.5.0 (#5111) --- Examples/Tests/laser_injection_from_file/inputs.1d_boost_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.1d_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.2d_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.3d_test | 2 +- Examples/Tests/laser_injection_from_file/inputs.RZ_test | 2 +- .../Tests/laser_injection_from_file/inputs.from_RZ_file_test | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test index adfde819531..ffc1865ee0f 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test +++ b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test @@ -47,7 +47,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_test b/Examples/Tests/laser_injection_from_file/inputs.1d_test index 6124b4552a2..6c392883418 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.1d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.2d_test b/Examples/Tests/laser_injection_from_file/inputs.2d_test index a3003a4978d..e5814471753 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.2d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.2d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.3d_test b/Examples/Tests/laser_injection_from_file/inputs.3d_test index e3096bf92d9..ad8159cb650 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.3d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.3d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.RZ_test b/Examples/Tests/laser_injection_from_file/inputs.RZ_test index fe5bb675212..2a539883fec 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.RZ_test +++ b/Examples/Tests/laser_injection_from_file/inputs.RZ_test @@ -41,7 +41,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test index ca50430c21a..f92440188b7 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test +++ b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test @@ -40,7 +40,7 @@ lasy_RZ_laser.e_max = 1.e14 # Maximum amplitude of the laser field lasy_RZ_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_RZ_laser.profile = from_file lasy_RZ_laser.time_chunk_size = 50 -lasy_RZ_laser.lasy_file_name = "laguerrelaserRZ_00000.h5" +lasy_RZ_laser.lasy_file_name = "diags/laguerrelaserRZ_00000.h5" lasy_RZ_laser.delay = 0.0 # Diagnostics From 53605cd77f0c3478065dec270ca6bf5955795e2f Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 7 Aug 2024 08:56:11 -0700 Subject: [PATCH 188/190] Fix bug in `PrintMainPICparameters` (uninitialized values) (#5114) --- Source/WarpX.H | 2 +- Source/WarpX.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Source/WarpX.H b/Source/WarpX.H index ad6e8a3abfd..da60824945e 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -233,7 +233,7 @@ public: //! If true, a correction is applied to the current in Fourier space, // to satisfy the continuity equation and charge conservation - bool current_correction; + bool current_correction = true; //! If true, the PSATD update equation for E contains both J and rho //! (default is false for standard PSATD and true for Galilean PSATD) diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index cd86587f2d8..4d51dd283a3 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -1544,7 +1544,6 @@ WarpX::ReadParameters () // Current correction activated by default, unless a charge-conserving // current deposition (Esirkepov, Vay) or the div(E) cleaning scheme // are used - current_correction = true; if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor || WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay || From db2fbcda27c318a87a4de3c6ce4ae62904a2ef42 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Wed, 7 Aug 2024 10:59:17 -0500 Subject: [PATCH 189/190] Temporarily disable `openbc_poisson_solver` (#5117) This test uncoveres a bug of an invalid pc passed early to I/O routines. We will add it back with the fix. --- .../{inputs_3d => README} | 0 Regression/WarpX-tests.ini | 26 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) rename Examples/Tests/openbc_poisson_solver/{inputs_3d => README} (100%) diff --git a/Examples/Tests/openbc_poisson_solver/inputs_3d b/Examples/Tests/openbc_poisson_solver/README similarity index 100% rename from Examples/Tests/openbc_poisson_solver/inputs_3d rename to Examples/Tests/openbc_poisson_solver/README diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index 96dbea5c380..bc470d7706b 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3952,16 +3952,16 @@ useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py -[openbc_poisson_solver] -buildDir = . -inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d -runtime_params = warpx.abort_on_warning_threshold = high -dim = 3 -addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON -restartTest = 0 -useMPI = 1 -numprocs = 2 -useOMP = 1 -numthreads = 1 -analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py +#[openbc_poisson_solver] +#buildDir = . +#inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d +#runtime_params = warpx.abort_on_warning_threshold = high +#dim = 3 +#addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE +#cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON +#restartTest = 0 +#useMPI = 1 +#numprocs = 2 +#useOMP = 1 +#numthreads = 1 +#analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py From ffb16f3509b166d1a31b90b81c5bd19e24f54c7a Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 7 Aug 2024 13:09:51 -0700 Subject: [PATCH 190/190] CI: Clean out Pip Caches (#5110) * Debugging CI tests * Run all Cartesian 3D tests * Clear cache after python installation commands * Update Regression/WarpX-tests.ini * `run_test.sh`: Detailed `df -h` Prints * Update checksum * Cleanup --------- Co-authored-by: Remi Lehe Co-authored-by: Axel Huebl --- .../{README => inputs_3d} | 0 .../benchmarks_json/collisionXYZ.json | 28 +++++++++---------- Regression/WarpX-tests.ini | 26 ++++++++--------- run_test.sh | 1 + 4 files changed, 28 insertions(+), 27 deletions(-) rename Examples/Tests/openbc_poisson_solver/{README => inputs_3d} (100%) diff --git a/Examples/Tests/openbc_poisson_solver/README b/Examples/Tests/openbc_poisson_solver/inputs_3d similarity index 100% rename from Examples/Tests/openbc_poisson_solver/README rename to Examples/Tests/openbc_poisson_solver/inputs_3d diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index c13f404857a..726573ef8b2 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -6,25 +6,25 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0, - "T_electron": 381957.7394223898, - "T_ion": 319565.6269784763 + "T_electron": 351982.0169218243, + "T_ion": 349599.6939052666 }, "electron": { - "particle_momentum_x": 8.631833485301232e-19, - "particle_momentum_y": 8.476316745254719e-19, - "particle_momentum_z": 8.514139891331418e-19, - "particle_position_x": 21253105.73686561, - "particle_position_y": 21282643.519070115, - "particle_position_z": 21239057.457948968, + "particle_momentum_x": 8.359982321196841e-19, + "particle_momentum_y": 8.192841151167721e-19, + "particle_momentum_z": 8.182985690701241e-19, + "particle_position_x": 21255110.08090505, + "particle_position_y": 21303488.6242626, + "particle_position_z": 21238676.122703437, "particle_weight": 7.168263344048695e+28 }, "ion": { - "particle_momentum_x": 1.9215585867122464e-18, - "particle_momentum_y": 1.7471481568315848e-18, - "particle_momentum_z": 1.7510887207292533e-18, - "particle_position_x": 21228900.948879313, - "particle_position_y": 21304564.011731848, - "particle_position_z": 21250585.221808463, + "particle_momentum_x": 2.0034830240966893e-18, + "particle_momentum_y": 1.8323959076577197e-18, + "particle_momentum_z": 1.827953230828629e-18, + "particle_position_x": 21246214.748882487, + "particle_position_y": 21280709.710960124, + "particle_position_z": 21206153.002106402, "particle_weight": 7.168263344048695e+28 } } diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index bc470d7706b..96dbea5c380 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -3952,16 +3952,16 @@ useOMP = 1 numthreads = 1 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py -#[openbc_poisson_solver] -#buildDir = . -#inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d -#runtime_params = warpx.abort_on_warning_threshold = high -#dim = 3 -#addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE -#cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON -#restartTest = 0 -#useMPI = 1 -#numprocs = 2 -#useOMP = 1 -#numthreads = 1 -#analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py +[openbc_poisson_solver] +buildDir = . +inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d +runtime_params = warpx.abort_on_warning_threshold = high +dim = 3 +addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py diff --git a/run_test.sh b/run_test.sh index f56a957c17a..4efa86eb36c 100755 --- a/run_test.sh +++ b/run_test.sh @@ -65,6 +65,7 @@ python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools wheel python3 -m pip install --upgrade cmake python3 -m pip install --upgrade -r warpx/Regression/requirements.txt +python3 -m pip cache purge # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git